#!/usr/bin/env python3 """ HOOK: block-fabrication EVENT: PostToolUse MATCHER: Agent TIMEOUT: 10 PostToolUse hook: scan Agent output for unverifiable claims. Fires after the Agent tool returns. Extracts verifiable claims from agent output (file creation, test results, deploys, commits) and checks whether evidence exists. If 2+ claims are unverifiable, logs to fabrication-flags.jsonl and returns a warning. Does NOT block — warning only. """ import json import sys import os import re import time from pathlib import Path import _vance_bootstrap # noqa: F401 — adds VANCE_HOME to sys.path from vance_config import VANCE_HOME, CONTEXT FLAGS_FILE = CONTEXT / "fabrication-flags.jsonl" from datetime import datetime, timezone def extract_agent_output(tool_output): """Get text from tool_output — may be string or dict with .content.""" if isinstance(tool_output, str): return tool_output if isinstance(tool_output, dict): content = tool_output.get("content", "") if isinstance(content, str): return content if isinstance(content, list): parts = [] for block in content: if isinstance(block, dict): parts.append(block.get("text", str(block))) else: parts.append(str(block)) return "\n".join(parts) return str(tool_output) return str(tool_output) def extract_file_path(text, match_start, match_end): """Try to extract a file path near a file-creation claim.""" window_start = max(0, match_start - 20) window_end = min(len(text), match_end + 200) window = text[window_start:window_end] path_patterns = [ r'["`\']((?:[A-Za-z]:)?(?:[/\\][\w.\-~]+)+(?:\.\w+)?)["`\']', r'((?:[A-Za-z]:)?(?:[/\\][\w.\-~]+){2,}(?:\.\w+)?)', ] for pattern in path_patterns: m = re.search(pattern, window) if m: return m.group(1) return None def check_recent_file(filepath, max_age=300): """Check if a file exists and was modified within max_age seconds.""" try: p = Path(filepath) if not p.exists(): return False mtime = p.stat().st_mtime return (time.time() - mtime) < max_age except Exception: return False def scan_claims(text): """Scan text for verifiable claims. Returns list of (claim_type, detail, verified).""" claims = [] # 1. File creation claims file_pattern = re.compile( r'(?:created|wrote|new)\s+(?:the\s+)?file\s+', re.IGNORECASE ) for m in file_pattern.finditer(text): path = extract_file_path(text, m.start(), m.end()) if path: exists = os.path.exists(path) claims.append(("file_created", path, exists)) else: claims.append(("file_created", "(path not extracted)", False)) # 2. Test pass claims test_pattern = re.compile( r'(?:tests?\s+pass(?:ed|es|ing)?|all\s+tests?\s+pass|build\s+succeeded)', re.IGNORECASE ) if test_pattern.search(text): results_file = VANCE_HOME / "test-results.json" verified = check_recent_file(results_file, max_age=300) claims.append(("test_pass", str(results_file), verified)) # 3. Deploy claims deploy_pattern = re.compile( r'(?:deployed\s+successfully|deploy\s+complete)', re.IGNORECASE ) if deploy_pattern.search(text): deploy_log = VANCE_HOME / "deploy-log.jsonl" verified = check_recent_file(deploy_log, max_age=300) claims.append(("deploy_success", str(deploy_log), verified)) # 4. Git commit/push claims (best-effort flag) git_pattern = re.compile( r'\b(?:committed|pushed)\b', re.IGNORECASE ) if git_pattern.search(text): claims.append(("git_action", "committed/pushed (best-effort flag)", False)) return claims def log_flags(claims, verification_results): """Append a flag entry to fabrication-flags.jsonl.""" try: FLAGS_FILE.parent.mkdir(parents=True, exist_ok=True) entry = { "timestamp": datetime.now(timezone.utc).isoformat(), "claims": claims, "verification_results": verification_results, } with open(FLAGS_FILE, "a", encoding="utf-8") as f: f.write(json.dumps(entry) + "\n") except Exception: pass def main(): try: input_data = json.load(sys.stdin) except Exception: print(json.dumps({})) sys.exit(0) tool_name = input_data.get("tool_name", "") # Only fire for Agent tool if tool_name != "Agent": print(json.dumps({})) sys.exit(0) tool_output = input_data.get("tool_output", "") text = extract_agent_output(tool_output) if not text or len(text) < 10: print(json.dumps({})) sys.exit(0) # Scan for claims claims = scan_claims(text) if not claims: print(json.dumps({})) sys.exit(0) # Count unverified unverified = [(ctype, detail) for ctype, detail, verified in claims if not verified] if len(unverified) >= 2: verification_results = {} for ctype, detail, verified in claims: verification_results[f"{ctype}:{detail}"] = verified claim_list = [f"{ctype}: {detail}" for ctype, detail in unverified] log_flags(claim_list, verification_results) flagged = "; ".join(f"{ctype}: {detail}" for ctype, detail in unverified) print(json.dumps({ "message": ( f"FABRICATION CHECK: {len(unverified)} unverifiable claims in agent output. " f"Flagged: [{flagged}]. Logged to fabrication-flags.jsonl. Verify before trusting." ) })) else: print(json.dumps({})) sys.exit(0) if __name__ == "__main__": main()