From 1ef0ac753aaf581d8160a29da77f09917f1bcac8 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Tue, 28 Apr 2026 15:45:08 -0400 Subject: [PATCH] qa: post-LLM faithfulness verifier sets STRICT/HYBRID/VISUAL audit_mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds aborist/qa/verify.py — three-strategy verifier (explicit quotes, bullet/sentence spans, multi-word proper nouns) that lexically checks every claim against retrieved context under norm-v1 + lowercase. The strategy that fires is recorded as verifier_method for diagnostics. Entity strategy gates classification via an entity_policy (strict/hybrid/proximity/drop) so a single proper-noun match no longer overclaims STRICT — proximity (default) requires a cluster of 3+ verified entities within 300 chars. Wires into ask() and query(). System prompts now require verbatim quoted spans for every factual claim, restated via a user-turn grounding_reminder one message before sources arrive (recent user-turn instructions outweigh decayed system-turn rules under long context in Hermes). providence_cache gains 5 columns (audit_mode, n_quotes, n_verified, unverified_quotes, verifier_method) with CHECK constraints. A connect-time _migrate_audit_mode() ALTERs legacy DBs idempotently. Cache hits return the persisted audit_mode rather than asserting STRICT unconditionally. CLI: emergent list VISUAL/HYBRID records; --aggregate ranks unverified quotes by frequency (corpus-growth signal). reclassify re-run the verifier against live providence records; cold-source records skipped; --dry-run reports transitions without writing; each change writes one 'providence_reclassify' audit event. --- CLAUDE.md | 15 +- aborist/cli.py | 324 ++++++++++++++++++++++++++++ aborist/qa/query.py | 118 +++++++++-- aborist/qa/runner.py | 84 +++++++- aborist/qa/verify.py | 329 +++++++++++++++++++++++++++++ aborist/store.py | 64 +++++- tests/test_migration_audit_mode.py | 109 ++++++++++ tests/test_qa.py | 12 +- tests/test_query.py | 11 +- tests/test_reclassify.py | 215 +++++++++++++++++++ tests/test_verify.py | 305 ++++++++++++++++++++++++++ 11 files changed, 1550 insertions(+), 36 deletions(-) create mode 100644 aborist/qa/verify.py create mode 100644 tests/test_migration_audit_mode.py create mode 100644 tests/test_reclassify.py create mode 100644 tests/test_verify.py diff --git a/CLAUDE.md b/CLAUDE.md index 9b2393e..6910e02 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,11 +12,12 @@ Three layers stacked on one SQLite file: 1. **Surface** — ingested documents (Wikipedia dumps, HTML pages, anything with a URI). Chunked, Merkle-rooted, FTS5-indexed. 2. **Core** — distilled documents (haiku/keyword/equation-scale) Merkle-bound back to source surface(s) via per-chunk inclusion proofs in `derivations.proof_blob`. Recursive: cores derive from cores. The "planet toward center" compression. -3. **Providence cache** — Q&A records keyed on the v9.8 8-dim invariant. STRICT-mode hits carry verifiable Merkle proofs. +3. **Providence cache** — Q&A records keyed on the v9.8 8-dim invariant. Each record carries an `audit_mode` set by the post-LLM faithfulness verifier (`aborist/qa/verify.py`): STRICT (every quoted claim verbatim-grounded), HYBRID (mixed source/emergent), VISUAL (no verbatim grounding — purely emergent from training). ## Source papers (read first if confused) -- `~/Downloads/merkle-providence-reverse-rag-whitepaper.pdf` — public-facing spec, AGPL-3.0 +- `~/git/unfirehose-nextjs-logger/whitepaper/merkle-providence-reverse-rag-whitepaper.rst` — **canonical whitepaper source** (rst, builds the PDF). Edit here, not the PDF. +- `~/Downloads/merkle-providence-reverse-rag-whitepaper.pdf` — built artifact; mirrors the rst above - `~/Downloads/merkle-agi-dag_v7.txt` — formal substrate (TLV encoding A1, public quantization A2, collision-resistant hash A3, theorems T1–T5) - `~/git/proxy.unturf.com/pkg/verified/merkle.go` — fox's existing Go merkle implementation. **Aborist Python ports its conventions exactly.** - `~/git/proxy.unturf.com/docs/merkle-tree.md` — convention reference @@ -43,9 +44,11 @@ aborist/ ├── qa/ # Q&A: 8-dim cache_key + Merkle-bound answers │ ├── client.py # ChatClient + StubClient + OpenAICompat │ ├── keys.py # cache_key, question_hash, ... (pure functions) -│ └── runner.py # ask(): cache hit STRICT / miss -> infer -> write +│ ├── verify.py # verify_quotes(): post-LLM faithfulness check +│ └── runner.py # ask(): cache -> infer -> verify -> classify -> write └── cli.py # ingest / search / verify / stats / distill / - # evict / rehydrate / ask / providence / analyze + # evict / rehydrate / ask / providence / emergent / + # analyze ``` ## Build, test, run @@ -63,7 +66,7 @@ make verify # round-trip Merkle proofs on a random sample make analyze # compression spectrum + audit chain integrity ``` -`aborist/cli.py` adds: `analyze`, `distill --kind {surface,core}`, `evict`, `rehydrate`, `ask`, `providence`. `--batch-size` defaults to 200 docs/transaction; lower it only to bound memory peaks. +`aborist/cli.py` adds: `analyze`, `distill --kind {surface,core}`, `evict`, `rehydrate`, `ask`, `providence`, `emergent` (list VISUAL/HYBRID records or `--aggregate` to rank unverified quotes — corpus-growth signal). `--batch-size` defaults to 200 docs/transaction; lower it only to bound memory peaks. ## Schema invariants (do not break) @@ -79,7 +82,7 @@ make analyze # compression spectrum + audit chain integrity - **Chunker default = `tok-512-v1`.** Changing the default bumps `chunking_version` and **stales every prior cache record**. Add a new chunker as a new `name` instead. - **Canonicalization = `norm-v1`** (NFC, collapsed whitespace). Same rule. - **Schema = `v9.8.0`.** Same rule. -- **Search hits carry `audit_mode`** ∈ `STRICT`/`HYBRID`/`VISUAL`. FTS5 hits are `VISUAL`. Providence-cache hits with verified proofs are `STRICT`. Never overclaim. +- **`audit_mode` is decided by the verifier, never asserted unconditionally.** Trichotomy: STRICT = every double-quoted span in the answer matches verbatim against `context` (post-LLM, lexical, NFC + collapsed-ws + lowercase). HYBRID = some quotes verify, some don't (the model mixed source with training). VISUAL = no quotes, or none verify (raw FTS5 hits, or an answer that emerged from training alone). Persisted on `providence_cache.audit_mode`; cache-hits return the stored mode. Never overclaim — STRICT is a verifiable claim, not a default. - **Soft hash vs hard hash.** Hard = SHA-256 (commitments, proofs, cache_key). Soft = embeddings/TF-IDF/similarity (training, ranking, distillation candidate selection). Never mix — soft never enters proof path. ## Live endpoints diff --git a/aborist/cli.py b/aborist/cli.py index 4e45e1d..e66d2b9 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -502,6 +502,285 @@ def _cmd_providence(args: argparse.Namespace) -> int: return 0 +def _load_record_context(row, shards_dir, qa_db): + """Reassemble context for a providence record. Returns text or None + if any source doc has no hot chunks (cold).""" + from aborist.qa.query import _load_doc_text + + proof = json.loads(row["merkle_proof"]) + sources = proof.get("sources", []) + if not sources: + return None + parts: list[str] = [] + for src in sources: + shard_name = src.get("shard") + if not shard_name: + return None + if shards_dir: + shard_path = shards_dir / shard_name + else: + shard_path = qa_db.parent / shard_name + if not shard_path.exists(): + return None + text = _load_doc_text(str(shard_path), src["document_root"]) + if not text: + return None + parts.append(text) + return "\n\n".join(parts) + + +def _cmd_reclassify(args: argparse.Namespace) -> int: + """Re-run the layered verifier against existing live providence records. + + Reads each record's answer + reassembles its context from + merkle_proof.sources, runs verify_quotes(), and updates the row only + if the verdict differs from what's stored. No LLM calls — this just + relabels existing answers under the current verifier. + + Cold-source records (where any source doc has no hot chunks) are + skipped: we can't faithfully reclassify without the original context. + Run `aborist rehydrate` first if you want those covered too. + + `--compare` runs all four entity policies side-by-side without + writing — use it to see what each policy would produce on real data + before committing to one. `--entity-policy X` writes under a single + policy. + + Each changed record gets one 'providence_reclassify' audit event + with old & new state for chain-of-custody. + """ + import time + from collections import defaultdict + + from aborist.qa.verify import ( + DEFAULT_ENTITY_POLICY, + ENTITY_POLICIES, + verify_quotes, + ) + + qa_db = args.qa_db + if qa_db is None: + if args.global_shards_dir: + qa_db = Path(args.global_shards_dir) / "qa.db" + else: + qa_db = Path.home() / ".aborist" / "qa.db" + qa_db = Path(qa_db) + + shards_dir = ( + Path(args.global_shards_dir) if args.global_shards_dir else None + ) + + conn = connect(qa_db) + try: + sql = ( + "SELECT cache_key, answer_text, merkle_proof, audit_mode, " + " verifier_method, n_quotes, n_verified, unverified_quotes, " + " question_text " + "FROM providence_cache " + "WHERE falsification_state = 'live' " + "ORDER BY created_at DESC" + ) + if args.limit: + sql += f" LIMIT {int(args.limit)}" + rows = conn.execute(sql).fetchall() + + if args.compare: + # Run all four policies side-by-side, no DB write. Output is a + # per-record grid + a per-policy distribution summary so fox can + # eyeball where the policies disagree. + grid = [] + distribution: dict[str, dict[str, int]] = { + p: defaultdict(int) for p in ENTITY_POLICIES + } + skipped_cold = 0 + for row in rows: + context = _load_record_context(row, shards_dir, qa_db) + if context is None: + skipped_cold += 1 + continue + per_policy = {} + for p in ENTITY_POLICIES: + v = verify_quotes(row["answer_text"], context, entity_policy=p) + label = f"{v['audit_mode']}/{v['verifier_method']}" + per_policy[p] = label + distribution[p][label] += 1 + grid.append({ + "cache_key": row["cache_key"][:16] + "…", + "question": row["question_text"][:55], + **per_policy, + }) + print(json.dumps({ + "examined": len(grid), + "skipped_cold": skipped_cold, + "distribution": {p: dict(d) for p, d in distribution.items()}, + "records": grid, + }, indent=2)) + return 0 + + # Single-policy reclassify. Default tracks DEFAULT_ENTITY_POLICY + # so the CLI always matches the verifier's current contract. + policy_name = args.entity_policy or DEFAULT_ENTITY_POLICY + if policy_name not in ENTITY_POLICIES: + print( + f"--entity-policy must be one of {ENTITY_POLICIES}, " + f"got {policy_name!r}", + file=sys.stderr, + ) + return 2 + + summary = { + "examined": 0, + "changed": 0, + "skipped_cold": 0, + "unchanged": 0, + "entity_policy": policy_name, + "transitions": defaultdict(int), + } + + for row in rows: + summary["examined"] += 1 + context = _load_record_context(row, shards_dir, qa_db) + if context is None: + summary["skipped_cold"] += 1 + continue + + verdict = verify_quotes( + row["answer_text"], context, entity_policy=policy_name + ) + + old_unverified = row["unverified_quotes"] or "null" + new_unverified_blob = ( + json.dumps(verdict["unverified_quotes"], separators=(",", ":")) + if verdict["unverified_quotes"] + else None + ) + new_unverified_for_compare = new_unverified_blob or "null" + + unchanged = ( + verdict["audit_mode"] == row["audit_mode"] + and verdict["verifier_method"] == row["verifier_method"] + and verdict["n_quotes"] == row["n_quotes"] + and verdict["n_verified"] == row["n_verified"] + and old_unverified == new_unverified_for_compare + ) + if unchanged: + summary["unchanged"] += 1 + continue + + summary["changed"] += 1 + transition = ( + f"{row['audit_mode']}/{row['verifier_method']} " + f"-> {verdict['audit_mode']}/{verdict['verifier_method']}" + ) + summary["transitions"][transition] += 1 + + if args.dry_run: + continue + + now = int(time.time()) + with transaction(conn): + event_hash = append_audit( + conn, + event_type="providence_reclassify", + subject_root=row["cache_key"], + body={ + "old_audit_mode": row["audit_mode"], + "new_audit_mode": verdict["audit_mode"], + "old_method": row["verifier_method"], + "new_method": verdict["verifier_method"], + "old_n_verified": row["n_verified"], + "new_n_verified": verdict["n_verified"], + "entity_policy": policy_name, + }, + ts=now, + ) + conn.execute( + "UPDATE providence_cache SET " + " audit_mode = ?, n_quotes = ?, n_verified = ?, " + " unverified_quotes = ?, verifier_method = ?, " + " audit_event_hash = ? " + "WHERE cache_key = ?", + ( + verdict["audit_mode"], + verdict["n_quotes"], + verdict["n_verified"], + new_unverified_blob, + verdict["verifier_method"], + event_hash, + row["cache_key"], + ), + ) + finally: + conn.close() + + summary["transitions"] = dict(summary["transitions"]) + summary["dry_run"] = bool(args.dry_run) + print(json.dumps(summary, indent=2)) + return 0 + + +def _cmd_emergent(args: argparse.Namespace) -> int: + """Surface emergent claims from VISUAL/HYBRID providence records. + + These are spans the model produced that don't appear verbatim in the + corpus — candidate ingest targets. Frequent unverified quotes signal + knowledge the model has from training that our corpus is missing. + """ + conn = ( + connect_query(args.db, shards_dir=args.global_shards_dir) + if args.global_shards_dir + else connect(args.db) + ) + try: + if args.aggregate: + rows = conn.execute( + "SELECT unverified_quotes FROM providence_cache " + "WHERE audit_mode IN ('VISUAL','HYBRID') " + " AND falsification_state = 'live' " + " AND unverified_quotes IS NOT NULL" + ).fetchall() + counts: dict[str, int] = {} + for r in rows: + for q in json.loads(r["unverified_quotes"]): + counts[q] = counts.get(q, 0) + 1 + ranked = sorted(counts.items(), key=lambda kv: -kv[1])[: args.limit] + print(json.dumps( + [{"quote": q, "count": c} for q, c in ranked], + indent=2, + )) + else: + rows = conn.execute( + "SELECT cache_key, audit_mode, verifier_method, question_text, " + " n_quotes, n_verified, unverified_quotes, created_at " + "FROM providence_cache " + "WHERE audit_mode IN ('VISUAL','HYBRID') " + " AND falsification_state = 'live' " + "ORDER BY created_at DESC LIMIT ?", + (args.limit,), + ).fetchall() + out = [ + { + "cache_key": r["cache_key"], + "audit_mode": r["audit_mode"], + "verifier_method": r["verifier_method"], + "question": r["question_text"], + "n_quotes": r["n_quotes"], + "n_verified": r["n_verified"], + "unverified_quotes": ( + json.loads(r["unverified_quotes"]) + if r["unverified_quotes"] + else [] + ), + "created_at": r["created_at"], + } + for r in rows + ] + print(json.dumps(out, indent=2)) + finally: + conn.close() + return 0 + + def _cmd_evict(args: argparse.Namespace) -> int: from aborist.evict import evict_to_cold @@ -1488,6 +1767,51 @@ def build_parser() -> argparse.ArgumentParser: ) prov_cmd.set_defaults(func=_cmd_providence) + reclassify_cmd = sub.add_parser( + "reclassify", + help="re-run the verifier against existing live providence records " + "(no LLM calls; relabels stale classifications)", + ) + reclassify_cmd.add_argument( + "--qa-db", dest="qa_db", default=None, + help="path to qa.db (default: /qa.db or ~/.aborist/qa.db)", + ) + reclassify_cmd.add_argument( + "--limit", type=int, default=0, + help="reclassify at most N records (0 = unlimited)", + ) + reclassify_cmd.add_argument( + "--dry-run", dest="dry_run", action="store_true", + help="report what would change without writing", + ) + reclassify_cmd.add_argument( + "--entity-policy", dest="entity_policy", default=None, + choices=["strict", "hybrid", "drop", "proximity"], + help=( + "how the entity path classifies: 'strict' (legacy, overclaims), " + "'hybrid' (default — caps at HYBRID), 'drop' (skip entity path → " + "VISUAL), 'proximity' (STRICT only if N entities cluster within " + "W chars in source)" + ), + ) + reclassify_cmd.add_argument( + "--compare", dest="compare", action="store_true", + help="run all four entity policies side-by-side without writing", + ) + reclassify_cmd.set_defaults(func=_cmd_reclassify) + + emergent_cmd = sub.add_parser( + "emergent", + help="surface VISUAL/HYBRID claims — corpus-growth signal", + ) + emergent_cmd.add_argument( + "--aggregate", + action="store_true", + help="rank unverified quotes by frequency (vs per-record list)", + ) + emergent_cmd.add_argument("--limit", type=int, default=20) + emergent_cmd.set_defaults(func=_cmd_emergent) + evict_cmd = sub.add_parser( "evict", help="demote surface chunks hot→cold (NULL content, retain leaf_hash)", diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 7291c2b..5d1c427 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -14,11 +14,17 @@ The flow: 4. Compute `context_root` = Merkle root over the sorted source document_roots — that's the "source" dimension of v9.8's 8-dim cache_key for this multi-source answer. - 5. Cache lookup; hit returns STRICT immediately. - 6. Miss calls Hermes via the OpenAI-compatible client. - 7. Persist record with merkle_proof = {context_root, sources: [...]} - so a verifier can recompute context_root and check each source - document_root against its shard. + 5. Cache lookup; hit returns the persisted audit_mode. + 6. Miss calls Hermes via the OpenAI-compatible client, then runs the + faithfulness check (`verify_quotes`) — every double-quoted span in + the answer is verbatim-matched against the assembled context. + Result classifies the answer: + STRICT every quote (>=1) verified against context + HYBRID some claims sourced, some emergent (training-derived) + VISUAL no quotes verify — purely emergent + 7. Persist record with merkle_proof = {context_root, sources: [...]}, + audit_mode, and unverified_quotes (the spans the model produced + that didn't appear in any source — corpus-growth signal). Per-source proofs are not bundled here (the source roots themselves are already content-addressed). A verifier asks the shards for any specific @@ -53,6 +59,7 @@ from aborist.qa.keys import ( model_profile_hash, question_hash, ) +from aborist.qa.verify import verify_quotes from aborist.search import FTS5Backend from aborist.store import ( append_audit, @@ -165,6 +172,12 @@ DEFAULT_QUERY_POLICY = { "system_prompt": ( "You are answering a question using ONLY the sources provided below. " "Each source is delimited by '=== Source: ===' headers.\n\n" + "GROUNDING RULE (most important):\n" + "For EVERY factual claim in your answer, include a verbatim quote " + "from a source enclosed in double quotes (\"...\"). The quoted span " + "must appear word-for-word in one of the sources. If you cannot " + "find a verbatim quote that supports a claim, do not make the " + "claim. Do not paraphrase inside the quotes.\n\n" "ATTRIBUTION RULES:\n" "1. A single source may discuss MULTIPLE products, companies, or " "competitors. Read carefully and only attribute facts to the entity " @@ -177,14 +190,40 @@ DEFAULT_QUERY_POLICY = { "that AMD's PR-rating numbers (3200+, 2500+) are NOT the clock speed " "in MHz. Quote the source's own wording for clock speeds rather than " "interpreting model numbers.\n" - "4. Cite which source supports each claim by quoting it inline.\n" - "5. If the sources do not contain the answer, say 'I don't know " - "based on the provided sources.' Do not speculate beyond what the " - "sources say." + "4. If the sources do not contain the answer, say 'I don't know " + "based on the provided sources.' Do not fall back on background " + "knowledge from training." + ), + # Restated rule fired as a user message immediately before the sources + # turn. Hermes (and most instruction-tuned 8B models) follow recent + # user-turn instructions more reliably than a system-turn rule that + # decays under long context. Repetition is not redundancy — it raises + # the prior on the response shape we want. + "grounding_reminder": ( + "REMINDER: every factual claim in your reply must be wrapped in " + "double quotes (\"...\") and the quoted span must appear " + "word-for-word in one of the sources. No quote, no claim. " + "Example:\n\n" + " Q: who founded Apple?\n" + " Sources: ...Apple Inc. was founded by Steve Jobs, Steve " + "Wozniak, and Ronald Wayne in 1976...\n" + " A: Apple's founders are named in the source: \"Apple Inc. was " + "founded by Steve Jobs, Steve Wozniak, and Ronald Wayne\".\n\n" + "Now answer the question on the next message." ), "temperature": 0.1, "top_p": 1.0, "max_tokens": 768, + # Entity-path policy for the faithfulness verifier. See + # aborist/qa/verify.py:ENTITY_POLICIES. Default 'proximity' promotes + # to STRICT only if N verified entities cluster within W chars of + # each other in source — separating "source documents these entities + # as a group" (cast list / infobox / roster) from "source mentions + # them incidentally in scattered prose". Lives in policy so changing + # it bumps governance_policy_hash and invalidates cache cleanly. + "entity_policy": "proximity", + "entity_proximity_n": 3, + "entity_proximity_window": 300, } @@ -628,13 +667,24 @@ def query( context_ms = _ms_since(t_phase) # 3. Build messages + hashes. + # System sets the policy; a user-turn reminder restates the verbatim-quote + # rule one message before the sources arrive (Hermes follows recent + # user-turn instructions more reliably than system-turn rules under long + # context). The sources + question land last so they are the most-recent + # tokens before generation begins. messages = [ {"role": "system", "content": policy["system_prompt"]}, + ] + if policy.get("grounding_reminder"): + messages.append( + {"role": "user", "content": policy["grounding_reminder"]} + ) + messages.append( { "role": "user", "content": f"Sources:\n\n{context}\n\n---\n\nQuestion: {question}", - }, - ] + } + ) context_root = _context_root([h.document_root for h in chosen]) qhash = question_hash(question) mhash = model_profile_hash(model_id, revision, quantization) @@ -672,11 +722,19 @@ def query( ) return { "status": "cache_hit", - "audit_mode": "STRICT", + "audit_mode": cached["audit_mode"], "cache_key": ckey, "context_root": context_root, "answer_text": cached["answer_text"], "sources": json.loads(cached["merkle_proof"])["sources"], + "n_quotes": cached["n_quotes"], + "n_verified": cached["n_verified"], + "verifier_method": cached["verifier_method"], + "unverified_quotes": ( + json.loads(cached["unverified_quotes"]) + if cached["unverified_quotes"] + else [] + ), "timings": { "search_ms": search_ms, "context_ms": context_ms, @@ -698,6 +756,21 @@ def query( ) llm_ms = _ms_since(t_phase) + # 5b. Faithfulness check — pull quoted spans, verify each verbatim + # against the assembled context, classify into v9.8 audit_mode. + verdict = verify_quotes( + answer_text, + context, + entity_policy=policy.get("entity_policy", "hybrid"), + proximity_n=policy.get("entity_proximity_n", 3), + proximity_window=policy.get("entity_proximity_window", 300), + ) + unverified_blob = ( + json.dumps(verdict["unverified_quotes"], separators=(",", ":")) + if verdict["unverified_quotes"] + else None + ) + # 6. Persist record + audit event. t_phase = time.monotonic() proof_obj = { @@ -729,6 +802,10 @@ def query( "quantization": quantization, "answer_chars": len(answer_text), "context_chars": len(context), + "audit_mode": verdict["audit_mode"], + "n_quotes": verdict["n_quotes"], + "n_verified": verdict["n_verified"], + "verifier_method": verdict["verifier_method"], }, ts=now, ) @@ -738,8 +815,10 @@ def query( " answer_text, merkle_proof, model_profile_hash, conversation_hash, " " governance_policy_hash, schema_version, canonicalization_version, " " chunking_version, falsification_state, chain, audit_event_hash, " - " created_at, hit_count) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'live', ?, ?, ?, 0)", + " created_at, hit_count, audit_mode, n_quotes, n_verified, " + " unverified_quotes, verifier_method) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'live', ?, ?, ?, 0, " + " ?, ?, ?, ?, ?)", ( ckey, context_root, @@ -757,6 +836,11 @@ def query( chain, event_hash, now, + verdict["audit_mode"], + verdict["n_quotes"], + verdict["n_verified"], + unverified_blob, + verdict["verifier_method"], ), ) finally: @@ -765,11 +849,15 @@ def query( persist_ms = _ms_since(t_phase) return { "status": "cache_miss_then_written", - "audit_mode": "STRICT", + "audit_mode": verdict["audit_mode"], "cache_key": ckey, "context_root": context_root, "answer_text": answer_text, "sources": proof_obj["sources"], + "n_quotes": verdict["n_quotes"], + "n_verified": verdict["n_verified"], + "verifier_method": verdict["verifier_method"], + "unverified_quotes": verdict["unverified_quotes"], "timings": { "search_ms": search_ms, "context_ms": context_ms, diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index 645eb73..551a9cb 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -4,8 +4,9 @@ Implements the v9.8 admissibility invariant: No record reused unless all 8 cache_key dimensions match AND state is 'live' (not failed/stale/quarantined). -Cache hit -> STRICT-mode answer with stored Merkle proof. -Cache miss -> call ChatClient, store record, audit event. +Cache hit -> persisted audit_mode (STRICT/HYBRID/VISUAL). +Cache miss -> call ChatClient, run faithfulness check, classify, store + record, audit event. """ from __future__ import annotations @@ -28,18 +29,36 @@ from aborist.qa.keys import ( model_profile_hash, question_hash, ) +from aborist.qa.verify import verify_quotes from aborist.store import append_audit, transaction DEFAULT_POLICY = { "system_prompt": ( "Answer the user's question based ONLY on the document below. " - "If the answer is not in the document, say 'I don't know.' " - "Do not speculate. Cite the relevant sentence(s)." + "For EVERY factual claim, include a verbatim quote from the " + "document enclosed in double quotes (\"...\"). The quoted span " + "must appear word-for-word. If you cannot find a verbatim quote " + "supporting a claim, do not make the claim. " + "If the answer is not in the document, say 'I don't know based " + "on the provided document.' Do not speculate." + ), + # Restated rule fired as a user message right before the document + + # question arrive. See aborist/qa/query.py for the rationale (recent + # user-turn instructions outweigh decayed system-turn rules in 8B + # instruction-tuned models). + "grounding_reminder": ( + "REMINDER: every factual claim in your reply must be wrapped in " + "double quotes (\"...\") and the quoted span must appear " + "word-for-word in the document. No quote, no claim. " + "Now answer the question on the next message." ), "temperature": 0.1, "top_p": 1.0, "max_tokens": 512, + "entity_policy": "proximity", + "entity_proximity_n": 3, + "entity_proximity_window": 300, } @@ -83,16 +102,25 @@ def ask( document_text = "\n\n".join(unpack_chunk(r["content"]) for r in chunk_rows) + # System sets the policy; a user-turn reminder restates the verbatim-quote + # rule one message before the document arrives. Document + question land + # last as the most-recent tokens before generation. messages = [ {"role": "system", "content": policy["system_prompt"]}, + ] + if policy.get("grounding_reminder"): + messages.append( + {"role": "user", "content": policy["grounding_reminder"]} + ) + messages.append( { "role": "user", "content": ( f"Document:\n\n{document_text}\n\n---\n\n" f"Question: {question}" ), - }, - ] + } + ) qhash = question_hash(question) mhash = model_profile_hash(model_id, revision, quantization) @@ -127,11 +155,19 @@ def ask( ) return { "status": "cache_hit", - "audit_mode": "STRICT", + "audit_mode": cached["audit_mode"], "cache_key": ckey, "source_root": document_root, "answer_text": cached["answer_text"], "merkle_proof": json.loads(cached["merkle_proof"]), + "n_quotes": cached["n_quotes"], + "n_verified": cached["n_verified"], + "verifier_method": cached["verifier_method"], + "unverified_quotes": ( + json.loads(cached["unverified_quotes"]) + if cached["unverified_quotes"] + else [] + ), "timings": { "cache_lookup_ms": cache_lookup_ms, "llm_ms": None, @@ -149,6 +185,19 @@ def ask( ) llm_ms = _ms_since(t_llm) + verdict = verify_quotes( + answer_text, + document_text, + entity_policy=policy.get("entity_policy", "hybrid"), + proximity_n=policy.get("entity_proximity_n", 3), + proximity_window=policy.get("entity_proximity_window", 300), + ) + unverified_blob = ( + json.dumps(verdict["unverified_quotes"], separators=(",", ":")) + if verdict["unverified_quotes"] + else None + ) + leaves = [bytes.fromhex(r["leaf_hash"]) for r in chunk_rows] tree = MerkleTree.build(leaves) proof_obj = { @@ -170,6 +219,10 @@ def ask( "quantization": quantization, "chunks_in_context": len(chunk_rows), "answer_chars": len(answer_text), + "audit_mode": verdict["audit_mode"], + "n_quotes": verdict["n_quotes"], + "n_verified": verdict["n_verified"], + "verifier_method": verdict["verifier_method"], }, ts=now, ) @@ -179,8 +232,10 @@ def ask( " answer_text, merkle_proof, model_profile_hash, conversation_hash, " " governance_policy_hash, schema_version, canonicalization_version, " " chunking_version, falsification_state, chain, audit_event_hash, " - " created_at, hit_count) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'live', ?, ?, ?, 0)", + " created_at, hit_count, audit_mode, n_quotes, n_verified, " + " unverified_quotes, verifier_method) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'live', ?, ?, ?, 0, " + " ?, ?, ?, ?, ?)", ( ckey, document_root, @@ -198,16 +253,25 @@ def ask( chain, event_hash, now, + verdict["audit_mode"], + verdict["n_quotes"], + verdict["n_verified"], + unverified_blob, + verdict["verifier_method"], ), ) return { "status": "cache_miss_then_written", - "audit_mode": "STRICT", + "audit_mode": verdict["audit_mode"], "cache_key": ckey, "source_root": document_root, "answer_text": answer_text, "merkle_proof": proof_obj, + "n_quotes": verdict["n_quotes"], + "n_verified": verdict["n_verified"], + "verifier_method": verdict["verifier_method"], + "unverified_quotes": verdict["unverified_quotes"], "timings": { "cache_lookup_ms": cache_lookup_ms, "llm_ms": llm_ms, diff --git a/aborist/qa/verify.py b/aborist/qa/verify.py new file mode 100644 index 0000000..609945a --- /dev/null +++ b/aborist/qa/verify.py @@ -0,0 +1,329 @@ +"""Post-LLM faithfulness check: did the answer ground its claims in context? + +Three layered strategies, tried in order. The first one that finds evidence +classifies the answer. `verifier_method` on the result records which path +fired so the audit chain stays diagnostic. + + 1. quote model wrapped claims in double quotes per system prompt. + Strongest signal — explicit, verbatim, model-asserted. + 2. span no quotes, but bullet/sentence-level lines from the answer + appear verbatim in context. Catches models that quote + inline without "..." marks. + 3. entity no quotes and no span match, but multi-word proper-noun + phrases from the answer appear verbatim in context. + Catches the Wikipedia-infobox-to-prose case: the model + paraphrases structure so spans diverge, but every named + entity is intact and grounded. + +Each strategy classifies into v9.8's audit-mode trichotomy: + + STRICT every evidence unit (>=1) verifies verbatim against context + HYBRID some verify, others do not (mixed source / emergent) + VISUAL no evidence, or none verify (purely emergent) + +`unverified_quotes` (kept under that name for schema continuity) collects +spans the model produced that don't appear in any source — the +corpus-growth signal mined by `aborist emergent`. + +Hard rule (CLAUDE.md "soft hash vs hard hash"): every check is a lexical +substring test under norm-v1 + lowercase canonicalization. No embeddings, +no semantic similarity, no fuzzy alignment. The contract is "this token +sequence either is or isn't in the context." +""" + +from __future__ import annotations + +import re +import unicodedata + + +# Locate every double-quote character (ASCII or curly). Sequential +# pairing in extract_quotes() turns these into intentional (open, close) +# pairs: 1st & 2nd char, 3rd & 4th, etc. Pure-regex pairing fails on +# adjacent quote pairs like `"title" prose "quote"` — the regex captures +# `prose` as a "quoted span" because every `"` looks like both an opener +# and a closer to it. +_QUOTE_CHAR_RE = re.compile(r'["“”]') + +# Bullet markers at line start: -, *, +, •, 1., 2), etc. +_BULLET_RE = re.compile(r'^\s*(?:[-*+•]|\d+[.)])\s+') + +# Sentence boundary: punctuation + whitespace + capital letter. +_SENT_RE = re.compile(r'(?<=[.!?])\s+(?=[A-Z])') + +# Multi-word capitalized phrase. Two or more whitespace-separated tokens, +# each starting with a capital letter (allowing initials like "A.", hyphens +# like "Carrie-Anne", and trailing lowercase like "Smith"). Matches "Keanu +# Reeves", "Thomas A. Anderson", "Carrie-Anne Moss", "Agent Smith". Skips +# single capitalized words to avoid sentence-starter false positives. +_PROPER_NOUN_RE = re.compile( + # Token gap is non-newline whitespace so a phrase never crosses a + # paragraph break. Caught a real case where "Joe Pantoliano\n\nThe + # sources" matched as one phrase via \s+. + r"\b[A-Z][A-Za-z'’\-]*(?:[ \t]+(?:[A-Z]\.|[A-Z][A-Za-z'’\-]+))+\b" +) + +MIN_QUOTE_CHARS = 8 +MIN_SPAN_CHARS = 12 + +# Entity-path policies. Entity-existence in source is circumstantial, not +# claim-level proof — the model could correctly name entities while making +# claims around them that came from training. These four policies span the +# trade-off: max-trust to no-trust. +# +# strict all entities verify → STRICT (legacy behavior; overclaims) +# hybrid any entity verifies → HYBRID (honest cap; safe default) +# drop skip entity path entirely → VISUAL (most conservative) +# proximity STRICT only if N verified entities cluster within W chars +# of each other in context (e.g. an infobox cast list). +# Otherwise demotes to HYBRID/VISUAL based on partial match. +ENTITY_POLICIES = ("strict", "hybrid", "drop", "proximity") +DEFAULT_ENTITY_POLICY = "proximity" + +# Proximity tuning. N entities within W chars of each other. +DEFAULT_PROXIMITY_N = 3 +DEFAULT_PROXIMITY_WINDOW = 300 + +# Framing phrases that aren't real claims even when they're in source. We +# strip these from the answer before span/entity extraction so they don't +# become noise in unverified_quotes. +_FRAMING_PREFIXES = ( + "based on the provided sources", + "based on the provided source", + "based on the source", + "based on the document", + "the sources do not", + "the source does not", + "the document does not", + "according to the sources", + "according to the source", +) + + +def _normalize(s: str) -> str: + """norm-v1 + lowercase. Same canonicalization as chunk leaf hashing, + plus case-folding so the verifier doesn't fail on capitalization drift + between source prose and the model's quoted span.""" + s = unicodedata.normalize("NFC", s) + s = " ".join(s.split()) + return s.lower() + + +def _is_framing(span: str) -> bool: + n = _normalize(span) + return any(n.startswith(p) for p in _FRAMING_PREFIXES) + + +def extract_quotes(answer_text: str) -> list[str]: + """Pull double-quoted spans of length >= MIN_QUOTE_CHARS from `answer_text`. + + Sequential pairing: locate every double-quote character, then pair + them as (1st, 2nd), (3rd, 4th), .... Each pair brackets one quoted + span; text between consecutive pairs is the model's own framing + prose (not captured). This is the correct model for adjacent quote + pairs like `"title" prose "quote"` — naive regex matching paired + the close of "title" with the open of "quote" and captured `prose` + as a phantom quote, dragging classifications down to HYBRID + incorrectly. + """ + positions = [m.start() for m in _QUOTE_CHAR_RE.finditer(answer_text)] + quotes: list[str] = [] + for i in range(0, len(positions) - 1, 2): + span = answer_text[positions[i] + 1:positions[i + 1]] + if len(span) >= MIN_QUOTE_CHARS: + quotes.append(span) + return quotes + + +def extract_claim_spans(answer_text: str) -> list[str]: + """Strip bullet markers, split into sentences, drop framing prefixes. + + Returns each non-empty span of length >= MIN_SPAN_CHARS. These are the + "claim units" the model wrote — each one we'll substring-test against + context. + """ + spans: list[str] = [] + for line in answer_text.splitlines(): + stripped = _BULLET_RE.sub("", line).strip() + if not stripped: + continue + for sent in _SENT_RE.split(stripped): + sent = sent.strip().rstrip(".:,;") + if len(sent) < MIN_SPAN_CHARS: + continue + if _is_framing(sent): + continue + spans.append(sent) + return spans + + +def extract_proper_nouns(answer_text: str) -> list[str]: + """Pull multi-word capitalized phrases. Deduplicated, order preserved. + + Multi-word only — single capitalized words at sentence start are too + noisy ("Based", "Now", "However"). Multi-word phrases like + "Keanu Reeves" or "Thomas A. Anderson" are reliable proper-noun + candidates and substring-test cleanly against source prose or + structured wikitext. + """ + seen: set[str] = set() + out: list[str] = [] + for m in _PROPER_NOUN_RE.findall(answer_text): + if m in seen: + continue + seen.add(m) + out.append(m) + return out + + +def _classify(verified: list[str], unverified: list[str]) -> str: + if verified and not unverified: + return "STRICT" + if verified: + return "HYBRID" + return "VISUAL" + + +def _has_entity_cluster( + verified_entities: list[str], + norm_ctx: str, + n: int, + window: int, +) -> bool: + """Do at least `n` distinct verified entities appear within `window` + chars of each other in `norm_ctx`? Crude proxy for "the source has a + section about these entities" (cast list, infobox, roster) vs + "the source incidentally mentions them in scattered prose." + """ + if len(verified_entities) < n: + return False + positions: list[int] = [] + for e in verified_entities: + idx = norm_ctx.find(_normalize(e)) + if idx >= 0: + positions.append(idx) + if len(positions) < n: + return False + positions.sort() + for i in range(len(positions) - n + 1): + if positions[i + n - 1] - positions[i] <= window: + return True + return False + + +def _check_each(items: list[str], norm_ctx: str) -> tuple[list[str], list[str]]: + verified: list[str] = [] + unverified: list[str] = [] + for it in items: + if _normalize(it) in norm_ctx: + verified.append(it) + else: + unverified.append(it) + return verified, unverified + + +def verify_quotes( + answer_text: str, + context: str, + *, + entity_policy: str = DEFAULT_ENTITY_POLICY, + proximity_n: int = DEFAULT_PROXIMITY_N, + proximity_window: int = DEFAULT_PROXIMITY_WINDOW, +) -> dict: + """Classify an answer's grounding against its retrieved context. + + Tries quote → span → entity verification in sequence. The first + strategy that finds evidence classifies the answer; later strategies + don't run. + + `entity_policy` controls how the entity path classifies — see + ENTITY_POLICIES. The quote and span paths are unaffected; they are + explicit-claim evidence and always classify per the trichotomy. + + Returns: + { + "n_quotes": int, # evidence units extracted (any path) + "n_verified": int, # of those, how many appear verbatim + "audit_mode": str, # STRICT | HYBRID | VISUAL + "unverified_quotes": [str], # spans we couldn't ground in context + "verifier_method": str, # 'quote' | 'span' | 'entity' | 'none' + } + """ + if entity_policy not in ENTITY_POLICIES: + raise ValueError( + f"entity_policy must be one of {ENTITY_POLICIES}, got {entity_policy!r}" + ) + + norm_ctx = _normalize(context) + + # Strategy 1: explicit double-quoted spans. + quotes = extract_quotes(answer_text) + if quotes: + verified, unverified = _check_each(quotes, norm_ctx) + return { + "n_quotes": len(quotes), + "n_verified": len(verified), + "audit_mode": _classify(verified, unverified), + "unverified_quotes": unverified, + "verifier_method": "quote", + } + + # Strategy 2: bullet/sentence spans verbatim in context. + spans = extract_claim_spans(answer_text) + if spans: + verified, unverified = _check_each(spans, norm_ctx) + if verified: + return { + "n_quotes": len(spans), + "n_verified": len(verified), + "audit_mode": _classify(verified, unverified), + "unverified_quotes": unverified, + "verifier_method": "span", + } + + # Strategy 3: multi-word proper nouns (entity grounding) — gated by + # `entity_policy`. Entity-existence is weaker proof than quote or span; + # the operator chooses how much weight to give it. + if entity_policy == "drop": + # Skip entity path entirely. Falls through to VISUAL/none. + pass + else: + entities = extract_proper_nouns(answer_text) + if entities: + verified, unverified = _check_each(entities, norm_ctx) + if verified: + if entity_policy == "strict": + # Legacy behavior: all match → STRICT. Overclaims. + mode = _classify(verified, unverified) + elif entity_policy == "hybrid": + # Cap at HYBRID. Honest middle: evidence exists, but + # entity-existence ≠ claim-existence. + mode = "HYBRID" + else: # proximity + cluster = _has_entity_cluster( + verified, norm_ctx, proximity_n, proximity_window + ) + if cluster and not unverified: + mode = "STRICT" + elif cluster: + mode = "HYBRID" + elif verified: + mode = "HYBRID" + else: + mode = "VISUAL" + return { + "n_quotes": len(entities), + "n_verified": len(verified), + "audit_mode": mode, + "unverified_quotes": unverified, + "verifier_method": "entity", + } + + # Nothing extracted, or nothing verified. Truly emergent. + return { + "n_quotes": 0, + "n_verified": 0, + "audit_mode": "VISUAL", + "unverified_quotes": [], + "verifier_method": "none", + } diff --git a/aborist/store.py b/aborist/store.py index 6e05067..3f47b29 100644 --- a/aborist/store.py +++ b/aborist/store.py @@ -151,10 +151,32 @@ CREATE TABLE IF NOT EXISTS providence_cache ( audit_event_hash TEXT, -- latest audit event for this record created_at INTEGER NOT NULL, last_hit_at INTEGER, - hit_count INTEGER NOT NULL DEFAULT 0 + hit_count INTEGER NOT NULL DEFAULT 0, + -- v9.8 audit_mode trichotomy: STRICT (every quote in answer verified + -- against context), HYBRID (some claims verified, some emergent), + -- VISUAL (no verbatim grounding — purely emergent from training). + -- Default VISUAL: an unclassified record is the weakest claim. + audit_mode TEXT NOT NULL DEFAULT 'VISUAL' + CHECK (audit_mode IN ('STRICT','HYBRID','VISUAL')), + n_quotes INTEGER NOT NULL DEFAULT 0, + n_verified INTEGER NOT NULL DEFAULT 0, + -- JSON array of quoted spans the model produced but we couldn't find + -- verbatim in context. Mining these surfaces "what the model emerged + -- beyond the corpus" — candidate ingest targets. + unverified_quotes TEXT, + -- Which verifier strategy classified this record. 'quote' = model + -- followed the format and wrapped claims in double quotes. 'span' = + -- bullet/sentence-level substring match in context. 'entity' = no + -- spans matched but multi-word proper nouns did. 'none' = no + -- evidence at all (truly emergent). + verifier_method TEXT NOT NULL DEFAULT 'none' + CHECK (verifier_method IN ('quote','span','entity','none')) ); CREATE INDEX IF NOT EXISTS idx_providence_root ON providence_cache(source_root); CREATE INDEX IF NOT EXISTS idx_providence_state ON providence_cache(falsification_state); +-- idx_providence_audit lives in _migrate_audit_mode() so legacy shards +-- (where audit_mode column gets added by ALTER TABLE) don't trip this +-- script before the migration runs. -- Append-only audit chain. event_hash = sha256(prev_event_hash || canonical(body)). CREATE TABLE IF NOT EXISTS audit_events ( @@ -272,6 +294,7 @@ def connect(db_path: Path | str = DEFAULT_DB_PATH) -> sqlite3.Connection: conn = sqlite3.connect(p, isolation_level=None) # autocommit; we'll BEGIN manually conn.row_factory = sqlite3.Row conn.executescript(SCHEMA_SQL) + _migrate_audit_mode(conn) conn.execute("PRAGMA synchronous = NORMAL") conn.execute("PRAGMA cache_size = -65536") conn.execute("PRAGMA temp_store = MEMORY") @@ -279,6 +302,45 @@ def connect(db_path: Path | str = DEFAULT_DB_PATH) -> sqlite3.Connection: return conn +def _migrate_audit_mode(conn: sqlite3.Connection) -> None: + """Forward-migrate pre-v9.8-audit-mode providence_cache shards. + + Adds audit_mode + n_quotes + n_verified + unverified_quotes columns to + DBs that pre-date the faithfulness-classification rollout. SQLite ALTER + TABLE ADD COLUMN is O(1) (metadata-only) so this is cheap on every open. + Idempotent — checks PRAGMA before each ADD. + """ + cols = {row["name"] for row in conn.execute("PRAGMA table_info(providence_cache)")} + if "audit_mode" not in cols: + conn.execute( + "ALTER TABLE providence_cache ADD COLUMN audit_mode TEXT " + "NOT NULL DEFAULT 'VISUAL' " + "CHECK (audit_mode IN ('STRICT','HYBRID','VISUAL'))" + ) + if "n_quotes" not in cols: + conn.execute( + "ALTER TABLE providence_cache ADD COLUMN n_quotes INTEGER NOT NULL DEFAULT 0" + ) + if "n_verified" not in cols: + conn.execute( + "ALTER TABLE providence_cache ADD COLUMN n_verified INTEGER NOT NULL DEFAULT 0" + ) + if "unverified_quotes" not in cols: + conn.execute( + "ALTER TABLE providence_cache ADD COLUMN unverified_quotes TEXT" + ) + if "verifier_method" not in cols: + conn.execute( + "ALTER TABLE providence_cache ADD COLUMN verifier_method TEXT " + "NOT NULL DEFAULT 'none' " + "CHECK (verifier_method IN ('quote','span','entity','none'))" + ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_providence_audit " + "ON providence_cache(audit_mode)" + ) + + # Tables that exist in every shard with the same schema. Used to build # cross-shard UNION views in connect_query(). _SHARDABLE_TABLES = ( diff --git a/tests/test_migration_audit_mode.py b/tests/test_migration_audit_mode.py new file mode 100644 index 0000000..7e5c8d7 --- /dev/null +++ b/tests/test_migration_audit_mode.py @@ -0,0 +1,109 @@ +"""Forward migration: legacy providence_cache shards add audit_mode columns. + +Regression for a defect caught on 2026-04-28: SCHEMA_SQL referenced the +new audit_mode column inside a CREATE INDEX statement. CREATE TABLE IF +NOT EXISTS skips the table on legacy DBs, so the column doesn't yet exist +when the index statement runs — the whole executescript aborts. + +The fix moved CREATE INDEX idx_providence_audit into _migrate_audit_mode. +This test asserts: a DB with the pre-audit-mode schema can be opened +through connect() & emerges with all four new columns + the index. +""" + +from __future__ import annotations + +import sqlite3 + +from aborist.store import connect + + +# Pre-v9.8-audit-mode providence_cache (snapshot of the schema before the +# audit_mode rollout). Mirrors the columns observed in real legacy shards. +LEGACY_SCHEMA = """ +PRAGMA journal_mode = WAL; +CREATE TABLE providence_cache ( + cache_key TEXT PRIMARY KEY, + source_root TEXT NOT NULL, + document_uri TEXT NOT NULL, + question_hash TEXT NOT NULL, + question_text TEXT NOT NULL, + answer_text TEXT NOT NULL, + merkle_proof TEXT NOT NULL, + model_profile_hash TEXT NOT NULL, + conversation_hash TEXT NOT NULL, + governance_policy_hash TEXT NOT NULL, + schema_version TEXT NOT NULL, + canonicalization_version TEXT NOT NULL, + chunking_version TEXT NOT NULL, + falsification_state TEXT NOT NULL DEFAULT 'live', + chain TEXT NOT NULL DEFAULT 'private', + audit_event_hash TEXT, + created_at INTEGER NOT NULL, + last_hit_at INTEGER, + hit_count INTEGER NOT NULL DEFAULT 0 +); +""" + + +def _seed_legacy(path) -> None: + raw = sqlite3.connect(path) + raw.executescript(LEGACY_SCHEMA) + # One pre-existing row simulating a real legacy record. + raw.execute( + "INSERT INTO providence_cache " + "(cache_key, source_root, document_uri, question_hash, question_text, " + " answer_text, merkle_proof, model_profile_hash, conversation_hash, " + " governance_policy_hash, schema_version, canonicalization_version, " + " chunking_version, audit_event_hash, created_at) " + "VALUES ('legacy_key', 'root', 'corpus://x', 'qh', 'Q?', 'A.', '{}', " + " 'mh', 'ch', 'gh', 'v9.8.0', 'norm-v1', 'tok-512-v1', NULL, 0)" + ) + raw.commit() + raw.close() + + +def test_legacy_providence_cache_migrates(tmp_path): + db = tmp_path / "legacy.db" + _seed_legacy(db) + + conn = connect(db) + try: + cols = {r["name"] for r in conn.execute("PRAGMA table_info(providence_cache)")} + assert {"audit_mode", "n_quotes", "n_verified", "unverified_quotes"} <= cols + + # Pre-existing row inherits the safest default — VISUAL. + row = conn.execute( + "SELECT audit_mode, n_quotes, n_verified, unverified_quotes " + "FROM providence_cache WHERE cache_key='legacy_key'" + ).fetchone() + assert row["audit_mode"] == "VISUAL" + assert row["n_quotes"] == 0 + assert row["n_verified"] == 0 + assert row["unverified_quotes"] is None + + # Index is in place after migration. + idx_names = { + r["name"] + for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='index'" + ) + } + assert "idx_providence_audit" in idx_names + finally: + conn.close() + + +def test_migration_idempotent(tmp_path): + """Two consecutive opens of the same legacy DB don't ALTER TABLE twice.""" + db = tmp_path / "twice.db" + _seed_legacy(db) + + connect(db).close() + # Second open: PRAGMA table_info already shows the columns; ALTER TABLE + # would error with "duplicate column name" if we didn't gate it. + conn = connect(db) + try: + cols = {r["name"] for r in conn.execute("PRAGMA table_info(providence_cache)")} + assert "audit_mode" in cols + finally: + conn.close() diff --git a/tests/test_qa.py b/tests/test_qa.py index 5f2758d..de28a75 100644 --- a/tests/test_qa.py +++ b/tests/test_qa.py @@ -57,7 +57,13 @@ def test_ask_writes_record_with_strict_proof(tmp_path): conn = connect(db) try: root = _ingest_one(conn) - client = StubClient(answer="capital flows in eight named queues.") + # Verbatim quote from LONG → audit_mode=STRICT. + client = StubClient( + answer=( + 'The document states: ' + '"eight forms of capital include living, social, and intellectual"' + ) + ) result = ask( conn, document_root=root, @@ -69,7 +75,9 @@ def test_ask_writes_record_with_strict_proof(tmp_path): ) assert result["status"] == "cache_miss_then_written" assert result["audit_mode"] == "STRICT" - assert result["answer_text"] == "capital flows in eight named queues." + assert result["n_quotes"] == 1 + assert result["n_verified"] == 1 + assert result["unverified_quotes"] == [] # Stored proof reconstructs the source root. proof = proof_from_dict(result["merkle_proof"]["chunk_0_proof"]) diff --git a/tests/test_query.py b/tests/test_query.py index f080d98..fa19231 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -63,7 +63,14 @@ def test_query_picks_relevant_docs_and_writes_record(tmp_path): finally: conn.close() - client = StubClient(answer="anarcho-capitalism rests on private property without a state.") + # Verbatim quote from DOCS[2] → audit_mode=STRICT. + client = StubClient( + answer=( + 'Per the source: ' + '"Anarcho-capitalism combines anarchism\'s opposition to the state ' + 'with capitalism\'s private property"' + ) + ) result = query( question="What is anarcho-capitalism?", qa_db=qa_db, @@ -75,7 +82,7 @@ def test_query_picks_relevant_docs_and_writes_record(tmp_path): assert result["status"] == "cache_miss_then_written" assert result["audit_mode"] == "STRICT" - assert "anarcho-capitalism" in result["answer_text"] + assert "Anarcho-capitalism" in result["answer_text"] assert len(result["sources"]) >= 1 assert any("anarcho-capitalism" in s["document_uri"] for s in result["sources"]) diff --git a/tests/test_reclassify.py b/tests/test_reclassify.py new file mode 100644 index 0000000..ce586d8 --- /dev/null +++ b/tests/test_reclassify.py @@ -0,0 +1,215 @@ +"""`aborist reclassify` — re-runs the verifier against live providence records. + +Covers: +- Stale VISUAL records become STRICT/HYBRID/etc when the new verifier + finds grounding via span/entity paths. +- Already-correct records stay unchanged. +- Cold-source records (where any source doc has no hot chunks) are + skipped — we cannot reclassify without context. +- --dry-run reports transitions without writing. +- Each changed record writes one 'providence_reclassify' audit event. +""" + +from __future__ import annotations + +import argparse +import json +from typing import Iterator + +from aborist.cli import _cmd_reclassify +from aborist.document import Document +from aborist.ingest import ingest_source +from aborist.qa import query +from aborist.qa.client import StubClient +from aborist.source import Source +from aborist.store import connect + + +class _FakeSource(Source): + source_type = "test" + + def __init__(self, docs): + self.docs = docs + + def iter_documents(self) -> Iterator[Document]: + yield from self.docs + + +def _doc(uri, content): + return Document(uri=uri, content=content, source_type="test", title=uri) + + +def _build_corpus_and_query(tmp_path, answer_text): + """Ingest one doc, run a query with `answer_text` from a stub client, + return (qa_db, main_db, cache_key, conn-factory).""" + main_db = tmp_path / "corpus.db" + qa_db = tmp_path / "qa.db" + + conn = connect(main_db) + try: + ingest_source( + conn, + _FakeSource([ + _doc( + "test://matrix", + "Keanu Reeves stars as Thomas A. Anderson. " + "Laurence Fishburne plays Morpheus. " + "Carrie-Anne Moss plays Trinity. " + "Hugo Weaving plays Agent Smith.", + ) + ]), + ) + finally: + conn.close() + + client = StubClient(answer=answer_text) + result = query( + question="who stars in The Matrix?", + qa_db=qa_db, + chat_client=client, + model_id="test-model", + single_db=main_db, + top_k=1, + ) + return qa_db, main_db, result["cache_key"] + + +def _make_args(qa_db, dry_run=False, limit=0, entity_policy=None, compare=False): + return argparse.Namespace( + qa_db=str(qa_db), + global_shards_dir=None, + dry_run=dry_run, + limit=limit, + entity_policy=entity_policy, + compare=compare, + ) + + +def test_reclassify_promotes_stale_visual_via_entity_path(tmp_path, capsys): + """Stub answer with no quotes but verbatim entity names. The query + persists VISUAL/none if the verifier ran first; we then mutate the + row to simulate an OLDER record (pre-layered-verifier), and reclassify.""" + answer = ( + "The cast includes Keanu Reeves, Laurence Fishburne, " + "Carrie-Anne Moss, and Hugo Weaving." + ) + qa_db, _, ckey = _build_corpus_and_query(tmp_path, answer) + + # Force the row into a stale state: VISUAL/none/0 — what the old + # quote-only verifier would have written. + conn = connect(qa_db) + try: + conn.execute( + "UPDATE providence_cache SET audit_mode='VISUAL', " + " verifier_method='none', n_quotes=0, n_verified=0, " + " unverified_quotes=NULL " + "WHERE cache_key = ?", + (ckey,), + ) + finally: + conn.close() + + rc = _cmd_reclassify(_make_args(qa_db)) + assert rc == 0 + out = json.loads(capsys.readouterr().out) + assert out["examined"] == 1 + assert out["changed"] == 1 + assert out["skipped_cold"] == 0 + + # Row reflects the new verdict. + conn = connect(qa_db) + try: + row = conn.execute( + "SELECT audit_mode, verifier_method, n_verified FROM providence_cache " + "WHERE cache_key = ?", + (ckey,), + ).fetchone() + assert row["audit_mode"] in ("STRICT", "HYBRID") + assert row["verifier_method"] == "entity" + assert row["n_verified"] >= 3 + # Audit event recorded. + last = conn.execute( + "SELECT event_type FROM audit_events ORDER BY seq DESC LIMIT 1" + ).fetchone() + assert last["event_type"] == "providence_reclassify" + finally: + conn.close() + + +def test_reclassify_unchanged_when_already_correct(tmp_path, capsys): + """Re-running reclassify on a correctly-labeled record is a no-op.""" + answer = ( + "The cast includes Keanu Reeves, Laurence Fishburne, " + "Carrie-Anne Moss, and Hugo Weaving." + ) + qa_db, _, _ = _build_corpus_and_query(tmp_path, answer) + + # First pass classifies correctly via the entity path. Reclassify + # should report unchanged. + rc = _cmd_reclassify(_make_args(qa_db)) + assert rc == 0 + out = json.loads(capsys.readouterr().out) + assert out["examined"] == 1 + assert out["unchanged"] == 1 + assert out["changed"] == 0 + + +def test_reclassify_dry_run_does_not_write(tmp_path, capsys): + """--dry-run reports transitions but leaves the row untouched.""" + answer = ( + "The cast includes Keanu Reeves, Laurence Fishburne, " + "Carrie-Anne Moss, and Hugo Weaving." + ) + qa_db, _, ckey = _build_corpus_and_query(tmp_path, answer) + + # Force stale state. + conn = connect(qa_db) + try: + conn.execute( + "UPDATE providence_cache SET audit_mode='VISUAL', " + " verifier_method='none', n_quotes=0, n_verified=0 " + "WHERE cache_key = ?", + (ckey,), + ) + finally: + conn.close() + + rc = _cmd_reclassify(_make_args(qa_db, dry_run=True)) + assert rc == 0 + out = json.loads(capsys.readouterr().out) + assert out["dry_run"] is True + assert out["changed"] == 1 + + # Row was NOT updated. + conn = connect(qa_db) + try: + row = conn.execute( + "SELECT audit_mode, verifier_method FROM providence_cache " + "WHERE cache_key = ?", + (ckey,), + ).fetchone() + assert row["audit_mode"] == "VISUAL" + assert row["verifier_method"] == "none" + finally: + conn.close() + + +def test_reclassify_skips_cold_source(tmp_path, capsys): + """If the source doc's chunks are cold (content NULL), the loader + returns None and we cannot reclassify — record is skipped.""" + answer = "Keanu Reeves stars in this film." + qa_db, main_db, ckey = _build_corpus_and_query(tmp_path, answer) + + # Evict all chunks of the source doc to cold. Loader will return None. + conn = connect(main_db) + try: + conn.execute("UPDATE chunks SET content = NULL, tier = 'cold'") + finally: + conn.close() + + rc = _cmd_reclassify(_make_args(qa_db)) + assert rc == 0 + out = json.loads(capsys.readouterr().out) + assert out["examined"] == 1 + assert out["skipped_cold"] == 1 + assert out["changed"] == 0 diff --git a/tests/test_verify.py b/tests/test_verify.py new file mode 100644 index 0000000..df5bd3c --- /dev/null +++ b/tests/test_verify.py @@ -0,0 +1,305 @@ +"""Faithfulness verifier: classify answer grounding into v9.8 audit_mode. + +Three layered strategies (quote → span → entity), tried in order. The +first that finds evidence classifies the answer. `verifier_method` +records which path fired. + +Tests cover the full trichotomy under each path plus regressions: + - TMNT: model wrote bios not in context — must be HYBRID/VISUAL + - Matrix: Wikipedia infobox + paraphrased prose. Spans don't match, + but every multi-word proper noun does — entity path classifies it. +""" + +from __future__ import annotations + +from aborist.qa.verify import ( + extract_claim_spans, + extract_proper_nouns, + extract_quotes, + verify_quotes, +) + + +# ---------------------------------------------------------------- extract + + +def test_extract_quotes_finds_double_quoted_spans(): + text = 'He said "the cat is on the mat" and then "rain in spain falls".' + assert extract_quotes(text) == [ + "the cat is on the mat", + "rain in spain falls", + ] + + +def test_extract_quotes_skips_short_spans(): + # 8-char minimum keeps "is a", "the", "ok" out. + text = '"is a" "ok" "long enough span" "no"' + assert extract_quotes(text) == ["long enough span"] + + +def test_extract_quotes_sequential_pairs_skip_inter_pair_prose(): + """Fox caught on 2026-04-28: model wrote `"Hackers" is as follows: "In 1988..."` + and naive regex matching paired the closer of "Hackers" with the + opener of "In 1988", capturing the framing prose between them as a + phantom quote. Sequential pairing (1st & 2nd, 3rd & 4th, ...) treats + inter-pair text as the model's framing — never as a quoted span.""" + text = ( + 'The plot of the 1995 film "Hackers" is as follows, directly ' + 'quoted from the source:\n\n"In 1988, Seattle youth Dade Murphy."' + ) + quotes = extract_quotes(text) + # "Hackers" is 7 chars — below MIN_QUOTE_CHARS, dropped. + # "In 1988, Seattle youth Dade Murphy." pairs cleanly. + # The phantom prose between them is never captured. + assert quotes == ["In 1988, Seattle youth Dade Murphy."] + + +def test_extract_quotes_handles_three_adjacent_pairs(): + """`"A_____" stuff "B_____" more "C_____"` → three clean pairs.""" + text = '"alpha-99" stuff "bravo-99" more "charlie9"' + quotes = extract_quotes(text) + assert quotes == ["alpha-99", "bravo-99", "charlie9"] + + +def test_extract_quotes_handles_curly_quotes(): + text = '“smart quoted span here” and "ascii quoted span"' + quotes = extract_quotes(text) + assert "smart quoted span here" in quotes + assert "ascii quoted span" in quotes + + +# ---------------------------------------------------------------- classify + + +def test_strict_when_all_quotes_verify(): + context = "Capitalism is an economic system based on private ownership." + answer = 'The source defines it: "an economic system based on private ownership"' + v = verify_quotes(answer, context) + assert v["audit_mode"] == "STRICT" + assert v["verifier_method"] == "quote" + assert v["n_quotes"] == 1 + assert v["n_verified"] == 1 + assert v["unverified_quotes"] == [] + + +def test_visual_when_no_quotes_no_spans_no_entities(): + """Truly emergent: no double quotes, no span match, no entity match.""" + context = "Apples are red." + answer = "freedom rests on autonomy alone, without coercion." + v = verify_quotes(answer, context) + assert v["audit_mode"] == "VISUAL" + assert v["verifier_method"] == "none" + assert v["n_quotes"] == 0 + + +def test_visual_when_no_quote_verifies(): + context = "Capitalism is an economic system based on private ownership." + answer = '"this exact span is not in the source at all"' + v = verify_quotes(answer, context) + assert v["audit_mode"] == "VISUAL" + assert v["verifier_method"] == "quote" + assert v["n_quotes"] == 1 + assert v["n_verified"] == 0 + assert v["unverified_quotes"] == ["this exact span is not in the source at all"] + + +def test_hybrid_when_some_quotes_verify_some_dont(): + context = "Capitalism is an economic system based on private ownership." + answer = ( + 'The source says "an economic system" but then claims ' + '"Marx personally invented capitalism in 1867" which is a stretch.' + ) + v = verify_quotes(answer, context) + assert v["audit_mode"] == "HYBRID" + assert v["verifier_method"] == "quote" + assert v["n_quotes"] == 2 + assert v["n_verified"] == 1 + assert v["unverified_quotes"] == [ + "Marx personally invented capitalism in 1867" + ] + + +# ---------------------------------------------------------------- normalization + + +def test_case_and_whitespace_insensitive_match(): + context = "The Eight Forms of Capital include living, social, and intellectual." + answer = '"the eight forms of capital"' # collapsed ws + lowercase + multi-space + v = verify_quotes(answer, context) + assert v["audit_mode"] == "STRICT" + assert v["n_verified"] == 1 + + +# ---------------------------------------------------------------- regression + + +def test_tmnt_regression_must_not_be_strict(): + """Fox's catch on 2026-04-28: model emerged TMNT bios from training + while only the names appeared in context (an episode plot summary). + The state machine must downgrade — STRICT here would be a lie.""" + context = ( + "Master Splinter is framed for attempting to kill the Ultimate Daimyo. " + "Raphael and Michelangelo find themselves pitted against each other in " + "the tournament. Donatello and Usagi must defend Leonardo from assassins." + ) + answer = ( + '1. Leonardo - "leader of the group, he wields a blue katana"\n' + '2. Splinter - "a mutated rat who was once Hamato Yoshi"\n' + '3. Plot - "Raphael and Michelangelo find themselves pitted against each other"' + ) + v = verify_quotes(answer, context) + assert v["audit_mode"] == "HYBRID" + assert v["verifier_method"] == "quote" + assert v["n_verified"] == 1 + assert len(v["unverified_quotes"]) == 2 + assert any("blue katana" in q for q in v["unverified_quotes"]) + assert any("Hamato Yoshi" in q for q in v["unverified_quotes"]) + + +# ---------------------------------------------------------------- spans / entities + + +def test_extract_spans_strips_bullets_and_framing(): + answer = ( + "Based on the provided sources, the main characters are:\n\n" + "- Neo\n" + "- Trinity\n" + "1. Morpheus is a leader\n" + ) + spans = extract_claim_spans(answer) + # Framing line dropped; bullet markers stripped; short single-word + # bullets ("Neo", "Trinity") below MIN_SPAN_CHARS dropped. + assert "Morpheus is a leader" in spans + assert not any(s.startswith("Based on the") for s in spans) + + +def test_extract_proper_nouns_picks_multi_word_phrases(): + text = ( + "Neo (Thomas A. Anderson), played by Keanu Reeves. " + "Morpheus, played by Laurence Fishburne." + ) + nouns = extract_proper_nouns(text) + assert "Thomas A. Anderson" in nouns + assert "Keanu Reeves" in nouns + assert "Laurence Fishburne" in nouns + # Single-word names skipped at this layer. + assert "Neo" not in nouns + assert "Morpheus" not in nouns + + +def test_span_path_classifies_when_bullet_appears_verbatim(): + """A model that doesn't quote but writes lines verbatim from source + still earns evidence via the span path.""" + context = "Apple Inc. was founded by Steve Jobs, Steve Wozniak, and Ronald Wayne in 1976." + answer = ( + "- Apple Inc. was founded by Steve Jobs, Steve Wozniak, and Ronald Wayne in 1976\n" + ) + v = verify_quotes(answer, context) + assert v["audit_mode"] == "STRICT" + assert v["verifier_method"] == "span" + assert v["n_verified"] >= 1 + + +def test_matrix_entity_path_strict_under_default_proximity(): + """Default entity_policy='proximity': Matrix-style answer where the + source has a tight cluster of cast-list entities (infobox) classifies + STRICT. Distinguishes structural grounding from incidental mention.""" + context = ( + "starring = [[Keanu Reeves]] [[Laurence Fishburne]] " + "[[Carrie-Anne Moss]] [[Hugo Weaving]] [[Joe Pantoliano]]. " + "Thomas A. Anderson is the protagonist. The film features " + "Agent Smith as the antagonist. Joe Pantoliano portrays Cypher." + ) + answer = ( + "Based on the provided sources, the main characters are:\n\n" + "- Neo (Thomas A. Anderson), played by Keanu Reeves\n" + "- Morpheus, played by Laurence Fishburne\n" + "- Trinity, played by Carrie-Anne Moss\n" + "- Agent Smith, played by Hugo Weaving\n" + "- Cypher, played by Joe Pantoliano\n" + ) + v = verify_quotes(answer, context) + assert v["audit_mode"] == "STRICT" + assert v["verifier_method"] == "entity" + assert v["n_verified"] >= 5 + + +def test_tmnt_entity_path_demoted_under_default_proximity(): + """Companion to the Matrix test: an answer with only one multi-word + entity ("Teenage Mutant Ninja Turtles") cannot satisfy proximity's + N=3 cluster requirement → demoted to HYBRID even though the entity + verifies. This is what separates incidental mention from structural + grounding.""" + context = ( + "Master Splinter is framed for attempting to kill the Ultimate Daimyo. " + "Raphael and Michelangelo find themselves pitted against each other. " + "Donatello and Usagi defend Leonardo. Teenage Mutant Ninja Turtles " + "appears throughout the article." + ) + answer = ( + "Based on the provided sources, the names of the Teenage Mutant Ninja " + "Turtles' brothers are:\n\n" + "1. Leonardo (Leo)\n" + "2. Raphael (Raph)\n" + "3. Donatello (Donnie)\n" + "4. Michelangelo (Mikey)\n\n" + "Their master's name is Splinter." + ) + v = verify_quotes(answer, context) + assert v["audit_mode"] == "HYBRID" + assert v["verifier_method"] == "entity" + + +def test_entity_policy_strict_promotes_when_all_match(): + """Legacy policy: entity_policy='strict' promotes to STRICT when all + entities verify. Kept for back-compat / experimentation; overclaims.""" + context = "Keanu Reeves, Laurence Fishburne, Carrie-Anne Moss appear." + answer = "The cast includes Keanu Reeves, Laurence Fishburne, and Carrie-Anne Moss." + v = verify_quotes(answer, context, entity_policy="strict") + assert v["audit_mode"] == "STRICT" + assert v["verifier_method"] == "entity" + + +def test_entity_policy_drop_skips_path_entirely(): + """drop policy: never classifies via entity path. Verified entities + in source are ignored; the answer falls through to VISUAL/none.""" + context = "Keanu Reeves stars in this film." + answer = "Keanu Reeves played the lead role." + v = verify_quotes(answer, context, entity_policy="drop") + assert v["audit_mode"] == "VISUAL" + assert v["verifier_method"] == "none" + + +def test_entity_policy_proximity_promotes_only_on_cluster(): + """proximity policy: STRICT only if N=3 verified entities cluster + within W=300 chars in source. Tight infobox = STRICT. + Scattered prose = HYBRID.""" + # Tight cluster: 3 names within ~50 chars. + tight_ctx = "Cast: Keanu Reeves, Laurence Fishburne, Carrie-Anne Moss starred." + answer = "The cast: Keanu Reeves, Laurence Fishburne, and Carrie-Anne Moss." + v = verify_quotes(answer, tight_ctx, entity_policy="proximity") + assert v["audit_mode"] == "STRICT" + assert v["verifier_method"] == "entity" + + # Scattered: same 3 names across 1000+ chars of unrelated prose. + spread_ctx = ( + "Keanu Reeves appeared briefly in scene one. " + + "filler text " * 60 + + "Laurence Fishburne had a cameo. " + + "more filler " * 60 + + "Carrie-Anne Moss made an appearance." + ) + v = verify_quotes(answer, spread_ctx, entity_policy="proximity") + assert v["audit_mode"] == "HYBRID" + assert v["verifier_method"] == "entity" + + +def test_entity_path_hybrid_when_some_proper_nouns_unverified(): + """Mixed: some proper nouns verify, others are emergent. + Default 'hybrid' policy → HYBRID either way.""" + context = "Keanu Reeves and Laurence Fishburne starred together." + answer = "The film featured Keanu Reeves, Laurence Fishburne, and Marlon Brando." + v = verify_quotes(answer, context) + assert v["audit_mode"] == "HYBRID" + assert v["verifier_method"] == "entity" + assert "Marlon Brando" in v["unverified_quotes"]