diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 79aa74e..f709d0c 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -65,6 +65,7 @@ from aborist.qa.keys import ( question_hash, ) from aborist.qa.dag import build_run_dag +from aborist.qa.repair import mechanical_repair from aborist.qa.verify import verify_quotes try: @@ -260,6 +261,15 @@ DEFAULT_QUERY_POLICY = { # answers under raw-wikitext policy stay distinct on lookup. No-op # if mwparserfromhell isn't installed. "base_version": _WIKITEXT_BASE_VERSION, + # Mechanical answer repair after first verify. Off by default so + # existing callers don't see answer text mutate. When on: + # synthetic_elision splits, trailing_artifact trims, and no_overlap + # claim drops are applied deterministically; the repaired answer is + # re-verified & persisted (cache_key inputs unchanged, only + # answer_text differs from what the LLM produced). One audit event + # `providence_repair` records the pre→post transition. Bumps + # governance_policy_hash so on/off agents share no cache silos. + "repair_enabled": False, } @@ -1055,6 +1065,39 @@ def query( proximity_n=policy.get("entity_proximity_n", 3), proximity_window=policy.get("entity_proximity_window", 300), ) + + # 5c. Optional mechanical repair pass. When `policy["repair_enabled"]` + # is on AND the first verdict isn't STRICT, walk the unverified + # quotes through the sidecar classifier; for each repairable + # diagnosis (synthetic_elision split, trailing_artifact trim, + # no_overlap remove) apply the suggestion deterministically & + # re-verify. Persist the post-repair answer + the change log; + # cache_key inputs are unchanged so the cache_key itself stays. + repair_changes: list[dict] = [] + pre_repair_verdict: dict | None = None + if ( + policy.get("repair_enabled") + and verdict["audit_mode"] != "STRICT" + and verdict.get("unverified_quotes") + ): + repair_result = mechanical_repair( + answer_text, verdict["unverified_quotes"], context + ) + if repair_result["changes"]: + new_verdict = verify_quotes( + repair_result["repaired_text"], + context, + entity_policy=policy.get("entity_policy", "hybrid"), + proximity_n=policy.get("entity_proximity_n", 3), + proximity_window=policy.get("entity_proximity_window", 300), + ) + # Accept the repair only if it didn't make things worse. + if new_verdict["n_verified"] >= verdict["n_verified"]: + pre_repair_verdict = verdict + answer_text = repair_result["repaired_text"] + verdict = new_verdict + repair_changes = repair_result["changes"] + unverified_blob = ( json.dumps(verdict["unverified_quotes"], separators=(",", ":")) if verdict["unverified_quotes"] @@ -1103,6 +1146,26 @@ def query( now = int(time.time()) with transaction(qa_conn): + # Record the repair event BEFORE the providence_query event so + # the audit chain shows: repair-happened, THEN we wrote the + # final record. Repair body links pre→post verdicts so an + # auditor can reconstruct what changed. + if repair_changes and pre_repair_verdict is not None: + append_audit( + qa_conn, + event_type="providence_repair", + subject_root=ckey, + body={ + "kind": "mechanical", + "n_changes": len(repair_changes), + "changes": repair_changes, + "pre_audit_mode": pre_repair_verdict["audit_mode"], + "post_audit_mode": verdict["audit_mode"], + "pre_n_verified": pre_repair_verdict["n_verified"], + "post_n_verified": verdict["n_verified"], + }, + ts=now, + ) event_hash = append_audit( qa_conn, event_type="providence_query", @@ -1168,6 +1231,10 @@ def query( "cache_key": ckey, "run_dag_root": run_dag["root"], "lookup_path": "miss", + "repair_changes": repair_changes, + "pre_repair_audit_mode": ( + pre_repair_verdict["audit_mode"] if pre_repair_verdict else None + ), "burned_existing": burned_existing, "context_root": context_root, "answer_text": answer_text, diff --git a/aborist/qa/repair.py b/aborist/qa/repair.py new file mode 100644 index 0000000..425efd4 --- /dev/null +++ b/aborist/qa/repair.py @@ -0,0 +1,135 @@ +"""Mechanical answer repair — apply sidecar repair suggestions deterministically. + +When the verifier flags an answer's quoted span as unverified, the +sidecar (`aborist/qa/inspect.py:_classify_span`) emits a `repair` +field naming a concrete fix: split a `[...]`-elided quote into two +verbatim quotes, trim a trailing `(Source: ...)` artifact, restore a +dropped parenthetical aside, drop a fully-invented claim. This module +applies those fixes by string substitution — no LLM call, no +non-determinism. + +Gated by `policy["repair_enabled"]`. Off by default so existing +callers don't see answer text mutate under their feet. + +Repair philosophy follows the toy-Hermes design pass (fox 2026-04-30): + +- Mechanical first. Cheap, deterministic, idempotent. +- Re-prompt second (TODO; one extra LLM call to ask the model to + rewrite around the failed claim). Adds latency; quality bonus. +- Repair stage is observational from the cache_key's perspective: the + cache_key inputs are unchanged, only the persisted answer text differs. + An audit event records the original→repaired transition. + +The verifier-stays-binary discipline holds: this module never adds +soft-signal fields to the verifier output. It produces a CHANGED +answer text + a CHANGE LOG that the audit chain stores. +""" + +from __future__ import annotations + +from .inspect import _classify_span, _normalize + + +def mechanical_repair( + answer_text: str, unverified_quotes: list[str], context: str +) -> dict: + """Apply repair suggestions to ``answer_text``. + + For each unverified quote, run the sidecar classifier to get a + repair suggestion, then apply it by string substitution where the + suggestion has a concrete replacement (split, trim, drop). Skips + suggestions that need semantic judgment (paraphrase downgrade, aside + restoration without a definitive verbatim form). + + Returns:: + + { + "repaired_text": str, # the modified answer (== input if no repair fired) + "changes": [dict], # per-quote repair log + } + + Each change entry:: + + { + "action": str, # "split_into_two_quotes" | "trim_trailing_artifact" | ... + "before": str, # the original quoted span + "after": str | None, # the replacement text, or None if removed + "diagnosis": str, + } + + Idempotent: applying repair twice produces the same result (the + repaired quotes substring-match in source, so a second pass finds + nothing more to fix). + """ + norm_ctx = _normalize(context) + repaired = answer_text + changes: list[dict] = [] + + for quote in unverified_quotes: + diag = _classify_span(quote, norm_ctx, norm_ctx) + repair = diag.get("repair") or {} + action = repair.get("action") + diagnosis = diag.get("diagnosis", "?") + + if action == "split_into_two_quotes": + # `"prefix [...] suffix"` becomes `"prefix" ... "suffix"`. The + # model's `[...]` ellipsis-marker becomes prose between two + # verbatim-quoted spans the verifier can independently check. + quotes = repair.get("quotes") or [] + if len(quotes) == 2 and all(quotes): + new_segment = f'"{quotes[0]}" ... "{quotes[1]}"' + old_segment = f'"{quote}"' + if old_segment in repaired: + repaired = repaired.replace(old_segment, new_segment, 1) + changes.append({ + "action": action, + "diagnosis": diagnosis, + "before": quote, + "after": new_segment, + }) + + elif action == "trim_trailing_artifact": + # `"prose. (Source: https://...)"` becomes `"prose."`. The + # verbatim prefix kept; the model-appended tail dropped. + kept = repair.get("kept_prefix") + if kept: + old_segment = f'"{quote}"' + new_segment = f'"{kept}"' + if old_segment in repaired: + repaired = repaired.replace(old_segment, new_segment, 1) + changes.append({ + "action": action, + "diagnosis": diagnosis, + "before": quote, + "after": kept, + }) + + elif action == "remove_claim": + # Drop the line containing this quote entirely. Catches + # full-invention spans (no_overlap diagnosis) where there's + # no verbatim source content to substitute. + new_lines = [] + removed = False + for line in repaired.splitlines(keepends=True): + if quote in line and not removed: + removed = True + continue + new_lines.append(line) + if removed: + repaired = "".join(new_lines) + changes.append({ + "action": action, + "diagnosis": diagnosis, + "before": quote, + "after": None, + }) + + # Skipped: include_aside_for_verbatim (needs precise source-span + # extraction we'd rather defer to a re-prompt), paraphrase + # downgrade (needs prose rewriting), partial_paraphrase + # (split_or_remove judgment). These cases pass through the + # mechanical loop unchanged & remain UNSUPPORTED in the + # post-repair verdict — falling back to a re-prompt loop is + # future work (TODO: re-prompt feedback path). + + return {"repaired_text": repaired, "changes": changes} diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index 8313b21..b0778e6 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -35,6 +35,7 @@ from aborist.qa.keys import ( question_hash, ) from aborist.qa.dag import build_run_dag +from aborist.qa.repair import mechanical_repair from aborist.qa.verify import verify_quotes from aborist.store import append_audit, transaction @@ -72,6 +73,9 @@ DEFAULT_POLICY = { "entity_policy": "proximity", "entity_proximity_n": 3, "entity_proximity_window": 300, + # Mechanical answer repair after first verify. Off by default; see + # aborist/qa/query.py for semantics. + "repair_enabled": False, # Strip wikitext markup before the LLM ever sees the context. Lets # Hermes quote prose verbatim and shrinks token bills (~43% on # Wikipedia chunks). Bumps governance_policy_hash so prior cached @@ -289,6 +293,32 @@ def ask( proximity_n=policy.get("entity_proximity_n", 3), proximity_window=policy.get("entity_proximity_window", 300), ) + + # Optional mechanical repair pass — see aborist/qa/query.py for shape. + repair_changes: list[dict] = [] + pre_repair_verdict: dict | None = None + if ( + policy.get("repair_enabled") + and verdict["audit_mode"] != "STRICT" + and verdict.get("unverified_quotes") + ): + repair_result = mechanical_repair( + answer_text, verdict["unverified_quotes"], document_text + ) + if repair_result["changes"]: + new_verdict = verify_quotes( + repair_result["repaired_text"], + document_text, + entity_policy=policy.get("entity_policy", "hybrid"), + proximity_n=policy.get("entity_proximity_n", 3), + proximity_window=policy.get("entity_proximity_window", 300), + ) + if new_verdict["n_verified"] >= verdict["n_verified"]: + pre_repair_verdict = verdict + answer_text = repair_result["repaired_text"] + verdict = new_verdict + repair_changes = repair_result["changes"] + unverified_blob = ( json.dumps(verdict["unverified_quotes"], separators=(",", ":")) if verdict["unverified_quotes"] @@ -328,6 +358,22 @@ def ask( now = int(time.time()) with transaction(conn): + if repair_changes and pre_repair_verdict is not None: + append_audit( + conn, + event_type="providence_repair", + subject_root=ckey, + body={ + "kind": "mechanical", + "n_changes": len(repair_changes), + "changes": repair_changes, + "pre_audit_mode": pre_repair_verdict["audit_mode"], + "post_audit_mode": verdict["audit_mode"], + "pre_n_verified": pre_repair_verdict["n_verified"], + "post_n_verified": verdict["n_verified"], + }, + ts=now, + ) event_hash = append_audit( conn, event_type="providence_write", @@ -389,6 +435,10 @@ def ask( "cache_key": ckey, "run_dag_root": run_dag["root"], "lookup_path": "miss", + "repair_changes": repair_changes, + "pre_repair_audit_mode": ( + pre_repair_verdict["audit_mode"] if pre_repair_verdict else None + ), "source_root": document_root, "answer_text": answer_text, "merkle_proof": proof_obj, diff --git a/tests/test_repair.py b/tests/test_repair.py new file mode 100644 index 0000000..21dd423 --- /dev/null +++ b/tests/test_repair.py @@ -0,0 +1,219 @@ +"""Mechanical answer-repair loop. + +`mechanical_repair` applies sidecar repair suggestions (synthetic_elision +split, trailing_artifact trim, no_overlap remove) to an answer text +deterministically. The query/ask runners gate this behind +`policy["repair_enabled"]` and re-verify the repaired text; if the +post-repair verdict isn't worse, the repaired answer is persisted with +a `providence_repair` audit event recording the pre→post transition. + +These tests cover: +- Each repair action produces the right substitution. +- Idempotence: running repair on already-clean text is a no-op. +- query() integration: repair_enabled=True can promote HYBRID/quote → + STRICT/quote on synthetic_elision cases without an extra LLM call. +- query() default repair_enabled=False leaves answer text untouched. +""" + +from __future__ import annotations + +from typing import Iterator + +from aborist.document import Document +from aborist.ingest import ingest_source +from aborist.qa import query +from aborist.qa.client import StubClient +from aborist.qa.repair import mechanical_repair +from aborist.source import Source +from aborist.store import connect + + +# ---------------------------------------------------------------- mechanical_repair + + +def test_mechanical_repair_splits_synthetic_elision(): + """`"prefix [...] suffix"` (both halves verbatim) becomes + `"prefix" ... "suffix"`.""" + context = ( + "The film centers on the fictional Isla Nublar, in Costa Rica. " + "Universal Studios acquired the rights to the novel." + ) + answer = ( + 'The plot states "The film centers on the fictional Isla Nublar [...] ' + 'Universal Studios acquired the rights to the novel".' + ) + bad_quote = ( + "The film centers on the fictional Isla Nublar [...] " + "Universal Studios acquired the rights to the novel" + ) + out = mechanical_repair(answer, [bad_quote], context) + assert len(out["changes"]) == 1 + assert out["changes"][0]["action"] == "split_into_two_quotes" + # Two separate quoted spans now appear: + assert '"The film centers on the fictional Isla Nublar"' in out["repaired_text"] + assert '"Universal Studios acquired the rights to the novel"' in out["repaired_text"] + # `[...]` no longer appears inside any single quoted span. + assert "[...]" not in out["repaired_text"] + + +def test_mechanical_repair_trims_trailing_citation(): + """`"prose. (Source: ...)"` becomes `"prose."`.""" + context = ( + "Pikachu can store electricity in its cheeks and release it in " + "lightning-based attacks. Pikachu evolves from Pichu." + ) + bad_quote = ( + "Pikachu can store electricity in its cheeks and release it in " + "lightning-based attacks. (Source: https://en.wikipedia.org/wiki/Pikachu)" + ) + answer = f'According to source: "{bad_quote}"' + out = mechanical_repair(answer, [bad_quote], context) + assert len(out["changes"]) == 1 + assert out["changes"][0]["action"] == "trim_trailing_artifact" + assert "(Source:" not in out["repaired_text"] + + +def test_mechanical_repair_removes_no_overlap_line(): + """Full-invention spans get the line stripped from the answer.""" + context = "Pikachu is a Pokémon species." + bad_quote = "The Roman Senate convened in 49 BC to debate Caesar's rebellion" + answer = ( + "- Pikachu lives in the wild\n" + f'- "{bad_quote}"\n' + "- Pichu evolves into Pikachu\n" + ) + out = mechanical_repair(answer, [bad_quote], context) + assert len(out["changes"]) == 1 + assert out["changes"][0]["action"] == "remove_claim" + assert bad_quote not in out["repaired_text"] + # Other bullets preserved. + assert "Pikachu lives in the wild" in out["repaired_text"] + assert "Pichu evolves into Pikachu" in out["repaired_text"] + + +def test_mechanical_repair_idempotent_on_clean_text(): + """No unverified quotes → no change.""" + context = "Cloud is the protagonist." + answer = 'The source: "Cloud is the protagonist".' + out = mechanical_repair(answer, [], context) + assert out["changes"] == [] + assert out["repaired_text"] == answer + + +# ---------------------------------------------------------------- query() integration + + +class FakeSource(Source): + source_type = "test" + + def __init__(self, docs): + self.docs = docs + + def iter_documents(self) -> Iterator[Document]: + yield from self.docs + + +def _doc(uri, content): + return Document(uri=uri, content=content, source_type="test", title=uri.rsplit("/", 1)[-1]) + + +def test_query_repair_disabled_default_leaves_answer_unchanged(tmp_path): + """`policy["repair_enabled"]` defaults to False — answer text in the + persisted record matches what the LLM produced.""" + main_db = tmp_path / "corpus.db" + qa_db = tmp_path / "qa.db" + long_text = ( + "Capitalism is an economic system based on private ownership " + "of the means of production. " * 20 + ) + docs = [_doc("test://capitalism", long_text)] + conn = connect(main_db) + try: + ingest_source(conn, FakeSource(docs)) + finally: + conn.close() + # Answer with a synthetic_elision-style bad quote. + bad_answer = ( + '"Capitalism is an economic system based on private ownership ' + '[...] of the means of production."' + ) + result = query( + question="What is capitalism?", + qa_db=qa_db, + chat_client=StubClient(answer=bad_answer), + model_id="m", + single_db=main_db, + ) + # Repair off → answer text unchanged from LLM output. + qa_conn = connect(qa_db) + try: + row = qa_conn.execute( + "SELECT answer_text FROM providence_cache WHERE cache_key=?", + (result["cache_key"],), + ).fetchone() + finally: + qa_conn.close() + assert row["answer_text"] == bad_answer + assert result["repair_changes"] == [] + + +def test_query_repair_enabled_promotes_synthetic_elision_to_strict(tmp_path): + """With `repair_enabled=True`, the mechanical loop splits a + `[...]`-elided quote into two verbatim spans, re-verifies, & lands + STRICT instead of HYBRID. Persisted answer is the repaired text.""" + main_db = tmp_path / "corpus.db" + qa_db = tmp_path / "qa.db" + # Source phrase the model will fail to quote verbatim: + src_text = ( + "Capitalism is an economic system based on private ownership " + "of the means of production. " * 20 + ) + docs = [_doc("test://capitalism", src_text)] + conn = connect(main_db) + try: + ingest_source(conn, FakeSource(docs)) + finally: + conn.close() + + bad_answer = ( + 'Per the source: "Capitalism is an economic system based on private ownership ' + '[...] of the means of production".' + ) + + # Build a policy variant with repair_enabled=True. + from aborist.qa.query import DEFAULT_QUERY_POLICY + policy = dict(DEFAULT_QUERY_POLICY) + policy["repair_enabled"] = True + + result = query( + question="What is capitalism?", + qa_db=qa_db, + chat_client=StubClient(answer=bad_answer), + model_id="m", + single_db=main_db, + policy=policy, + ) + + # Repair fired & promoted the verdict. + assert result["pre_repair_audit_mode"] in ("HYBRID", "UNGROUNDED") + assert result["audit_mode"] == "STRICT" + assert len(result["repair_changes"]) >= 1 + assert result["repair_changes"][0]["action"] == "split_into_two_quotes" + + # Persisted answer is the REPAIRED text (no `[...]` inside any quote). + qa_conn = connect(qa_db) + try: + row = qa_conn.execute( + "SELECT answer_text FROM providence_cache WHERE cache_key=?", + (result["cache_key"],), + ).fetchone() + # Audit chain has the providence_repair event. + evt = qa_conn.execute( + "SELECT event_type, body FROM audit_events " + "WHERE event_type='providence_repair' ORDER BY seq DESC LIMIT 1" + ).fetchone() + finally: + qa_conn.close() + assert "[...]" not in row["answer_text"] + assert evt is not None + assert evt["event_type"] == "providence_repair"