diff --git a/aborist/cli.py b/aborist/cli.py index 6086f3d..a9af4d0 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -434,6 +434,7 @@ _SOFT_DEMOTE_VIOLATION_KINDS = frozenset({ "POINTER_OVERFLOW_TRIMMED", "TOO_MANY_CLAIMS", "BARE_NAME_CLAIM", + "DEFLECTION_DETECTED", }) diff --git a/aborist/qa/keys.py b/aborist/qa/keys.py index 4a454ec..ebb2669 100644 --- a/aborist/qa/keys.py +++ b/aborist/qa/keys.py @@ -207,6 +207,7 @@ _VERIFIER_POLICY_FIELDS = frozenset({ "claim_lattice_json_stop_sequences", # Warrant-lite (relation-question hard check, Ticket H, 2026-05-01) "claim_lattice_warrant_check_enabled", + "claim_lattice_deflection_check_enabled", # Quote-mode entity policy "entity_policy", "entity_proximity_n", diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 1c5cac0..c356300 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -433,6 +433,13 @@ DEFAULT_QUERY_POLICY = { "claim_lattice_lazy_anchor_demote_threshold": 0.5, "claim_lattice_lazy_anchor_demote_min_pairs": 3, "claim_lattice_warrant_check_enabled": True, + # Deflection check: subject-anchor heuristic catches "model deflected + # to an adjacent grounded question" (e.g. "who burns the amazon + # river?" answered about rainforest deforestation, "river" never + # in answer). Promoted from sidecar to soft-demote 2026-05-02 — + # DEFLECTION_DETECTED downgrades EVIDENCE-WARRANTED → ANCHOR- + # WARRANTED via the existing soft-demote ladder path. + "claim_lattice_deflection_check_enabled": True, # Claim-count ceiling — see runner.DEFAULT_POLICY for rationale. # Bench finding (york-england "tell me all there is to know") # caught the runaway shape; cap of 12 admits entity-list @@ -2141,6 +2148,9 @@ def query( warrant_check_enabled=bool(policy.get( "claim_lattice_warrant_check_enabled", True )), + deflection_check_enabled=bool(policy.get( + "claim_lattice_deflection_check_enabled", True + )), ) rendered = verdict["rendered_text"] answer_text = rendered if rendered else raw_answer @@ -2172,6 +2182,9 @@ def query( warrant_check_enabled=bool(policy.get( "claim_lattice_warrant_check_enabled", True )), + deflection_check_enabled=bool(policy.get( + "claim_lattice_deflection_check_enabled", True + )), ) rendered = verdict["rendered_text"] answer_text = rendered if rendered else raw_answer diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index 81b3350..e6b8bc3 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -242,6 +242,7 @@ DEFAULT_POLICY = { # voice-actor bio. WARRANT_MISSING violations cap audit_mode # at HYBRID. See aborist/qa/warrant.py. "claim_lattice_warrant_check_enabled": True, + "claim_lattice_deflection_check_enabled": True, # Claim-count ceiling. Bench finding (2026-04-30 york-england): # "tell me all there is to know about X" prompted Hermes to spam # 26-59 encyclopedic claims sourced from training, only 2-4 of @@ -621,6 +622,9 @@ def ask( warrant_check_enabled=bool(policy.get( "claim_lattice_warrant_check_enabled", True )), + deflection_check_enabled=bool(policy.get( + "claim_lattice_deflection_check_enabled", True + )), ) # Rendered prose (literal spans interpolated) is the user-facing # answer text — never the model's raw pointer-line output. If @@ -656,6 +660,9 @@ def ask( warrant_check_enabled=bool(policy.get( "claim_lattice_warrant_check_enabled", True )), + deflection_check_enabled=bool(policy.get( + "claim_lattice_deflection_check_enabled", True + )), ) rendered = verdict["rendered_text"] answer_text = rendered if rendered else raw_answer diff --git a/aborist/qa/verify.py b/aborist/qa/verify.py index 07cbfc0..4a7fe67 100644 --- a/aborist/qa/verify.py +++ b/aborist/qa/verify.py @@ -49,6 +49,10 @@ import unicodedata from aborist.qa.warrant import warrant_check +# Deferred import: aborist.qa.inspect imports aborist.compress & +# aborist.store at module load. The verifier doesn't need either +# until deflection actually runs, so defer to call-site to keep +# import order clean if anything else imports verify.py. try: from aborist.wikitext import to_base as _wikitext_to_base except ImportError: # pragma: no cover @@ -1035,6 +1039,7 @@ def verify_claim_lattice( max_claims_per_answer: int = DEFAULT_MAX_CLAIMS_PER_ANSWER, question: str | None = None, warrant_check_enabled: bool = True, + deflection_check_enabled: bool = True, ) -> dict: """Deterministic verifier for ``answer_mode="claim_lattice_pointer"``. @@ -1474,6 +1479,40 @@ def verify_claim_lattice( if title_mismatch_claims and audit_mode == "STRICT": audit_mode = "HYBRID" + # Deflection check (soft demote, promoted from sidecar 2026-05-02). + # When the question's subject anchor is missing from the answer, + # the model deflected — answered an adjacent grounded question + # instead of the user's specific one. Caught the live cases: + # "who burns the amazon river?" → answered about Amazon + # Rainforest deforestation, "river" never in answer + # "what culture burns the amazon rain forest?" → answered + # "what causes burning", "culture" never in answer + # Both passed every other check but the user's question wasn't + # structurally answered. DEFLECTION_DETECTED downgrades + # EVIDENCE-WARRANTED → ANCHOR-WARRANTED via the soft-demote + # path. Render-layer ladder picks this up automatically. + deflection_detected = False + if deflection_check_enabled and question and rendered_text: + # Deferred import to avoid pulling aborist.compress + aborist.store + # at verify.py module-load time when callers may not need them. + from aborist.qa.inspect import diagnose_deflection + signal = diagnose_deflection(question, rendered_text) + if signal.get("kind") == "deflection": + deflection_detected = True + violations.append({ + "kind": "DEFLECTION_DETECTED", + "subject_anchor": signal.get("subject_anchor"), + "overlap_ratio": signal.get("overlap_ratio"), + "rationale": ( + "answer's content tokens omit the question's " + "subject anchor — model answered an adjacent " + "grounded question rather than the user's " + "specific one" + ), + }) + if deflection_detected and audit_mode == "STRICT": + audit_mode = "HYBRID" + return { "n_quotes": n_pairs, "n_verified": n_pairs_verified, @@ -1490,6 +1529,7 @@ def verify_claim_lattice( "lazy_anchor_demoted": lazy_anchor_demoted, "warrant_missing_claim_idxs": warrant_missing_claims, "title_mismatch_claim_idxs": title_mismatch_claims, + "deflection_detected": deflection_detected, } @@ -1551,6 +1591,7 @@ def verify_claim_lattice_json( max_claims_per_answer: int = DEFAULT_MAX_CLAIMS_PER_ANSWER, question: str | None = None, warrant_check_enabled: bool = True, + deflection_check_enabled: bool = True, ) -> dict: """Deterministic verifier for ``answer_mode="claim_lattice"`` (JSON). @@ -1866,6 +1907,29 @@ def verify_claim_lattice_json( if title_mismatch_claims and audit_mode == "STRICT": audit_mode = "HYBRID" + # Deflection check (parallel to pointer-variant promotion). + deflection_detected = False + if deflection_check_enabled and question and rendered_text: + # Deferred import to avoid pulling aborist.compress + aborist.store + # at verify.py module-load time when callers may not need them. + from aborist.qa.inspect import diagnose_deflection + signal = diagnose_deflection(question, rendered_text) + if signal.get("kind") == "deflection": + deflection_detected = True + violations.append({ + "kind": "DEFLECTION_DETECTED", + "subject_anchor": signal.get("subject_anchor"), + "overlap_ratio": signal.get("overlap_ratio"), + "rationale": ( + "answer's content tokens omit the question's " + "subject anchor — model answered an adjacent " + "grounded question rather than the user's " + "specific one" + ), + }) + if deflection_detected and audit_mode == "STRICT": + audit_mode = "HYBRID" + # Same `verifier_method` as the pointer variant ("claim_lattice") # so the providence_cache CHECK constraint accepts both. The mode # is disambiguated downstream via `answer_mode` on the run-DAG & diff --git a/tests/test_qa_quality_live.py b/tests/test_qa_quality_live.py index 72b6a7e..65f8ce2 100644 --- a/tests/test_qa_quality_live.py +++ b/tests/test_qa_quality_live.py @@ -669,3 +669,161 @@ def test_boss_baby_post_2010_correctly_ungrounded(): f"name Boss Baby (warrant gap), or the corpus has been re-ingested " f"with newer content (update test). Answer head: {txt[:300]}" ) + + +# ----------------------------------------------------- river/rainforest/deflection family +# +# Three queries from the same topical area exercising different +# verifier paths. Together they gate the river↔rainforest topic-shift +# class fox surfaced 2026-05-02: + + +def _violation_kinds(result: dict) -> set[str]: + """Extract violation kinds from a query() result. Empty if cache-hit + (the providence_cache schema doesn't restore violations) — tests + that run with BURN=1 / cache_miss_then_written see them.""" + return {v.get("kind") for v in (result.get("violations") or []) if v.get("kind")} + + +def test_amazon_river_burning_who_deflects_or_grounds_river(): + """Question: "who burns the amazon river?" + + The corpus has Deforestation-of-the-Amazon-Rainforest content but + no "amazon river burning" content (rivers don't burn). Two + acceptable behaviors: + + 1. The model deflects to rainforest deforestation (saw this live + 2026-05-02: cited Deforestation-of-the-Amazon-Rainforest E10). + In that case the answer omits "river" — DEFLECTION_DETECTED + soft-demote violation should fire & the verdict should not + be EVIDENCE-WARRANTED (it should drop to ANCHOR-WARRANTED via + the soft-demote path). + 2. The model grounds with "river" actually in the answer (e.g. + refutes "rivers don't burn" + cites real rainforest content). + Then EVIDENCE-WARRANTED is fine. + + Failure: EVIDENCE-WARRANTED (no DEFLECTION_DETECTED) AND no + "river" in the answer — the silent topic shift fox flagged. + """ + r = _ask("who burns the amazon river?") + audit = r.get("audit_mode") + txt = _answer_lower(r) + kinds = _violation_kinds(r) + + # Path 1: deflection caught → soft-demote violation present. + if "DEFLECTION_DETECTED" in kinds: + return + # Path 1 alt: model refused honestly → UNGROUNDED. + if audit == "UNGROUNDED": + return + # Path 2: model kept the river framing → "river" in answer. + if "river" in txt: + return + + pytest.fail( + f"audit={audit} grounded an answer about Amazon Rainforest " + f"deforestation for a question about the Amazon RIVER without " + f"naming 'river' — the silent topic shift the deflection " + f"check is supposed to catch. violations={sorted(kinds)} " + f"answer head: {txt[:300]}" + ) + + +def test_amazon_river_burning_culture_refused_or_ungrounded(): + """Question: "what culture burns the amazon river?" + + Adversarial-impossible premise (rivers don't burn). Mirrors the + Mars-BDFL pattern — model should refuse OR the verifier should + return UNGROUNDED. + + Live observed (2026-05-02): UNGROUNDED 0/1, model wrote "The + Amazon River is not burned by any culture..." + + Failure: STRICT/HYBRID with an affirmative claim that some culture + burns the river. + """ + r = _ask("what culture burns the amazon river?") + audit = r.get("audit_mode") + txt = _answer_lower(r) + kinds = _violation_kinds(r) + + # 1) Honest UNGROUNDED — verifier refused to ground a corpus-impossible claim. + if audit == "UNGROUNDED": + return + # 2) Deflection caught — model may have said something tangential + # that triggered the deflection signal. + if "DEFLECTION_DETECTED" in kinds: + return + # 3) Refutation — model named the false premise. + refutation_markers = ( + "not burned", + "do not burn", + "rivers do not", + "river is not", + "not a culture", + "no culture", + "natural river", + "is not the same", + ) + if any(m in txt for m in refutation_markers): + return + + pytest.fail( + f"audit={audit} affirmed a 'culture burns the amazon river' " + f"claim with grounding — adversarial-impossible premise must " + f"refuse, deflect, or refute. violations={sorted(kinds)} " + f"answer head: {txt[:300]}" + ) + + +def test_amazon_rainforest_burning_culture_grounds_or_deflection_caught(): + """Question: "what culture burns the amazon rain forest?" + + The corpus has rainforest deforestation content but no specific + "culture" framing (the Wikipedia article frames it as "human + settlement / agricultural practices / soy farmers"). Two + acceptable behaviors: + + 1. Model emits the deforestation-driver framing without naming + a culture. Subject-anchor "forest" is present in the answer + (in "deforestation"). Deflection sidecar may or may not + trigger — answer doesn't echo "culture" but the heuristic + uses the LAST content token of the question. If + DEFLECTION_DETECTED fires, it's a soft-demote and ladder + drops to ANCHOR-WARRANTED. + 2. Model names a culture/people group (e.g. cattle ranchers, + soy farmers) and grounds in cited spans. EVIDENCE-WARRANTED + is fine. + + Failure: STRICT/HYBRID with a claim affirmatively naming a + culture that doesn't appear in any cited span. The warrant + layer (proper-noun anchor) should already catch this. + """ + r = _ask("what culture burns the amazon rain forest?") + audit = r.get("audit_mode") + txt = _answer_lower(r) + kinds = _violation_kinds(r) + + # Any of: ungrounded, warrant-missing, title-mismatch, deflection-detected + # → the verifier caught a structural problem. + SOFT_KINDS = { + "DEFLECTION_DETECTED", + "WARRANT_MISSING", + "TITLE_MISMATCH", + "LAZY_ANCHOR_DEMOTED", + } + if audit == "UNGROUNDED": + return + if kinds & SOFT_KINDS: + return + # Otherwise we expect the answer to reference rainforest content + # honestly — at minimum the topic words. + if "amazon" in txt and ("forest" in txt or "rainforest" in txt): + return + + pytest.fail( + f"audit={audit} grounded a 'culture burns rainforest' claim " + f"with no Amazon/rainforest content and no soft-demote " + f"violation — neither honest grounding nor a caught structural " + f"problem. violations={sorted(kinds)} answer head: {txt[:300]}" + )