From 873cfa8b31f75527c6c7eeaa22e6211d08df02f2 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Thu, 30 Apr 2026 12:36:59 -0400 Subject: [PATCH] qa: bare-name guard, lazy-anchor demote, game tie-in noisy markers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three layered fixes for the JP-dinosaurs lazy-anchor failure mode where "what dinosaurs were in the first jurassic park film?" returned HYBRID 5/8 with citations to a video-game tie-in chunk (Operation Genesis): 1. Bare-name claim guard. New policy field `claim_lattice_min_claim_content_tokens` (default 2). Claims with fewer content tokens (>=4 chars, post-spotlight-stopword) classify as SCHEMA_INVALID. "Triceratops. [E16]" had 1 content token; lexical overlap on a single token is too thin to verify "X is in the FIRST film" semantics. Forcing sentence shape raises the coverage bar so off-topic chunks can no longer satisfy the citation. Threshold of 2 keeps narrow factoids viable (e.g. "Steve Jobs co-founded Apple" tokenizes to 3 content tokens after stopword strip). 2. Smell → demote. The lazy-anchor smell sidecar was advisory only; the verdict could be STRICT while every claim cited the same magnet chunk. New policy fields `claim_lattice_lazy_anchor_demote_threshold` (0.5) and `claim_lattice_lazy_anchor_demote_min_pairs` (3) cap audit_mode at HYBRID when ratio + pair-count both met. STRICT now requires diverse anchoring across pointers; emits a LAZY_ANCHOR_DEMOTE violation record so the auditor can see what fired. 3. Noisy title markers extended. `_NOISY_TITLE_MARKERS` now includes "operation genesis", "the game", "video games" so spinoff/tie-in articles (Jurassic Park: Operation Genesis, X (NES game), etc.) classify as noisy_background_source. Default claim_lattice_allowed_source_roles excludes that role, so noisy chunks never enter the evidence map for pointer-mode queries. Brittle (case-by-case markers) but addresses the JP failure directly. Live JP-dinosaurs after fix: UNGROUNDED 0/11 (was HYBRID 5/8 with bogus citations). Honest refusal beats false confidence — model could recover by writing sentence-shape claims; bare-name shortcut blocked. Test fixtures using minimal placeholder claims ("Trex appears. [E1]") opt out of the bare-name check via `min_claim_content_tokens=0` since those tests exercise orthogonal behaviors. 460 tests pass. --- aborist/qa/query.py | 22 +++++++++++-- aborist/qa/runner.py | 28 +++++++++++++++++ aborist/qa/verify.py | 61 +++++++++++++++++++++++++++++++++++++ tests/test_claim_lattice.py | 26 +++++++++++----- 4 files changed, 128 insertions(+), 9 deletions(-) diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 2510c18..5b39945 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -381,6 +381,9 @@ DEFAULT_QUERY_POLICY = { # cached records. "claim_lattice_max_chunks_per_source": 2, "claim_lattice_min_citation_coverage": 0.30, + "claim_lattice_min_claim_content_tokens": 2, + "claim_lattice_lazy_anchor_demote_threshold": 0.5, + "claim_lattice_lazy_anchor_demote_min_pairs": 3, # JSON variant — `answer_mode="claim_lattice"`. Pairs with grammar- # constrained inference (vLLM guided_json, Claude/GPT-4 native # JSON, Qwen 3.6 reasoner). Lenient pre-parser keeps the path @@ -444,9 +447,15 @@ SOURCE_ROLE_BUDGET_WEIGHTS = { } # Title patterns that demote a source's role. Lower-cased substring match. +# 2026-04-30: extended to catch tie-in spinoff titles. The JP-dinosaurs +# query lazy-anchored on "Jurassic Park: Operation Genesis" (a video game) +# whose enumerative dinosaur tables pattern-matched the question shape +# more cleanly than the actual film article's prose. Adding the explicit +# game subtitle plus generic markers ("the game", "video games") so +# similar tie-ins classify as noisy and drop out of the evidence map. _NOISY_TITLE_MARKERS = ( - "score", "music", "soundtrack", "video game", "merchandise", - "discography", + "score", "music", "soundtrack", "video game", "video games", + "merchandise", "discography", "operation genesis", "the game", ) _SEQUEL_TITLE_MARKERS = ( " iii", " ii)", " ii ", " iv", " v ", " v)", "lost world", "sequel", @@ -1526,6 +1535,15 @@ def query( min_citation_coverage=float(policy.get( "claim_lattice_min_citation_coverage", 0.30 )), + min_claim_content_tokens=int(policy.get( + "claim_lattice_min_claim_content_tokens", 3 + )), + lazy_anchor_demote_threshold=float(policy.get( + "claim_lattice_lazy_anchor_demote_threshold", 0.5 + )), + lazy_anchor_demote_min_pairs=int(policy.get( + "claim_lattice_lazy_anchor_demote_min_pairs", 3 + )), ) rendered = verdict["rendered_text"] answer_text = rendered if rendered else raw_answer diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index 3ccf93b..bfcf756 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -204,6 +204,25 @@ DEFAULT_POLICY = { # like "Steve Jobs co-founded Apple" still pass. Folds into # governance_policy_hash on change. "claim_lattice_min_citation_coverage": 0.30, + # Bare-name claim guard. A claim with fewer than this many content + # tokens (>=4 chars, post-spotlight-stopword) is rejected as + # SCHEMA_INVALID. Catches the JP-dinosaurs lazy-anchor where + # "Triceratops. [E16]" passes lexical overlap on a single token + # even when E16 is a video-game tie-in chunk rather than the film + # article. Default 2: bare-entity-name claims (one content token + # after stopword strip) fail; sentence-shape claims pass. Note + # ``_content_tokens`` already filters "appears", "shown", etc. so + # "Trex appears" → 1 content token (filtered), "Trex appears in + # the film" → 2 content tokens (passes). Folds into + # governance_policy_hash on change. + "claim_lattice_min_claim_content_tokens": 2, + # Lazy-anchor smell auto-demote. When >= threshold of verified + # pointer-pairs cite a single pointer AND there are >= min_pairs + # total, cap audit_mode at HYBRID. The smell sidecar was advisory + # only pre-2026-04-30; now it's load-bearing. STRICT requires + # diverse anchoring across pointers. + "claim_lattice_lazy_anchor_demote_threshold": 0.5, + "claim_lattice_lazy_anchor_demote_min_pairs": 3, # JSON variant — `answer_mode="claim_lattice"`. Mirrors the # multi-source query path. Pairs with grammar-constrained inference # (vLLM guided_json, Claude/GPT-4 native JSON, Qwen 3.6 reasoner). @@ -517,6 +536,15 @@ def ask( min_citation_coverage=float(policy.get( "claim_lattice_min_citation_coverage", 0.30 )), + min_claim_content_tokens=int(policy.get( + "claim_lattice_min_claim_content_tokens", 3 + )), + lazy_anchor_demote_threshold=float(policy.get( + "claim_lattice_lazy_anchor_demote_threshold", 0.5 + )), + lazy_anchor_demote_min_pairs=int(policy.get( + "claim_lattice_lazy_anchor_demote_min_pairs", 3 + )), ) # Rendered prose (literal spans interpolated) is the user-facing # answer text — never the model's raw pointer-line output. If diff --git a/aborist/qa/verify.py b/aborist/qa/verify.py index 92ecb89..c64eecd 100644 --- a/aborist/qa/verify.py +++ b/aborist/qa/verify.py @@ -825,6 +825,9 @@ def _claim_textually_overlaps_evidence( DEFAULT_MAX_POINTERS_PER_CLAIM = 2 +DEFAULT_MIN_CLAIM_CONTENT_TOKENS = 2 +DEFAULT_LAZY_ANCHOR_DEMOTE_THRESHOLD = 0.5 +DEFAULT_LAZY_ANCHOR_DEMOTE_MIN_PAIRS = 3 def verify_claim_lattice( @@ -834,6 +837,9 @@ def verify_claim_lattice( allowed_source_roles: tuple[str, ...] = DEFAULT_ALLOWED_SOURCE_ROLES, max_pointers_per_claim: int = DEFAULT_MAX_POINTERS_PER_CLAIM, min_citation_coverage: float = DEFAULT_MIN_CITATION_COVERAGE, + min_claim_content_tokens: int = DEFAULT_MIN_CLAIM_CONTENT_TOKENS, + lazy_anchor_demote_threshold: float = DEFAULT_LAZY_ANCHOR_DEMOTE_THRESHOLD, + lazy_anchor_demote_min_pairs: int = DEFAULT_LAZY_ANCHOR_DEMOTE_MIN_PAIRS, ) -> dict: """Deterministic verifier for ``answer_mode="claim_lattice_pointer"``. @@ -974,6 +980,38 @@ def verify_claim_lattice( evidence_id_pairs.append([]) continue + # Bare-name claim guard. A claim like "Tyrannosaurus rex. [E15]" + # has 1 content token; the citation passes any span that mentions + # T-rex anywhere, even when E15 is a video-game-behavior chunk + # rather than a film-context one. Forcing a sentence-shape claim + # ("Tyrannosaurus rex appeared in the first JP film") raises the + # token-coverage bar so an off-topic chunk can no longer satisfy + # the citation. Folds into governance_policy_hash via + # ``claim_lattice_min_claim_content_tokens``. + from aborist.qa.evidence import _content_tokens as _ct + claim_content_tokens = _ct(claim_text) + if len(claim_content_tokens) < min_claim_content_tokens: + violations.append({ + "kind": "SCHEMA_INVALID", + "claim_idx": idx, + "reason": ( + f"bare-name claim ({len(claim_content_tokens)} content " + f"tokens < {min_claim_content_tokens}); write a sentence" + ), + }) + claim_statuses.append({ + "claim_idx": idx, + "text": claim_text, + "pointer_ids": pointer_ids, + "evidence_ids": [], + "status": "SCHEMA_INVALID", + "reasons": ["bare-name claim — write a sentence with predicate"], + }) + n_pairs += len(pointer_ids) + evidence_id_pairs.append([]) + unverified.append(claim_text) + continue + # Pointer-count cap (Rule 9). Catches the encyclopedic-mega- # claim where the model produces one line citing every # pointer at once. Counts every pointer toward the denominator @@ -1115,6 +1153,28 @@ def verify_claim_lattice( else 0.0 ) + # Smell → demote. Pre-2026-04-30 the lazy-anchor signal was advisory + # only; the verdict could still be STRICT while every claim cited + # the same magnet chunk. Now: when ratio ≥ threshold AND total pairs + # ≥ floor, cap audit_mode at HYBRID. STRICT becomes unreachable for + # answers where one pointer carries every claim — that pattern is + # almost never honest verbatim grounding. UNGROUNDED is left alone + # (a verdict with zero verified pairs has no smell to flag). + lazy_anchor_demoted = False + if ( + audit_mode == "STRICT" + and total_pointers >= lazy_anchor_demote_min_pairs + and lazy_anchor_ratio >= lazy_anchor_demote_threshold + ): + audit_mode = "HYBRID" + lazy_anchor_demoted = True + violations.append({ + "kind": "LAZY_ANCHOR_DEMOTE", + "ratio": round(lazy_anchor_ratio, 3), + "min_pairs": lazy_anchor_demote_min_pairs, + "threshold": lazy_anchor_demote_threshold, + }) + return { "n_quotes": n_pairs, "n_verified": n_pairs_verified, @@ -1128,6 +1188,7 @@ def verify_claim_lattice( "evidence_id_pairs": evidence_id_pairs, "pointer_id_distribution": pointer_distribution, "lazy_anchor_ratio": lazy_anchor_ratio, + "lazy_anchor_demoted": lazy_anchor_demoted, } diff --git a/tests/test_claim_lattice.py b/tests/test_claim_lattice.py index 1586217..7f9994b 100644 --- a/tests/test_claim_lattice.py +++ b/tests/test_claim_lattice.py @@ -264,7 +264,9 @@ def test_no_evidence_pointer_downgrades(): "Trex appears in the film.\n" # no pointer "Velociraptors are shown. [E2]\n" ) - v = verify_claim_lattice(answer, em) + # Opt out of bare-name check (Velociraptors stems to 1 content token + # after spotlight stopword strip); test exercises NO_EVIDENCE_POINTER. + v = verify_claim_lattice(answer, em, min_claim_content_tokens=0) # Mixed: one verified, one NO_EVIDENCE_POINTER → HYBRID. assert v["audit_mode"] == "HYBRID" assert any(viol["kind"] == "NO_EVIDENCE_POINTER" for viol in v["violations"]) @@ -279,7 +281,9 @@ def test_partial_resolution_yields_hybrid(): "Tyrannosaurus rex appears. [E1]\n" "Made-up. [E99]\n" ) - v = verify_claim_lattice(answer, em) + # Test exercises mixed-pointer-resolution behavior; bare-name check + # is orthogonal so opt out. + v = verify_claim_lattice(answer, em, min_claim_content_tokens=0) assert v["audit_mode"] == "HYBRID" assert v["n_verified"] == 1 assert v["n_quotes"] == 2 @@ -288,7 +292,7 @@ def test_partial_resolution_yields_hybrid(): def test_partial_pointer_within_claim_yields_partial(): em = build_evidence_map(_sample_chunks()) answer = "Tyrannosaurus rex appears. [E1,E99]\n" - v = verify_claim_lattice(answer, em) + v = verify_claim_lattice(answer, em, min_claim_content_tokens=0) assert v["audit_mode"] == "HYBRID" assert v["claim_statuses"][0]["status"] == "EVIDENCE_LINKED_PARTIAL" # Renderer keeps only resolved ids. @@ -895,7 +899,10 @@ def test_lazy_anchor_ratio_one_when_all_share_pointer(): "Tyrannosaurus rex roars. [E1]\n" "Tyrannosaurus rex broke out. [E1]\n" ) - v = verify_claim_lattice(answer, em) + # Test exercises the smell ratio counter; bare-name check is + # orthogonal so opt out so all three claims reach EVIDENCE_LINKED + # and populate pointer_distribution. + v = verify_claim_lattice(answer, em, min_claim_content_tokens=0) assert v["lazy_anchor_ratio"] == 1.0 assert v["pointer_id_distribution"] == {"E1": 3} @@ -908,7 +915,7 @@ def test_lazy_anchor_ratio_one_third_when_diversified(): "Velociraptors stalk workers. [E2]\n" "The climactic Tyrannosaurus moment. [E1]\n" ) - v = verify_claim_lattice(answer, em) + v = verify_claim_lattice(answer, em, min_claim_content_tokens=0) # 2 cite E1, 1 cites E2 → max share 2/3. assert v["pointer_id_distribution"] == {"E1": 2, "E2": 1} assert abs(v["lazy_anchor_ratio"] - 2 / 3) < 1e-9 @@ -942,7 +949,9 @@ def test_citation_mismatch_partial_when_one_pointer_overlaps(): # E1 contains "Tyrannosaurus rex"; E2 contains "Velociraptors". # Claim mentions Tyrannosaurus → E1 overlaps, E2 does not. answer = "Tyrannosaurus rex appears. [E1,E2]\n" - v = verify_claim_lattice(answer, em) + # Test exercises CITATION_MISMATCH on a single pointer; bare-name + # check is orthogonal so opt out. + v = verify_claim_lattice(answer, em, min_claim_content_tokens=0) assert v["audit_mode"] == "HYBRID" assert v["n_verified"] == 1 assert v["claim_statuses"][0]["status"] == "EVIDENCE_LINKED_PARTIAL" @@ -962,7 +971,10 @@ def test_pure_stopword_claim_passes_overlap_check_vacuously(): # below has effectively no content tokens after stopword filter # (just function words), so the overlap check shouldn't fail it. answer = "It was so. [E1]\n" - v = verify_claim_lattice(answer, em) + # Bare-name check would reject this for 0 content tokens; opt out + # since this test is specifically about the overlap rule's + # vacuous-True branch for claims with no content tokens. + v = verify_claim_lattice(answer, em, min_claim_content_tokens=0) # Overlap check passed; claim is short but valid. assert v["claim_statuses"][0]["status"] == "EVIDENCE_LINKED"