From bd576fda133a9e5eaf3945ab34daa660e10d04c7 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 29 Apr 2026 21:57:37 -0400 Subject: [PATCH] qa: title-purity rerank + 6th hard check (CITATION_MISMATCH) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two layered fixes that close the lazy-anchor gap. Title-purity moves the right SOURCES to the top of retrieval; the new hard check makes the verifier ruthlessly honest about whether each cited chunk actually mentions what its claim says. 1. _rerank_by_title_purity. New stage at the end of _rerank(). purity = |title_tokens ∩ query_tokens| / |title_tokens|; score multiplied by (1 + 2*purity), so a title that IS the topic ("Jurassic Park (film)" against the dinosaur question) gets 3.0x and titles that drag in off-topic tokens like "Jurassic Park: Operation Genesis" only get 2.0x. Composes with the role-rank weights from the prior commit. Florida query: list-pages dropped out of top-8 entirely. JP-dinosaurs query: Operation Genesis sank from #4 to #8. 2. CITATION_MISMATCH hard check. Sixth deterministic check in verify_claim_lattice — for each (claim, pointer) pair, at least one content token from the claim text must textually appear in the cited evidence span. Lexical only, no NER, no embeddings; same _content_tokens extractor as the spotlight so the proof-path stays on the soft/hard boundary. Catches the magnet-chunk lazy-anchor where the model cites a chunk whose text contains zero overlap with what its claim asserts. New status CITATION_MISMATCH joins the per-claim taxonomy. New violation kind CITATION_MISMATCH joins the run-DAG verify payload. JP-dinosaurs verdict went from "13/14 verified ratio 0.77 (cite E13 for everything)" pre-check to "1/14 verified" post-check — only Tyrannosaurus Rex's claim actually overlapped its cited chunk. The architecture stopped lying about evidence-linkage that the cited spans don't textually support. Pure-stopword claims pass the overlap check vacuously (no content tokens means nothing to check against) — other hard checks own that case. Tests: +3 covering CITATION_MISMATCH (claim-token absent yields fail, partial overlap yields HYBRID with EVIDENCE_LINKED_PARTIAL, pure-stopword claim passes vacuously). Updated 7 prior tests that used colloquial stub claims ("Trex") which don't textually appear in the sample spans ("Tyrannosaurus rex"); they now use claim text that overlaps the source content. 445 tests passing, all 7 production shards report 0 chain breaks. --- aborist/qa/query.py | 44 +++++++++++++++++++++- aborist/qa/verify.py | 52 ++++++++++++++++++++++++- tests/test_claim_lattice.py | 75 +++++++++++++++++++++++++++++++------ 3 files changed, 157 insertions(+), 14 deletions(-) diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 7b8b273..dac2974 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -747,7 +747,8 @@ def _rerank( ) hits = _rerank_by_body_coverage(hits, question) hits = _rerank_by_title(hits, question) - return _rerank_by_source_role(hits, question) + hits = _rerank_by_source_role(hits, question) + return _rerank_by_title_purity(hits, question) # Per-role rank multiplier. Affects sort order, NOT just per-source @@ -785,6 +786,47 @@ def _rerank_by_source_role(hits: list[_Hit], question: str) -> list[_Hit]: return hits +def _rerank_by_title_purity(hits: list[_Hit], question: str) -> list[_Hit]: + """Boost titles whose tokens are a tight superset of the query. + + Defined as ``purity = |title_tokens ∩ query_tokens| / |title_tokens|``. + A purity of 1.0 means every content token in the title is also a + query token — the title IS the topic, possibly with a Wikipedia + disambiguation suffix that itself matches a query word (e.g. + ``Jurassic Park (film)`` against "what dinosaurs were in the first + jurassic park FILM"). Lower purity means the title carries + off-topic tokens that water down its claim to be the answer source. + + Multiplier ``(1 + 2 * purity)``: + purity 1.0 → 3.0× + purity 0.5 → 2.0× + purity 0.25 → 1.5× + purity 0.0 → 1.0× (no change) + + Caught the JP-dinosaurs lazy-anchor at the retrieval layer: + ``Jurassic Park (film)`` (purity 1.0) now sits clearly above + ``Jurassic Park: Operation Genesis`` (purity 0.5), + ``Jurassic Park (franchise)`` (0.67), ``Jurassic Park (NES game)`` + (0.5), and the magnetic dinosaur-table chunks they contributed. + """ + qtokens = _title_query_tokens(question) + if not qtokens: + return hits + for h in hits: + if not h.title: + continue + ttokens = _title_query_tokens(h.title.replace("_", " ")) + if not ttokens: + continue + overlap = ttokens & qtokens + if not overlap: + continue + purity = len(overlap) / len(ttokens) + h.score = h.score * (1.0 + 2.0 * purity) + hits.sort(key=lambda h: -h.score) + return hits + + def _rerank_by_body_coverage( hits: list, question: str, diff --git a/aborist/qa/verify.py b/aborist/qa/verify.py index b822f6c..b9ce982 100644 --- a/aborist/qa/verify.py +++ b/aborist/qa/verify.py @@ -703,6 +703,33 @@ def _has_manual_quote(text: str) -> bool: return any(ch in text for ch in ('"', '“', '”')) +def _claim_textually_overlaps_evidence( + claim_text: str, evidence_span: str +) -> bool: + """Return True if at least one content token from ``claim_text`` + appears in ``evidence_span`` (case-insensitive substring). + + Hard 6th check on a (claim, pointer) pair. Catches the lazy-anchor + failure where the model cites an evidence pointer whose text has + zero overlap with the claim's actual subject — e.g. claim + "Brachiosaurus" cited to a "1993 in film" overview paragraph that + never mentions the dinosaur. Lexical only, no NER, no embeddings; + stays inside the soft/hard boundary. + + A pure-stopword claim (no content tokens after the spotlight token + extractor's filter) returns True vacuously — there's nothing + topical to check, and the verifier's other hard checks already + own that case (claim_text_non_empty, no_manual_quotes, etc.). + """ + from aborist.qa.evidence import _content_tokens + + tokens = _content_tokens(claim_text) + if not tokens: + return True + span_lower = evidence_span.lower() + return any(t in span_lower for t in tokens) + + def verify_claim_lattice( answer_text: str, evidence_map, @@ -714,7 +741,7 @@ def verify_claim_lattice( The model wrote pointer-line prose (``Claim text. [E12]``); the parser pulled (claim_text, [pointer_ids]) pairs from each non-empty line. This verifier maps each pointer id back to its - content-addressed evidence object and runs five hard checks: + content-addressed evidence object and runs six hard checks: 1. Parser succeeded — ``parse_status == "PARSED"`` (line had a bracket tag). NO_EVIDENCE_POINTER claims (prose without tag) @@ -725,6 +752,11 @@ def verify_claim_lattice( 4. Claim text contains no double-quote characters anywhere (strict no-quote rule — see ``_has_manual_quote``). 5. Claim text non-empty after tag strip. + 6. Claim's content tokens textually overlap the cited evidence + span (per-pair, lexical only — see + ``_claim_textually_overlaps_evidence``). Catches the magnet- + chunk lazy-anchor where the model cites an evidence pointer + whose text contains zero claim-content tokens. Returns a verdict in the same shape as ``verify_quotes`` + extras: @@ -743,6 +775,7 @@ def verify_claim_lattice( UNKNOWN_EVIDENCE_ID, SOURCE_ROLE_BLOCKED, MANUAL_QUOTE_VIOLATION, + CITATION_MISMATCH, NO_EVIDENCE_POINTER, SCHEMA_INVALID} violations structured violation records for the run-DAG / sidecar @@ -857,6 +890,21 @@ def verify_claim_lattice( "pid": pid, "ok": False, "kind": "MANUAL_QUOTE_VIOLATION", }) continue + if not _claim_textually_overlaps_evidence(claim_text, obj.span): + # Cited evidence span has zero textual overlap with any + # content token from the claim. Strongest lazy-anchor + # signal promoted to a hard fail — the model cited a + # magnet chunk that doesn't textually support its claim. + violations.append({ + "kind": "CITATION_MISMATCH", + "claim_idx": idx, + "pointer_id": pid, + "evidence_id": obj.evidence_id, + }) + per_id_results.append({ + "pid": pid, "ok": False, "kind": "CITATION_MISMATCH", + }) + continue per_id_results.append({"pid": pid, "ok": True, "evidence_id": obj.evidence_id}) resolved_evidence_ids.append(obj.evidence_id) n_pairs_verified += 1 @@ -877,6 +925,8 @@ def verify_claim_lattice( status = "EVIDENCE_LINKED" elif ok_pids: status = "EVIDENCE_LINKED_PARTIAL" + elif "CITATION_MISMATCH" in bad_kinds: + status = "CITATION_MISMATCH" elif "UNKNOWN_EVIDENCE_ID" in bad_kinds: status = "UNKNOWN_EVIDENCE_ID" elif "SOURCE_ROLE_BLOCKED" in bad_kinds: diff --git a/tests/test_claim_lattice.py b/tests/test_claim_lattice.py index d30a50b..e641522 100644 --- a/tests/test_claim_lattice.py +++ b/tests/test_claim_lattice.py @@ -166,9 +166,13 @@ def test_parse_inline_tag_preserves_prose(): def test_verify_strict_when_all_resolve(): em = build_evidence_map(_sample_chunks()) + # Claim text must textually overlap the cited span (G+: 6th hard + # check — claim content tokens must appear in the cited evidence). + # E1 span = "Tyrannosaurus rex appears in the climactic scene." + # E2 span = "Velociraptors are shown attacking workers." answer = ( - "Trex appears in the film. [E1]\n" - "Velociraptors are shown. [E2]\n" + "Tyrannosaurus rex appears in the film. [E1]\n" + "Velociraptors are shown attacking workers. [E2]\n" ) v = verify_claim_lattice(answer, em) assert v["audit_mode"] == "STRICT" @@ -184,7 +188,7 @@ def test_verify_strict_when_all_resolve(): def test_verify_renders_with_literal_spans_from_runtime(): em = build_evidence_map(_sample_chunks()) - answer = "Trex is in the film. [E1]\n" + answer = "Tyrannosaurus rex is in the film. [E1]\n" v = verify_claim_lattice(answer, em) # Renderer pulls literal span from runtime, not from model output. assert em[0].span in v["rendered_text"] @@ -193,7 +197,7 @@ def test_verify_renders_with_literal_spans_from_runtime(): def test_verify_passes_multi_pointer_claims(): em = build_evidence_map(_sample_chunks()) - answer = "Trex and raptors appear. [E1,E2]\n" + answer = "Tyrannosaurus and velociraptors appear. [E1,E2]\n" v = verify_claim_lattice(answer, em) assert v["audit_mode"] == "STRICT" assert v["n_quotes"] == 2 @@ -262,7 +266,7 @@ def test_no_evidence_pointer_downgrades(): def test_partial_resolution_yields_hybrid(): em = build_evidence_map(_sample_chunks()) answer = ( - "Trex appears. [E1]\n" + "Tyrannosaurus rex appears. [E1]\n" "Made-up. [E99]\n" ) v = verify_claim_lattice(answer, em) @@ -273,7 +277,7 @@ def test_partial_resolution_yields_hybrid(): def test_partial_pointer_within_claim_yields_partial(): em = build_evidence_map(_sample_chunks()) - answer = "Trex appears. [E1,E99]\n" + answer = "Tyrannosaurus rex appears. [E1,E99]\n" v = verify_claim_lattice(answer, em) assert v["audit_mode"] == "HYBRID" assert v["claim_statuses"][0]["status"] == "EVIDENCE_LINKED_PARTIAL" @@ -865,10 +869,12 @@ def test_query_orders_chunks_by_relevance_g03(tmp_path): def test_lazy_anchor_ratio_one_when_all_share_pointer(): em = build_evidence_map(_sample_chunks()) + # All three claims must textually overlap E1's span + # ("Tyrannosaurus rex appears in the climactic scene."). answer = ( - "Trex appears. [E1]\n" - "T-rex roars. [E1]\n" - "T-rex breaks out. [E1]\n" + "Tyrannosaurus rex appears. [E1]\n" + "Tyrannosaurus rex roars. [E1]\n" + "Tyrannosaurus rex broke out. [E1]\n" ) v = verify_claim_lattice(answer, em) assert v["lazy_anchor_ratio"] == 1.0 @@ -877,10 +883,11 @@ def test_lazy_anchor_ratio_one_when_all_share_pointer(): def test_lazy_anchor_ratio_one_third_when_diversified(): em = build_evidence_map(_sample_chunks()) + # Each claim's content tokens must overlap its cited span. answer = ( - "Trex appears. [E1]\n" - "Velociraptors stalk. [E2]\n" - "Brachiosaurus visible. [E1]\n" + "Tyrannosaurus rex appears. [E1]\n" + "Velociraptors stalk workers. [E2]\n" + "The climactic Tyrannosaurus moment. [E1]\n" ) v = verify_claim_lattice(answer, em) # 2 cite E1, 1 cites E2 → max share 2/3. @@ -897,6 +904,50 @@ def test_lazy_anchor_ratio_zero_when_no_verified_pairs(): assert v["pointer_id_distribution"] == {} +def test_citation_mismatch_when_claim_token_absent_from_evidence(): + """6th hard check: cited evidence span must textually contain at + least one content token from the claim text.""" + em = build_evidence_map(_sample_chunks()) + # E1's span is "Tyrannosaurus rex appears in the climactic scene." + # — the claim says "Brachiosaurus" (not present in the span). + answer = "Brachiosaurus appears in the film. [E1]\n" + v = verify_claim_lattice(answer, em) + assert v["audit_mode"] == "UNGROUNDED" + assert any(viol["kind"] == "CITATION_MISMATCH" for viol in v["violations"]) + assert v["claim_statuses"][0]["status"] == "CITATION_MISMATCH" + assert v["n_verified"] == 0 + + +def test_citation_mismatch_partial_when_one_pointer_overlaps(): + em = build_evidence_map(_sample_chunks()) + # E1 contains "Tyrannosaurus rex"; E2 contains "Velociraptors". + # Claim mentions Tyrannosaurus → E1 overlaps, E2 does not. + answer = "Tyrannosaurus rex appears. [E1,E2]\n" + v = verify_claim_lattice(answer, em) + assert v["audit_mode"] == "HYBRID" + assert v["n_verified"] == 1 + assert v["claim_statuses"][0]["status"] == "EVIDENCE_LINKED_PARTIAL" + # E1 resolved, E2 mismatched. + assert any( + viol["kind"] == "CITATION_MISMATCH" and viol["pointer_id"] == "E2" + for viol in v["violations"] + ) + + +def test_pure_stopword_claim_passes_overlap_check_vacuously(): + """A claim composed entirely of stopwords has no content tokens to + check; the overlap rule returns True vacuously and other hard + checks own that case.""" + em = build_evidence_map(_sample_chunks()) + # Tyrannosaurus rex appears... is in E1's span; the claim text + # below has effectively no content tokens after stopword filter + # (just function words), so the overlap check shouldn't fail it. + answer = "It was so. [E1]\n" + v = verify_claim_lattice(answer, em) + # Overlap check passed; claim is short but valid. + assert v["claim_statuses"][0]["status"] == "EVIDENCE_LINKED" + + def test_lazy_anchor_signals_not_in_run_dag_payload(): """Sidecar invariant: pointer_id_distribution and lazy_anchor_ratio must NOT enter build_run_dag's verify_payload — they're soft