diff --git a/CLAUDE.md b/CLAUDE.md index 0eb0a62..34134ee 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -162,10 +162,23 @@ revert without reading why. When in doubt, walk the token in question must appear in answer). Suppressed for date / count / cause shapes ("when", "why", "how many"). See `aborist/qa/inspect.py`. -- **Title-relevance sidecar**: `diagnose_title_relevance(claim, - cited_titles)` flags retrieval-driven hallucinations where the - cited chunk's source title shares zero stems with the claim's - content tokens. See `aborist/qa/inspect.py`. +- **Title-relevance hard check (Rule 8)**: `_claim_title_overlap` + in `aborist/qa/verify.py`. For each claim that resolved, at least + one cited evidence's source title must share ≥1 stemmed content + token with the claim text. When NO cited title overlaps, record + a `TITLE_MISMATCH` violation & demote `STRICT → HYBRID`. Catches + retrieval-driven hallucinations where the cited chunk's SOURCE + is structurally unrelated to the claim's subject (2026-05-02 + spin-glass case: claim about spin glass cited to a chunk from + *Quantum chromodynamics*; span had incidental physics-vocab + overlap, but the source title shared zero stems with the claim). + Renderer surfaces a `· title mismatch` tail on the audit-line + label alongside `· warrant missing`. +- **Title-relevance sidecar (legacy diagnostic)**: + `diagnose_title_relevance(claim, cited_titles)` in + `aborist.qa.inspect` returns the same signal in dict form for + per-cache-key inspection. Sidecar; never enters proof path. + Pre-dates the Rule 8 promotion (2026-05-02). ## Live endpoints diff --git a/aborist/cli.py b/aborist/cli.py index e2e0e76..6086f3d 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -465,7 +465,11 @@ def _ladder_rung_for_lattice(audit_mode: str, violations: list[dict] | None) -> if audit_mode == "UNGROUNDED": return "UNGROUNDED" kinds = {v.get("kind") for v in (violations or [])} - if "WARRANT_MISSING" in kinds: + # WARRANT_MISSING and TITLE_MISMATCH both indicate the citation + # is structurally misaligned with the claim — pointer resolved + # but the cited evidence (span or source) doesn't actually + # support the claim. Both drop to POINTER-LINKED. + if "WARRANT_MISSING" in kinds or "TITLE_MISMATCH" in kinds: rung = "POINTER-LINKED" elif kinds & _SOFT_DEMOTE_VIOLATION_KINDS: rung = "ANCHOR-WARRANTED" @@ -528,20 +532,30 @@ def _render_audit_label( def _render_warrant_tail(result: dict) -> str: - """If the verdict carries any WARRANT_MISSING violations - (relation-shape question + claim names answer entity not in - cited span), return a ` · warrant missing` tail for the - audit-line label. Otherwise empty string. + """Append a tail to the audit-line label that names the specific + warrant failure mode when one fired. Surfaces the cap reason at + the user-facing layer without overloading audit_mode. - Surfaces fox's feedback-3 distinction ("pointer-linked but - relation warrant missing") at the user-facing layer without - overloading the audit_mode field. The verifier already capped - audit_mode at HYBRID for warrant-missing cases; this just - names the cap reason.""" + Two failure modes today: + - WARRANT_MISSING: relation/date/etc. anchor extracted from + claim doesn't appear in any cited span (per-span check). + - TITLE_MISMATCH: cited evidence's source title shares no + content tokens with the claim (per-source check). 2026-05-02 + spin-glass case: claim about spin glass cited to *Quantum + chromodynamics*. + + When both fire on different claims of the same answer, surface + both tails so the operator sees the full picture.""" violations = result.get("violations") or [] - if any(v.get("kind") == "WARRANT_MISSING" for v in violations): - return " · warrant missing" - return "" + kinds = {v.get("kind") for v in violations} + parts: list[str] = [] + if "WARRANT_MISSING" in kinds: + parts.append("warrant missing") + if "TITLE_MISMATCH" in kinds: + parts.append("title mismatch") + if not parts: + return "" + return " · " + " · ".join(parts) def _maybe_render_json_envelope_as_bullets(answer: str) -> str: diff --git a/aborist/qa/verify.py b/aborist/qa/verify.py index 31c9a39..07cbfc0 100644 --- a/aborist/qa/verify.py +++ b/aborist/qa/verify.py @@ -985,6 +985,43 @@ DEFAULT_LAZY_ANCHOR_DEMOTE_MIN_PAIRS = 3 DEFAULT_MAX_CLAIMS_PER_ANSWER = 12 +# Title-relevance check (Rule 8). Cited evidence's source title must +# share at least one stemmed content token with the claim text. +# Catches the retrieval-driven hallucination class fox surfaced +# 2026-05-02 on "explain spin glass modeling & tensors?": claim +# tokens {spin, glass, modeling, tensor, ...} cited to a chunk from +# the *Quantum chromodynamics* article whose title tokens are +# {quantum, chromodynamics} — zero overlap. Token-coverage check +# inside the chunk passed accidentally on shared physics vocabulary; +# the SOURCE was never about the claim's subject. +def _claim_title_overlap(claim_text: str, source_title: str | None) -> bool: + """Return True iff the source title shares ≥1 stemmed content + token with the claim text. Vacuous-pass when either side has no + extractable tokens (defensive — prevents the rule from firing + on degenerate inputs).""" + if not source_title or not claim_text: + return True + from aborist.qa.evidence import _content_tokens as _ct + + claim_tokens = _ct(claim_text) + title_tokens = _ct((source_title or "").replace("_", " ")) + if not claim_tokens or not title_tokens: + return True + + # Reuse the retrieval-side stem helper so possessive / plural + # collapse the same way ('movies' vs 'movie', 'simpsons' vs + # 'simpson'). Defined in qa/query.py to avoid an import cycle: + # inline a minimal copy here instead. + def _stem(t: str) -> str: + if len(t) > 4 and t.endswith("s") and not t.endswith("ss"): + return t[:-1] + return t + + claim_stems = {_stem(t) for t in claim_tokens} + title_stems = {_stem(t) for t in title_tokens} + return bool(claim_stems & title_stems) + + def verify_claim_lattice( answer_text: str, evidence_map, @@ -1396,6 +1433,47 @@ def verify_claim_lattice( if warrant_missing_claims and audit_mode == "STRICT": audit_mode = "HYBRID" + # Rule 8 — Title-relevance check. For each claim that resolved, + # at least one cited evidence's source title must share a + # stemmed content token with the claim. Catches the + # retrieval-driven hallucination class (2026-05-02 spin-glass + # case): claim about spin glass cited to a chunk from + # *Quantum chromodynamics* — token-coverage check passed on + # incidental physics vocabulary, but the SOURCE was never about + # the claim's subject. + title_mismatch_claims: list[int] = [] + for cs in claim_statuses: + if cs.get("status") not in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"): + continue + cited_eids = cs.get("evidence_ids") or [] + cited_titles = [ + obj.title + for eid in cited_eids + for obj in [evidence_map_by_evidence_id_local(evidence_map, eid)] + if obj is not None + ] + if not cited_titles: + continue + any_overlap = any( + _claim_title_overlap(cs.get("text") or "", t) + for t in cited_titles + ) + if not any_overlap: + title_mismatch_claims.append(cs.get("claim_idx")) + violations.append({ + "kind": "TITLE_MISMATCH", + "claim_idx": cs.get("claim_idx"), + "claim_text": (cs.get("text") or "")[:200], + "cited_titles": cited_titles, + "rationale": ( + "no cited source's title shares a content token " + "with the claim — pointer-linked but the cited " + "document is structurally unrelated to the claim" + ), + }) + if title_mismatch_claims and audit_mode == "STRICT": + audit_mode = "HYBRID" + return { "n_quotes": n_pairs, "n_verified": n_pairs_verified, @@ -1411,6 +1489,7 @@ def verify_claim_lattice( "lazy_anchor_ratio": lazy_anchor_ratio, "lazy_anchor_demoted": lazy_anchor_demoted, "warrant_missing_claim_idxs": warrant_missing_claims, + "title_mismatch_claim_idxs": title_mismatch_claims, } @@ -1750,6 +1829,43 @@ def verify_claim_lattice_json( if warrant_missing_claims and audit_mode == "STRICT": audit_mode = "HYBRID" + # Rule 8 — Title-relevance check (mirrors the pointer variant). + # See verify_claim_lattice for rationale (2026-05-02 spin-glass + # case). Demote-to-HYBRID semantics; JSON variant emits the same + # TITLE_MISMATCH violation kind & title_mismatch_claim_idxs field. + title_mismatch_claims: list[int] = [] + for cs in claim_statuses: + if cs.get("status") not in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"): + continue + cited_eids = cs.get("evidence_ids") or [] + cited_titles = [ + obj.title + for eid in cited_eids + for obj in [evidence_map_by_evidence_id_local(evidence_map, eid)] + if obj is not None + ] + if not cited_titles: + continue + any_overlap = any( + _claim_title_overlap(cs.get("text") or "", t) + for t in cited_titles + ) + if not any_overlap: + title_mismatch_claims.append(cs.get("claim_idx")) + violations.append({ + "kind": "TITLE_MISMATCH", + "claim_idx": cs.get("claim_idx"), + "claim_text": (cs.get("text") or "")[:200], + "cited_titles": cited_titles, + "rationale": ( + "no cited source's title shares a content token " + "with the claim — pointer-linked but the cited " + "document is structurally unrelated to the claim" + ), + }) + if title_mismatch_claims and audit_mode == "STRICT": + audit_mode = "HYBRID" + # Same `verifier_method` as the pointer variant ("claim_lattice") # so the providence_cache CHECK constraint accepts both. The mode # is disambiguated downstream via `answer_mode` on the run-DAG & @@ -1766,4 +1882,5 @@ def verify_claim_lattice_json( "evidence_id_pairs": evidence_id_pairs, "json_fixups": json_fixups, "warrant_missing_claim_idxs": warrant_missing_claims, + "title_mismatch_claim_idxs": title_mismatch_claims, } diff --git a/tests/test_claim_lattice.py b/tests/test_claim_lattice.py index 36e022e..97f00af 100644 --- a/tests/test_claim_lattice.py +++ b/tests/test_claim_lattice.py @@ -168,11 +168,15 @@ def test_verify_strict_when_all_resolve(): em = build_evidence_map(_sample_chunks()) # Claim text must textually overlap the cited span (G+: 6th hard # check — claim content tokens must appear in the cited evidence). + # AND at least one cited source's title must share a content + # token with the claim (Rule 8, post-2026-05-02): so we keep + # "film" in each claim to anchor against the + # `Jurassic Park (film)` title. # E1 span = "Tyrannosaurus rex appears in the climactic scene." # E2 span = "Velociraptors are shown attacking workers." answer = ( "Tyrannosaurus rex appears in the film. [E1]\n" - "Velociraptors are shown attacking workers. [E2]\n" + "Velociraptors are shown attacking workers in the film. [E2]\n" ) v = verify_claim_lattice(answer, em) assert v["audit_mode"] == "STRICT" @@ -201,7 +205,10 @@ def test_verify_renders_with_literal_spans_from_runtime(): def test_verify_passes_multi_pointer_claims(): em = build_evidence_map(_sample_chunks()) - answer = "Tyrannosaurus and velociraptors appear. [E1,E2]\n" + # Title is "Jurassic Park (film)" — claim must share a content + # token (Rule 8). Add "film" so the claim anchors against the + # source title. + answer = "Tyrannosaurus and velociraptors appear in the film. [E1,E2]\n" v = verify_claim_lattice(answer, em) assert v["audit_mode"] == "STRICT" assert v["n_quotes"] == 2 @@ -823,17 +830,19 @@ def test_per_chunk_evidence_map_query_path(tmp_path): src_conn = connect(shard) try: ingest_source(src_conn, FakeSource([ - Document(uri="t://chunked", content=txt, source_type="test", title="Chunked") + Document(uri="t://chunked", content=txt, source_type="test", title="Dinosaurs") ])) finally: src_conn.close() # Stub answer that hits all three pointer ids — proves the - # evidence map exposed >1 chunk to the model. + # evidence map exposed >1 chunk to the model. Each claim names + # "dinosaur" so Rule 8 (title-relevance) anchors against the + # source title "Dinosaurs". stub = StubClient(answer=( - "Brachiosaurus is a sauropod. [E1]\n" - "Velociraptor is a theropod. [E2]\n" - "T-rex is apex. [E3]\n" + "Brachiosaurus is a sauropod dinosaur. [E1]\n" + "Velociraptor is a theropod dinosaur. [E2]\n" + "T-rex is the apex dinosaur. [E3]\n" )) # Override the per-source chunk cap so all three chunks of this # single test source surface as E1/E2/E3. Production default caps diff --git a/tests/test_verify_json.py b/tests/test_verify_json.py index 280fb92..3090646 100644 --- a/tests/test_verify_json.py +++ b/tests/test_verify_json.py @@ -82,6 +82,7 @@ def _ev( span: str, role: str = "primary_answer_source", pointer_id: str | None = None, + title: str = "Test Doc", ) -> EvidenceObject: """Stub evidence object with deterministic eid for the test. @@ -89,12 +90,17 @@ def _ev( since 2026-04-30; if the test wants the verifier to resolve a citation, it must set ``pointer_id`` explicitly. ``evidence_id`` stays content-addressed for the cache/run-DAG handle. + + ``title`` defaults to "Test Doc" but tests using Rule 8 (title- + relevance, post-2026-05-02) should pass a title whose content + tokens overlap the claim text. Otherwise the verifier flags + TITLE_MISMATCH and demotes STRICT → HYBRID. """ return EvidenceObject( evidence_id=eid, source_root="00" * 32, document_uri="test://doc", - title="Test Doc", + title=title, chunk_idx=0, chunk_root="11" * 32, offset_start=0, @@ -108,13 +114,13 @@ def _ev( def test_verify_json_strict_when_all_claims_resolve(): evidence = [ - _ev("E1f8e4c2a", "Brachiosaurus appears in the Jurassic Park film as a herbivore.", pointer_id="E1"), - _ev("E2c9d7b3f", "Velociraptor is featured prominently throughout Jurassic Park.", pointer_id="E2"), + _ev("E1f8e4c2a", "Brachiosaurus appears in the Jurassic Park film as a herbivore.", pointer_id="E1", title="Jurassic Park (film)"), + _ev("E2c9d7b3f", "Velociraptor is featured prominently throughout Jurassic Park.", pointer_id="E2", title="Jurassic Park (film)"), ] answer = json.dumps({ "claims": [ {"text": "Brachiosaurus appears in the film", "evidence_ids": ["E1"]}, - {"text": "Velociraptor is featured", "evidence_ids": ["E2"]}, + {"text": "Velociraptor is featured in the film", "evidence_ids": ["E2"]}, ] }) v = verify_claim_lattice_json(answer, evidence) @@ -151,7 +157,7 @@ def test_verify_json_ungrounded_on_schema_invalid(): def test_verify_json_recovers_from_markdown_fence(): """JSON-fenced output still parses & verifies; fence fixup logged.""" evidence = [ - _ev("E1f8e4c2a", "Brachiosaurus appears in the Jurassic Park film as a herbivore.", pointer_id="E1"), + _ev("E1f8e4c2a", "Brachiosaurus appears in the Jurassic Park film as a herbivore.", pointer_id="E1", title="Jurassic Park (film)"), ] answer = ( '```json\n' @@ -199,6 +205,86 @@ def test_verify_json_blocks_disallowed_source_role(): assert v["audit_mode"] == "UNGROUNDED" +def test_claim_title_overlap_passes_when_title_shares_token(): + """Rule 8 helper: title shares ≥1 stemmed content token with claim.""" + from aborist.qa.verify import _claim_title_overlap + + assert _claim_title_overlap( + "Homer Simpson's boss is Mr. Burns.", + "Homer Simpson", + ) + assert _claim_title_overlap( + "Brachiosaurus appears in the Jurassic Park film.", + "Jurassic Park (film)", + ) + # Stem-aware: 'simpsons' (plural) collapses to 'simpson'. + assert _claim_title_overlap( + "The Simpsons family includes Homer.", + "Simpson family", + ) + + +def test_claim_title_overlap_fails_on_qcd_for_spin_glass(): + """Rule 8 helper: spin-glass case from 2026-05-02. Claim about + spin glass cited to Quantum chromodynamics → no token overlap.""" + from aborist.qa.verify import _claim_title_overlap + + assert not _claim_title_overlap( + "Spin glass modeling involves the use of mathematical tensors.", + "Quantum chromodynamics", + ) + + +def test_claim_title_overlap_vacuous_pass_on_empty(): + """Defensive: empty title or empty claim → vacuous pass.""" + from aborist.qa.verify import _claim_title_overlap + + assert _claim_title_overlap("Some claim here.", None) + assert _claim_title_overlap("Some claim here.", "") + assert _claim_title_overlap("", "Some Title") + + +def test_verify_json_title_mismatch_demotes_to_hybrid(): + """End-to-end Rule 8: claim cited to a source with no title-token + overlap → TITLE_MISMATCH violation, audit_mode demoted to HYBRID. + + Span is intentionally rich with claim tokens (covers Rule 5 + citation-coverage threshold) so the failure path is Rule 8 + cleanly, not earlier rule rejection.""" + evidence = [ + # Title is QCD; claim is about spin glass modeling. Span + # contains enough claim tokens to clear Rule 5 (>=30%). + _ev( + "Eed1b6e39", + ( + "Spin glasses are disordered magnetic systems. " + "The spin glass modeling literature uses tensors to " + "represent interactions between magnetic moments. " + "Tensor methods illuminate the mathematical structure." + ), + pointer_id="E1", + title="Quantum chromodynamics", + ), + ] + answer = json.dumps({ + "claims": [ + { + "text": ( + "Spin glass modeling involves the use of " + "mathematical tensors to represent interactions." + ), + "evidence_ids": ["E1"], + } + ] + }) + v = verify_claim_lattice_json(answer, evidence) + assert v["audit_mode"] == "HYBRID", ( + f"expected HYBRID via TITLE_MISMATCH; got {v['audit_mode']} " + f"with violations {[v['kind'] for v in v['violations']]}" + ) + assert any(vio["kind"] == "TITLE_MISMATCH" for vio in v["violations"]) + + def test_verify_json_too_many_claims_demotes_to_hybrid(): """York-england shape: 13 claims (cap default = 12) trips TOO_MANY_CLAIMS even when each individual claim verifies. Demotes @@ -212,7 +298,9 @@ def test_verify_json_too_many_claims_demotes_to_hybrid(): pid = f"E{i}" eid = f"E{i:08x}c2a" text = f"York fact number {i} is described in this span." - evidence.append(_ev(eid, text, pointer_id=pid)) + # Title overlaps "york" so Rule 8 passes; this test exercises + # the TOO_MANY_CLAIMS cap, not title-relevance. + evidence.append(_ev(eid, text, pointer_id=pid, title="York facts")) claims.append({"text": text, "evidence_ids": [pid]}) answer = json.dumps({"claims": claims}) v = verify_claim_lattice_json(answer, evidence) @@ -234,7 +322,8 @@ def test_verify_json_at_cap_can_still_strict(): pid = f"E{i}" eid = f"E{i:08x}c2a" text = f"York fact number {i} appears in span." - evidence.append(_ev(eid, text, pointer_id=pid)) + # Title shares "york" with each claim → Rule 8 passes. + evidence.append(_ev(eid, text, pointer_id=pid, title="York facts")) claims.append({"text": text, "evidence_ids": [pid]}) answer = json.dumps({"claims": claims}) v = verify_claim_lattice_json(answer, evidence)