From 26e3af9033e6a7cfd6576d3bd518cf4b425874ab Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Fri, 1 May 2026 10:19:44 -0400 Subject: [PATCH] =?UTF-8?q?qa/inspect:=20title-relevance=20sidecar=20?= =?UTF-8?q?=E2=80=94=20flag=20retrieval-driven=20hallucinations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Empirical finding 2026-05-01 on 'explain spin glass modeling, tensors?': returned STRICT 1/1 with claim 'Spin glass modeling involves... mathematical tools such as tensors' cited to a chunk from the *Quantum chromodynamics* article. Token-coverage check inside the chunk passed accidentally on shared physics vocabulary (spin/model/etc.), but the article isn't about spin glasses — the claim is a retrieval-driven hallucination. Root retrieval cause: AND-mode FTS5 returned zero hits (Spin_glass article has 0 occurrences of 'modeling' or 'tensors'); OR-mode fallback ranked by raw BM25, so 'Glass' (glass*274) outranked 'Spin glass' (spin*29 + glass*15) despite Spin_glass matching 2 distinct query tokens vs Glass's 1. Verifier then accepted the hallucinated claim because the cited QCD chunk has incidental physics vocabulary overlap. New ``diagnose_title_relevance(claim_text, cited_source_titles)`` sidecar: claim's content tokens (stemmed) must overlap the union of cited source titles. Zero overlap → ``title_mismatch``. Catches the spin-tensors case (claim about spin glass, cited title 'Quantum chromodynamics' → zero stem overlap) without false-positives on legitimate cross-document grounding (Lois_Lane article cited for a Lois-Lane claim still has lois+lane overlap). Sidecar-only for now — staying out of the binary verifier output until bench evidence justifies hard-promotion. Not yet wired into inspect_cache_key (per-claim citation source mapping isn't in providence_cache yet) or the bench. Available as a utility for operators auditing specific (claim, source_title) pairs. 5 new unit tests covering match/mismatch/no-titles/no-claim-tokens and underscore-normalization for Wikipedia titles. --- aborist/qa/inspect.py | 60 +++++++++++++++++++++++++++++++++++++++ tests/test_inspect.py | 66 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/aborist/qa/inspect.py b/aborist/qa/inspect.py index 0a5292a..8385ef0 100644 --- a/aborist/qa/inspect.py +++ b/aborist/qa/inspect.py @@ -424,6 +424,66 @@ def _question_is_numeric_or_cause_shape(question_text: str) -> bool: return any(q.startswith(lead) for lead in _DEFLECTION_NUMERIC_OR_CAUSE_LEADERS) +def diagnose_title_relevance( + claim_text: str, + cited_source_titles: list[str], +) -> dict[str, Any]: + """Soft signal: do the cited chunk's source titles overlap the + claim's content tokens? + + Empirically observed 2026-05-01 on 'explain spin glass modeling, + tensors?' — verifier returned STRICT 1/1 on a claim 'Spin glass + modeling involves... mathematical tools such as tensors' cited + to a chunk from the *Quantum chromodynamics* article. The claim + has zero stemmed-token overlap with the source title 'Quantum + chromodynamics'. Token-coverage check inside the chunk passed + accidentally on shared physics vocabulary (spin/model/etc.), but + the article isn't about spin glasses. + + This sidecar surfaces the mismatch without modifying the binary + verifier output. ``kind`` of ``TITLE_MISMATCH`` means the cited + source's topic is structurally unrelated to the claim's subject; + the claim is likely a retrieval-driven hallucination grounded + against an incidentally-overlapping chunk. + + Returns: + { + "kind": "title_match" | "title_mismatch" | "no_claim_tokens" | "no_titles", + "claim_tokens": [...], + "title_tokens": [...], # union across all cited titles + "overlap": [...], + } + + Sidecar-only — never enters the binary verifier output. Title + underscores in titles ('Quantum_chromodynamics' / 'Lois_Lane') + are normalized to spaces before tokenization.""" + claim_set = set(_content_tokens_in_order(claim_text or "")) + if not claim_set: + return { + "kind": "no_claim_tokens", + "claim_tokens": [], + "title_tokens": [], + "overlap": [], + } + if not cited_source_titles: + return { + "kind": "no_titles", + "claim_tokens": sorted(claim_set), + "title_tokens": [], + "overlap": [], + } + title_set: set[str] = set() + for t in cited_source_titles: + title_set |= set(_content_tokens_in_order((t or "").replace("_", " "))) + overlap = claim_set & title_set + return { + "kind": "title_match" if overlap else "title_mismatch", + "claim_tokens": sorted(claim_set), + "title_tokens": sorted(title_set), + "overlap": sorted(overlap), + } + + def diagnose_deflection(question_text: str, answer_text: str) -> dict[str, Any]: """Soft signal: did the answer change topic? diff --git a/tests/test_inspect.py b/tests/test_inspect.py index 9c273f5..52170f3 100644 --- a/tests/test_inspect.py +++ b/tests/test_inspect.py @@ -22,6 +22,7 @@ from aborist.qa.inspect import ( _classify_span, _normalize, diagnose_deflection, + diagnose_title_relevance, inspect_cache_key, ) from aborist.store import append_audit, connect, transaction @@ -538,3 +539,68 @@ def test_deflection_not_suppressed_for_who_questions(): assert d["shape_suppressed"] is False assert d["kind"] == "deflection" assert d["subject_anchor"] == "mars" + + +# --------------------------------------------------------------------------- +# title-relevance sidecar +# --------------------------------------------------------------------------- + + +def test_title_mismatch_on_qcd_cited_for_spin_glass_claim(): + """Empirical 2026-05-01: 'explain spin glass modeling, tensors?' + returned STRICT 1/1 with claim 'Spin glass modeling involves... + mathematical tools such as tensors' cited to a chunk from + Quantum_chromodynamics. Claim tokens have zero overlap with QCD + title — sidecar should flag TITLE_MISMATCH so an operator can + see the retrieval-driven hallucination signature.""" + d = diagnose_title_relevance( + "Spin glass modeling involves using the concept of spin glasses, " + "which are disordered magnetic systems, to study complex systems " + "and phenomena in physics and other fields. Spin glasses are " + "characterized by random interactions. Tensors are used to " + "represent the interactions between spins.", + ["Quantum_chromodynamics"], + ) + assert d["kind"] == "title_mismatch" + assert d["overlap"] == [] + # Claim has 'spin', 'glass', 'modeling', etc. — title has 'quantum', + # 'chromodynamics'. Zero stem overlap. + assert "spin" in d["claim_tokens"] + assert "quantum" in d["title_tokens"] + + +def test_title_match_on_lois_lane_cited_for_supermans_girlfriend(): + """Sanity: legit citation (Lois Lane article cited for Lois-Lane + claim) passes title-relevance even though Superman isn't in the + title.""" + d = diagnose_title_relevance( + "Lois Lane is the longtime love interest of Superman.", + ["Lois_Lane"], + ) + assert d["kind"] == "title_match" + assert "lois" in d["overlap"] + assert "lane" in d["overlap"] + + +def test_title_match_underscores_normalized(): + """Wikipedia titles use underscores; normalization to spaces lets + multi-word titles tokenize correctly.""" + d = diagnose_title_relevance( + "The Berlin Wall fell in 1989.", + ["Berlin_Wall"], + ) + assert d["kind"] == "title_match" + assert "berlin" in d["overlap"] + assert "wall" in d["overlap"] + + +def test_title_mismatch_no_titles(): + """Defensive: empty title list returns no_titles.""" + d = diagnose_title_relevance("Some claim.", []) + assert d["kind"] == "no_titles" + + +def test_title_mismatch_no_claim_tokens(): + """Defensive: empty/stopword-only claim returns no_claim_tokens.""" + d = diagnose_title_relevance("the a an", ["Anything"]) + assert d["kind"] == "no_claim_tokens"