diff --git a/aborist/qa/inspect.py b/aborist/qa/inspect.py index 0a5292a..8385ef0 100644 --- a/aborist/qa/inspect.py +++ b/aborist/qa/inspect.py @@ -424,6 +424,66 @@ def _question_is_numeric_or_cause_shape(question_text: str) -> bool: return any(q.startswith(lead) for lead in _DEFLECTION_NUMERIC_OR_CAUSE_LEADERS) +def diagnose_title_relevance( + claim_text: str, + cited_source_titles: list[str], +) -> dict[str, Any]: + """Soft signal: do the cited chunk's source titles overlap the + claim's content tokens? + + Empirically observed 2026-05-01 on 'explain spin glass modeling, + tensors?' — verifier returned STRICT 1/1 on a claim 'Spin glass + modeling involves... mathematical tools such as tensors' cited + to a chunk from the *Quantum chromodynamics* article. The claim + has zero stemmed-token overlap with the source title 'Quantum + chromodynamics'. Token-coverage check inside the chunk passed + accidentally on shared physics vocabulary (spin/model/etc.), but + the article isn't about spin glasses. + + This sidecar surfaces the mismatch without modifying the binary + verifier output. ``kind`` of ``TITLE_MISMATCH`` means the cited + source's topic is structurally unrelated to the claim's subject; + the claim is likely a retrieval-driven hallucination grounded + against an incidentally-overlapping chunk. + + Returns: + { + "kind": "title_match" | "title_mismatch" | "no_claim_tokens" | "no_titles", + "claim_tokens": [...], + "title_tokens": [...], # union across all cited titles + "overlap": [...], + } + + Sidecar-only — never enters the binary verifier output. Title + underscores in titles ('Quantum_chromodynamics' / 'Lois_Lane') + are normalized to spaces before tokenization.""" + claim_set = set(_content_tokens_in_order(claim_text or "")) + if not claim_set: + return { + "kind": "no_claim_tokens", + "claim_tokens": [], + "title_tokens": [], + "overlap": [], + } + if not cited_source_titles: + return { + "kind": "no_titles", + "claim_tokens": sorted(claim_set), + "title_tokens": [], + "overlap": [], + } + title_set: set[str] = set() + for t in cited_source_titles: + title_set |= set(_content_tokens_in_order((t or "").replace("_", " "))) + overlap = claim_set & title_set + return { + "kind": "title_match" if overlap else "title_mismatch", + "claim_tokens": sorted(claim_set), + "title_tokens": sorted(title_set), + "overlap": sorted(overlap), + } + + def diagnose_deflection(question_text: str, answer_text: str) -> dict[str, Any]: """Soft signal: did the answer change topic? diff --git a/tests/test_inspect.py b/tests/test_inspect.py index 9c273f5..52170f3 100644 --- a/tests/test_inspect.py +++ b/tests/test_inspect.py @@ -22,6 +22,7 @@ from aborist.qa.inspect import ( _classify_span, _normalize, diagnose_deflection, + diagnose_title_relevance, inspect_cache_key, ) from aborist.store import append_audit, connect, transaction @@ -538,3 +539,68 @@ def test_deflection_not_suppressed_for_who_questions(): assert d["shape_suppressed"] is False assert d["kind"] == "deflection" assert d["subject_anchor"] == "mars" + + +# --------------------------------------------------------------------------- +# title-relevance sidecar +# --------------------------------------------------------------------------- + + +def test_title_mismatch_on_qcd_cited_for_spin_glass_claim(): + """Empirical 2026-05-01: 'explain spin glass modeling, tensors?' + returned STRICT 1/1 with claim 'Spin glass modeling involves... + mathematical tools such as tensors' cited to a chunk from + Quantum_chromodynamics. Claim tokens have zero overlap with QCD + title — sidecar should flag TITLE_MISMATCH so an operator can + see the retrieval-driven hallucination signature.""" + d = diagnose_title_relevance( + "Spin glass modeling involves using the concept of spin glasses, " + "which are disordered magnetic systems, to study complex systems " + "and phenomena in physics and other fields. Spin glasses are " + "characterized by random interactions. Tensors are used to " + "represent the interactions between spins.", + ["Quantum_chromodynamics"], + ) + assert d["kind"] == "title_mismatch" + assert d["overlap"] == [] + # Claim has 'spin', 'glass', 'modeling', etc. — title has 'quantum', + # 'chromodynamics'. Zero stem overlap. + assert "spin" in d["claim_tokens"] + assert "quantum" in d["title_tokens"] + + +def test_title_match_on_lois_lane_cited_for_supermans_girlfriend(): + """Sanity: legit citation (Lois Lane article cited for Lois-Lane + claim) passes title-relevance even though Superman isn't in the + title.""" + d = diagnose_title_relevance( + "Lois Lane is the longtime love interest of Superman.", + ["Lois_Lane"], + ) + assert d["kind"] == "title_match" + assert "lois" in d["overlap"] + assert "lane" in d["overlap"] + + +def test_title_match_underscores_normalized(): + """Wikipedia titles use underscores; normalization to spaces lets + multi-word titles tokenize correctly.""" + d = diagnose_title_relevance( + "The Berlin Wall fell in 1989.", + ["Berlin_Wall"], + ) + assert d["kind"] == "title_match" + assert "berlin" in d["overlap"] + assert "wall" in d["overlap"] + + +def test_title_mismatch_no_titles(): + """Defensive: empty title list returns no_titles.""" + d = diagnose_title_relevance("Some claim.", []) + assert d["kind"] == "no_titles" + + +def test_title_mismatch_no_claim_tokens(): + """Defensive: empty/stopword-only claim returns no_claim_tokens.""" + d = diagnose_title_relevance("the a an", ["Anything"]) + assert d["kind"] == "no_claim_tokens"