From 4d9a08f115acb8e16d2343f66ea782c63a91ebeb Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Mon, 1 Jun 2026 19:33:29 -0400 Subject: [PATCH] =?UTF-8?q?spotlight:=20density=20=C3=97=20rarity=20rank?= =?UTF-8?q?=20(rare=20anchor=20wins=20over=20repeated=20common=20tokens)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure density ranking ("count distinct tokens within ±half-window") loses on Sonic-shaped chunks: "Sonic" appears 20+ times, "1990" appears once, so any Sonic-anchored slice scores 4 distinct tokens while the one 1990-anchored slice scores 2. The 1990 slice IS the load-bearing claim evidence; the Sonic slice is just where the chunk happens to lead. Fox flagged 2026-06-01: STRICT-verified claim "Sonic the Hedgehog was created in 1990" anchored to chunk E5 (franchise timeline), but the spotlit excerpt showed the AM8/Sega-Team paragraph, not the "April 1990" year entry where the date claim grounds. Fix: multiply density by 1/freq(anchor_token) in the span. A token appearing once gets full weight; a token appearing 20× gets 0.05. Rare anchors (years, named entities, version numbers) now win even with fewer neighbors in the window — the rare token marks the slice the verifier actually verified. Sample: synthetic Sonic-shaped chunk + claim "created in 1990" → excerpt now centers on "In 1990, Sega began developing the Sonic the Hedgehog character." (was the Sonic Team / AM8 paragraph). Live: `make query Q="when was sonic the hedgehog created?"` → spotlight now leads with "In April 1990, Sega commissioned its AM8 R&D department to create a character…" Tie-break stays smallest-idx (deterministic). 175 claim_lattice + verify + inspect tests pass. --- arborist/qa/evidence.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/arborist/qa/evidence.py b/arborist/qa/evidence.py index 4b1b9d2..60a0041 100644 --- a/arborist/qa/evidence.py +++ b/arborist/qa/evidence.py @@ -437,19 +437,30 @@ def _spotlight_excerpt(claim_text: str, span: str, *, window: int) -> str: suffix = "..." if new_end < len(span) else "" return f"{span[new_start:new_end]}{suffix}" - # Density rank: for each candidate position, count distinct - # tokens whose match positions fall within ±half. Pick the - # position with max distinct count; tie-break on smallest idx - # (deterministic, reproducible). - best_score = -1 + # Density × rarity rank. For each candidate position, count + # distinct tokens whose match positions fall within ±half AND + # weight by 1/freq(anchor_token) in the span. A rare anchor + # (year, named entity, version number) appearing once in a + # Sonic-article-length chunk wins over a 20×-repeated "sonic" + # — the rare token marks the load-bearing slice for the claim. + # Without this, "Sonic the Hedgehog was created in 1990" cited + # to the franchise-timeline chunk spotlit a Sonic-Team paragraph + # instead of the "1990" year entry (2026-06-01 fox report). + # Tie-break on smallest idx (deterministic, reproducible). + freq: dict[str, int] = {} + for tok in tokens: + # Count overlapping occurrences using positions we already + # collected — same definition the density count uses. + freq[tok] = sum(1 for t, _ in positions if t == tok) or 1 + best_score = -1.0 match_idx = positions[0][1] - for _tok, idx in positions: + for tok, idx in positions: distinct: set[str] = set() lo, hi = idx - half, idx + half for tok2, idx2 in positions: if lo <= idx2 <= hi: distinct.add(tok2) - score = len(distinct) + score = len(distinct) / freq[tok] if score > best_score or (score == best_score and idx < match_idx): best_score = score match_idx = idx