From d608c39880ef52a272d786cbe230dcc350b4e8b7 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Mon, 27 Apr 2026 19:30:48 -0400 Subject: [PATCH] fix retrieval recall: title-search no longer dominates top-K MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two structural defects in the corpus retrieval path were causing the Intel-CPU query (and any query whose terms hit many article titles generically) to surface 80486-era legacy articles instead of contemporary topic articles: 1. _search_titles entries arrived with a 50-point baseline score (50 + 10*overlap) plus _rerank_by_title doubled the overlap bonus on top. A single-token title match against a generic word like "intel" landed at score 70-80 standalone, swamping FTS5 BM25 hits that scored 5-15. Result: every Intel_*-prefixed article in the corpus rose to the top of every "intel" query, regardless of whether the article was actually about a fast CPU. Drop the 50-point baseline. Title overlap now scores `10*overlap` from this path, with `_rerank_by_title` adding another `10*overlap` for any hit (FTS5 or title) whose title overlaps query tokens. Net effect: title-only-overlap=1 hits score ~10-20, FTS5 hits with title-overlap score ~25-45, FTS5-only-no-title-overlap score ~5-25. Body relevance now beats single-token title coincidence. 2. _body_density_passes counted ANY query token's mentions toward `min_mentions`, so Intel_8086 (with 30 "Intel" mentions and zero "fastest" or "cpu" mentions) passed the gate for a query of {intel, fastest, cpu}. Tighten to require breadth: at least half the query tokens must each appear at least once, AND total mentions must clear min_mentions. For 1-token queries this is identical to the prior behavior. Verified end-to-end on the fresh 2010-11 enwiki corpus (3.47M docs, no distill cores yet). Pre-fix: top-K all Intel_8086/80286/80386/etc., Hermes refused to identify a fastest CPU. Post-fix: Intel Core i7 in top-K, Hermes correctly identifies Core i7-980X Extreme Edition Gulftown (Intel's 2010 flagship — 32nm, 6 cores, 3.33 GHz, 12 MB L3). Same 2003-corpus AMD-query result preserved (already had Athlon / Athlon_XP_3200+ in top-K via FTS5 BM25 + multi-token TF-IDF cores). 125 tests + 1 skipped, no regressions. --- aborist/qa/query.py | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 19ffdc8..3c7f49f 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -280,12 +280,18 @@ def _docs_with_core_keyword_match( def _body_density_passes( conn, document_root: str, qtokens: set[str], min_mentions: int = 3 ) -> bool: - """Body-token relevance: docs that mention a query token >= N times pass. + """Body-token CO-OCCURRENCE relevance. - Cheap proxy for "this doc is actually about the topic" when title - doesn't help. Counts case-insensitive substring matches in concatenated - chunk content. min_mentions=3 skips docs that mention the term in - passing without being topical. + Pre-2026-04-27 this counted any token's mentions in the body (so + Intel_8086, with 30 "Intel" mentions, would pass for a query of + {intel, fastest, cpu} despite never mentioning "fastest"). The + revised gate requires at least HALF the query tokens to each appear + at least once, AND the total mention count to clear `min_mentions`. + + For a 1-token query (e.g. {anarchism}) this simplifies to "the body + must mention the token at least min_mentions times" — same as before. + For multi-token queries, both signals must hold: breadth (most + tokens present) and depth (enough total mentions). """ if not qtokens: return False @@ -296,12 +302,11 @@ def _body_density_passes( if not rows: return False body = " ".join(unpack_chunk(r["content"]) or "" for r in rows).lower() - total_mentions = 0 - for t in qtokens: - total_mentions += body.count(t.lower()) - if total_mentions >= min_mentions: - return True - return False + counts = {t: body.count(t.lower()) for t in qtokens} + distinct_present = sum(1 for n in counts.values() if n > 0) + total_mentions = sum(counts.values()) + breadth_threshold = max(1, (len(qtokens) + 1) // 2) + return distinct_present >= breadth_threshold and total_mentions >= min_mentions def _search_corpus( @@ -357,12 +362,23 @@ def _search_corpus( ) root_to_shard[h.document_root] = str(p.resolve()) # Parallel title search using synonym-expanded tokens. + # The score starts deliberately low so this signal can't drown + # FTS5 BM25 + body relevance. _rerank_by_title later adds + # `overlap*10` to every hit (FTS5 and title-search alike) that + # has title-token overlap, so a doc whose title genuinely IS + # the topic ends up rewarded twice (once here, once in rerank). + # Single-token title matches against generic terms ("intel", + # "cpu") used to score 60+ standalone, dominating top-K with + # legacy 80486-era articles for queries like "fastest intel + # CPU?". Now that contribution is the same scale as FTS5 body + # BM25, which lets Pentium_4 (high body relevance, no title + # overlap) win on its actual topical fit. for r in _search_titles(conn, list(accept_tokens), over_fetch): title_lower = (r["title"] or "").lower().replace("_", " ") overlap = sum(1 for t in accept_tokens if t in title_lower) if overlap == 0: continue - title_score = 50.0 + overlap * 10.0 + title_score = overlap * 10.0 raw.append( ( title_score,