diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 19ffdc8..3c7f49f 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -280,12 +280,18 @@ def _docs_with_core_keyword_match( def _body_density_passes( conn, document_root: str, qtokens: set[str], min_mentions: int = 3 ) -> bool: - """Body-token relevance: docs that mention a query token >= N times pass. + """Body-token CO-OCCURRENCE relevance. - Cheap proxy for "this doc is actually about the topic" when title - doesn't help. Counts case-insensitive substring matches in concatenated - chunk content. min_mentions=3 skips docs that mention the term in - passing without being topical. + Pre-2026-04-27 this counted any token's mentions in the body (so + Intel_8086, with 30 "Intel" mentions, would pass for a query of + {intel, fastest, cpu} despite never mentioning "fastest"). The + revised gate requires at least HALF the query tokens to each appear + at least once, AND the total mention count to clear `min_mentions`. + + For a 1-token query (e.g. {anarchism}) this simplifies to "the body + must mention the token at least min_mentions times" — same as before. + For multi-token queries, both signals must hold: breadth (most + tokens present) and depth (enough total mentions). """ if not qtokens: return False @@ -296,12 +302,11 @@ def _body_density_passes( if not rows: return False body = " ".join(unpack_chunk(r["content"]) or "" for r in rows).lower() - total_mentions = 0 - for t in qtokens: - total_mentions += body.count(t.lower()) - if total_mentions >= min_mentions: - return True - return False + counts = {t: body.count(t.lower()) for t in qtokens} + distinct_present = sum(1 for n in counts.values() if n > 0) + total_mentions = sum(counts.values()) + breadth_threshold = max(1, (len(qtokens) + 1) // 2) + return distinct_present >= breadth_threshold and total_mentions >= min_mentions def _search_corpus( @@ -357,12 +362,23 @@ def _search_corpus( ) root_to_shard[h.document_root] = str(p.resolve()) # Parallel title search using synonym-expanded tokens. + # The score starts deliberately low so this signal can't drown + # FTS5 BM25 + body relevance. _rerank_by_title later adds + # `overlap*10` to every hit (FTS5 and title-search alike) that + # has title-token overlap, so a doc whose title genuinely IS + # the topic ends up rewarded twice (once here, once in rerank). + # Single-token title matches against generic terms ("intel", + # "cpu") used to score 60+ standalone, dominating top-K with + # legacy 80486-era articles for queries like "fastest intel + # CPU?". Now that contribution is the same scale as FTS5 body + # BM25, which lets Pentium_4 (high body relevance, no title + # overlap) win on its actual topical fit. for r in _search_titles(conn, list(accept_tokens), over_fetch): title_lower = (r["title"] or "").lower().replace("_", " ") overlap = sum(1 for t in accept_tokens if t in title_lower) if overlap == 0: continue - title_score = 50.0 + overlap * 10.0 + title_score = overlap * 10.0 raw.append( ( title_score,