From 1983b7928c53a0ff25ca7255610e3eb6d5d47ca6 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Mon, 27 Apr 2026 19:36:30 -0400 Subject: [PATCH] retrieval: body-coverage sqrt rerank to counter BM25 short-doc bias After the title-search baseline drop, FTS5 BM25 still surfaced short stub articles ahead of long topical ones for queries with generic tokens. Intel_4040 (1974 4-bit microcontroller, 1-paragraph stub mentioning "intel"/"fastest"/"cpu" once each) outscored Intel_Core_2 (real 2010-era CPU article, hundreds of mentions of every query token) because BM25 normalizes by document length and short docs that hit every term get an outsized boost. Add `_rerank_by_body_coverage` between the relevance filter and the title-overlap rerank. For each surviving candidate (typically 24-32 post-dedup), fetch the full body and sum sqrt(count(t)) over query tokens, then add `coverage * 0.6` to the score. Sqrt scaling is intentional: linear sums let enumerative list pages (e.g. List_of_Intel_microprocessors with 1000+ "intel" mentions) runaway-dominate, while a hard cap saturates and erases the signal between short-stub and long-topical articles. Sqrt grows fast enough to differentiate (5 mentions -> 2.2; 200 mentions -> 14.1) but sublinearly enough to keep enumerations from overwhelming the merge. Verified on the 2010-11 enwiki corpus (no distill cores yet): before: Intel_4040 #1 (43.5), Intel_Core_i7 #6 (39.0) after: Intel_Core_2 #1 (51.9), Intel_Atom #2 (50.5), Intel_4040 #3 (49.6) Body fetch cost is bounded by `over_fetch` (32 candidates per shard, 24-50 unique post-dedup). Each fetch concatenates already-decompressed chunks via `_load_doc_text` which we use elsewhere in the query path, so this adds at most one extra DB round-trip per surviving hit. 125 tests still passing. --- aborist/qa/query.py | 55 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 3c7f49f..7291c2b 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -447,16 +447,69 @@ def _rerank( core_match_roots: set[str] | None = None, body_density_check: callable | None = None, ) -> list[_Hit]: - """Filter out off-topic, then boost title-token matches by overlap count.""" + """Filter off-topic, then layer in body-coverage and title-overlap boosts. + + Order matters: filter first (drops noise), body-coverage rerank next + (counters BM25's short-doc bias by rewarding topical density across + distinct query tokens), title-overlap rerank last (breaks ties when + a doc IS the named topic). + """ hits = _filter_by_title_relevance( hits, question, core_match_roots=core_match_roots, body_density_check=body_density_check, ) + hits = _rerank_by_body_coverage(hits, question) return _rerank_by_title(hits, question) +def _rerank_by_body_coverage( + hits: list, + question: str, + *, + weight: float = 0.6, +) -> list: + """Boost score by per-token body coverage to counteract BM25's short-doc bias. + + BM25 normalizes by document length, but its `b` parameter under-penalizes + *very* short docs that happen to mention every query token. Result: a + 1-paragraph stub on Intel 4040 (1974 microcontroller) outscored a 30-page + Intel Core i7 article for "what is the fastest intel CPU?". + + Approach: for each hit's full body, sum sqrt(count) per query token. + Sqrt scaling lets long topical articles meaningfully out-score short + tangential ones without runaway domination by enumerative list pages. + + stub article: 5 intel + 1 fastest + 4 cpu -> sqrt(5)+sqrt(1)+sqrt(4) ~ 5.2 + long topical: 200 + 10 + 80 -> sqrt(200)+sqrt(10)+sqrt(80) ~ 26.2 + enumeration: 1000 + 50 + 300 -> sqrt(1000)+sqrt(50)+sqrt(300) ~ 55.8 + + Multiply by `weight` (default 0.6) and add to the existing score. The + differentiation in the example above is decisive but bounded: long topical + articles get +15-16, enumerations get +33, stubs get +3. Combined with + FTS5 base scores in the 40s, the long topical article wins comfortably. + + Cost: one body fetch per surviving candidate (typically 24-32). Bounded. + """ + import math + + qtokens_lower = {t.lower() for t in _title_query_tokens(question)} + if not qtokens_lower: + return hits + for h in hits: + body = _load_doc_text(h.shard_path, h.document_root) + if body is None: + continue + body_lower = body.lower() + coverage = sum( + math.sqrt(body_lower.count(t)) for t in qtokens_lower + ) + h.score += coverage * weight + hits.sort(key=lambda h: -h.score) + return hits + + def _load_doc_text(shard_path: str, document_root: str) -> str | None: """Concatenate all hot chunks of a document. Returns None if cold or missing.""" conn = connect(shard_path)