diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 3c7f49f..7291c2b 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -447,16 +447,69 @@ def _rerank( core_match_roots: set[str] | None = None, body_density_check: callable | None = None, ) -> list[_Hit]: - """Filter out off-topic, then boost title-token matches by overlap count.""" + """Filter off-topic, then layer in body-coverage and title-overlap boosts. + + Order matters: filter first (drops noise), body-coverage rerank next + (counters BM25's short-doc bias by rewarding topical density across + distinct query tokens), title-overlap rerank last (breaks ties when + a doc IS the named topic). + """ hits = _filter_by_title_relevance( hits, question, core_match_roots=core_match_roots, body_density_check=body_density_check, ) + hits = _rerank_by_body_coverage(hits, question) return _rerank_by_title(hits, question) +def _rerank_by_body_coverage( + hits: list, + question: str, + *, + weight: float = 0.6, +) -> list: + """Boost score by per-token body coverage to counteract BM25's short-doc bias. + + BM25 normalizes by document length, but its `b` parameter under-penalizes + *very* short docs that happen to mention every query token. Result: a + 1-paragraph stub on Intel 4040 (1974 microcontroller) outscored a 30-page + Intel Core i7 article for "what is the fastest intel CPU?". + + Approach: for each hit's full body, sum sqrt(count) per query token. + Sqrt scaling lets long topical articles meaningfully out-score short + tangential ones without runaway domination by enumerative list pages. + + stub article: 5 intel + 1 fastest + 4 cpu -> sqrt(5)+sqrt(1)+sqrt(4) ~ 5.2 + long topical: 200 + 10 + 80 -> sqrt(200)+sqrt(10)+sqrt(80) ~ 26.2 + enumeration: 1000 + 50 + 300 -> sqrt(1000)+sqrt(50)+sqrt(300) ~ 55.8 + + Multiply by `weight` (default 0.6) and add to the existing score. The + differentiation in the example above is decisive but bounded: long topical + articles get +15-16, enumerations get +33, stubs get +3. Combined with + FTS5 base scores in the 40s, the long topical article wins comfortably. + + Cost: one body fetch per surviving candidate (typically 24-32). Bounded. + """ + import math + + qtokens_lower = {t.lower() for t in _title_query_tokens(question)} + if not qtokens_lower: + return hits + for h in hits: + body = _load_doc_text(h.shard_path, h.document_root) + if body is None: + continue + body_lower = body.lower() + coverage = sum( + math.sqrt(body_lower.count(t)) for t in qtokens_lower + ) + h.score += coverage * weight + hits.sort(key=lambda h: -h.score) + return hits + + def _load_doc_text(shard_path: str, document_root: str) -> str | None: """Concatenate all hot chunks of a document. Returns None if cold or missing.""" conn = connect(shard_path)