From 3b010279fec5081d28958f09559b79da4f8239df Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Mon, 27 Apr 2026 12:30:00 -0400 Subject: [PATCH] =?UTF-8?q?title-LIKE=20search=20alongside=20FTS5=20?= =?UTF-8?q?=E2=80=94=20rescues=20topic=20articles=20BM25=20misses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "what is HTTP?" query returned only Main_Page/Temp because BM25 over chunk content rated URL-heavy list pages (List_of_planned_cities, Mary_Kay_Letourneau) above the actual HTTP protocol article. Many http:// links in body trumped the topic article that names "HTTP" once in title and a few times in body. Same shape on AMD/Stoicism/etc. when the topic article is short or when other articles incidentally repeat the query token in body text. Fix: parallel SQL `LOWER(title) LIKE` search per shard alongside FTS5. Title hits get a 50-baseline + 10-per-overlap score (well above body BM25 scores), so an article whose title literally is the topic rises to the top regardless of body term frequency. After: Q "what is HTTP?" → HTTP, Apache_HTTP_Server, HTTP_cookie Hermes pulls the protocol definition directly from the HTTP article body. Q "fastest AMD CPU?" → AMD, AMD_Duron, CPU_design, ... Answer: Athlon 1400 MHz on Thunderbird, grounded in the AMD article. Q "Athlon processor" → Athlon ranks in top sources, answer correctly attributes to AMD. The title-relevance filter still applies post-merge: hits without title-token overlap are dropped if any title-matchers exist. So poison sources (Pentium_4 in AMD queries) stay out of the context window. 57 tests passing. --- aborist/qa/query.py | 86 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 69 insertions(+), 17 deletions(-) diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 31d4436..82d609b 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -75,15 +75,18 @@ def _title_query_tokens(s: str) -> set[str]: } -def _rerank_by_title(hits: list, question: str, boost: float = 8.0) -> list: - """Boost hits whose title contains query tokens — title is high signal.""" +def _rerank_by_title( + hits: list, + question: str, + boost: float = 10.0, +) -> list: + """Boost hits whose title overlaps query tokens. Pure ordering aid.""" qtokens = _title_query_tokens(question) if not qtokens: return hits for h in hits: if not h.title: continue - # Replace _ with space: "Back_to_the_Future" -> ["back", "future"] ttokens = _title_query_tokens(h.title.replace("_", " ")) overlap = len(qtokens & ttokens) if overlap: @@ -92,15 +95,17 @@ def _rerank_by_title(hits: list, question: str, boost: float = 8.0) -> list: return hits -def _filter_by_title_relevance(hits: list, question: str, *, min_keep: int = 3) -> list: - """Drop hits whose title shares no token with the query. +def _filter_by_title_relevance(hits: list, question: str) -> list: + """Hard-filter: when ANY hit's title overlaps the query, drop the rest. - Title is the strongest topical signal — a Pentium_4 article has no - business in the context of an AMD question even if BM25 ranks it high - on body-token frequency. If the filter would leave fewer than - `min_keep` hits, falls back to the original ranked list (the corpus - might just not have a topical match — better to give the LLM - something than nothing, then let it say "I don't know"). + Reranking by score wasn't strong enough — Pentium_4 with score 7 still + ended up in the top-K context window for "fastest AMD CPU" and Hermes + conflated it as the AMD answer. Fix: if we have ANY title-matchers, use + only those. Off-topic articles never enter the context. + + If zero hits match by title (corpus has nothing topical), keep the top + one so the LLM sees a single source and can honestly say "I don't know + based on these sources." Empty context produces fabrication. """ qtokens = _title_query_tokens(question) if not qtokens: @@ -110,8 +115,8 @@ def _filter_by_title_relevance(hits: list, question: str, *, min_keep: int = 3) if h.title and (qtokens & _title_query_tokens(h.title.replace("_", " "))) ] - if len(kept) < min_keep: - return hits + if not kept: + return hits[:1] if hits else [] return kept @@ -139,13 +144,42 @@ class _Hit: chunk_idx: int +def _search_titles(conn, qtokens: list[str], limit: int) -> list[tuple]: + """SQL LIKE over documents.title — finds the HTTP article that FTS5 + misses because list-pages with many URLs have higher 'http' term + frequency than the actual protocol article. Returns rows shaped + to match the FTS5 hit tuple.""" + if not qtokens: + return [] + clauses = " OR ".join(["LOWER(title) LIKE ?"] * len(qtokens)) + params = [f"%{t.lower()}%" for t in qtokens] + params.append(limit) + rows = conn.execute( + f"SELECT document_root, document_uri, title FROM documents " + f"WHERE {clauses} LIMIT ?", + params, + ).fetchall() + return rows + + def _search_corpus( shards_dir: Path | None, single_db: Path | None, question: str, over_fetch: int, ) -> list[_Hit]: - """FTS5 across shards (or single DB). Dedupe by document_root.""" + """Two parallel searches across shards, merged: + + - FTS5 body search (BM25 over chunk content). + - SQL title-LIKE search (catches articles whose body is short on the + query terms but whose title is the literal topic — e.g., the HTTP + protocol article doesn't out-frequency-score URL-heavy list pages + but it IS the topic). + + Dedupe by document_root. Title hits get a baseline score that + out-ranks FTS5 body hits so the actual topic article rises to the top. + """ + qtokens = _title_query_tokens(question) paths: list[Path] if shards_dir is not None: paths = discover_shards(shards_dir) @@ -159,8 +193,7 @@ def _search_corpus( conn = connect(p) try: backend = FTS5Backend(conn) - hits = backend.search(question, limit=over_fetch) - for h in hits: + for h in backend.search(question, limit=over_fetch): raw.append( ( h.score, @@ -171,6 +204,25 @@ def _search_corpus( str(p.resolve()), ) ) + # Parallel title search. Score is overlap-weighted and + # baseline-elevated so an exact title match (HTTP, Stoicism) + # comes above body BM25. + for r in _search_titles(conn, qtokens, over_fetch): + title_lower = (r["title"] or "").lower().replace("_", " ") + overlap = sum(1 for t in qtokens if t.lower() in title_lower) + if overlap == 0: + continue + title_score = 50.0 + overlap * 10.0 + raw.append( + ( + title_score, + r["document_root"], + r["document_uri"], + r["title"], + 0, + str(p.resolve()), + ) + ) finally: conn.close() @@ -195,7 +247,7 @@ def _search_corpus( def _rerank(hits: list[_Hit], question: str) -> list[_Hit]: - """Title-relevance filter, then title-token boost ranking.""" + """Filter out off-topic titles, then boost matches by overlap count.""" hits = _filter_by_title_relevance(hits, question) return _rerank_by_title(hits, question)