title-LIKE search alongside FTS5 — rescues topic articles BM25 misses

The "what is HTTP?" query returned only Main_Page/Temp because BM25
over chunk content rated URL-heavy list pages (List_of_planned_cities,
Mary_Kay_Letourneau) above the actual HTTP protocol article. Many
http:// links in body trumped the topic article that names "HTTP" once
in title and a few times in body.

Same shape on AMD/Stoicism/etc. when the topic article is short or
when other articles incidentally repeat the query token in body text.

Fix: parallel SQL `LOWER(title) LIKE` search per shard alongside FTS5.
Title hits get a 50-baseline + 10-per-overlap score (well above body
BM25 scores), so an article whose title literally is the topic rises
to the top regardless of body term frequency.

After:
  Q "what is HTTP?"           → HTTP, Apache_HTTP_Server, HTTP_cookie
                                Hermes pulls the protocol definition
                                directly from the HTTP article body.

  Q "fastest AMD CPU?"        → AMD, AMD_Duron, CPU_design, ...
                                Answer: Athlon 1400 MHz on Thunderbird,
                                grounded in the AMD article.

  Q "Athlon processor"        → Athlon ranks in top sources, answer
                                correctly attributes to AMD.

The title-relevance filter still applies post-merge: hits without
title-token overlap are dropped if any title-matchers exist. So
poison sources (Pentium_4 in AMD queries) stay out of the context
window.

57 tests passing.
This commit is contained in:
russell@unturf.com 2026-04-27 12:30:00 -04:00
parent 13ae4cfb24
commit 3b010279fe
No known key found for this signature in database

View file

@ -75,15 +75,18 @@ def _title_query_tokens(s: str) -> set[str]:
}
def _rerank_by_title(hits: list, question: str, boost: float = 8.0) -> list:
"""Boost hits whose title contains query tokens — title is high signal."""
def _rerank_by_title(
hits: list,
question: str,
boost: float = 10.0,
) -> list:
"""Boost hits whose title overlaps query tokens. Pure ordering aid."""
qtokens = _title_query_tokens(question)
if not qtokens:
return hits
for h in hits:
if not h.title:
continue
# Replace _ with space: "Back_to_the_Future" -> ["back", "future"]
ttokens = _title_query_tokens(h.title.replace("_", " "))
overlap = len(qtokens & ttokens)
if overlap:
@ -92,15 +95,17 @@ def _rerank_by_title(hits: list, question: str, boost: float = 8.0) -> list:
return hits
def _filter_by_title_relevance(hits: list, question: str, *, min_keep: int = 3) -> list:
"""Drop hits whose title shares no token with the query.
def _filter_by_title_relevance(hits: list, question: str) -> list:
"""Hard-filter: when ANY hit's title overlaps the query, drop the rest.
Title is the strongest topical signal — a Pentium_4 article has no
business in the context of an AMD question even if BM25 ranks it high
on body-token frequency. If the filter would leave fewer than
`min_keep` hits, falls back to the original ranked list (the corpus
might just not have a topical match — better to give the LLM
something than nothing, then let it say "I don't know").
Reranking by score wasn't strong enough — Pentium_4 with score 7 still
ended up in the top-K context window for "fastest AMD CPU" and Hermes
conflated it as the AMD answer. Fix: if we have ANY title-matchers, use
only those. Off-topic articles never enter the context.
If zero hits match by title (corpus has nothing topical), keep the top
one so the LLM sees a single source and can honestly say "I don't know
based on these sources." Empty context produces fabrication.
"""
qtokens = _title_query_tokens(question)
if not qtokens:
@ -110,8 +115,8 @@ def _filter_by_title_relevance(hits: list, question: str, *, min_keep: int = 3)
if h.title
and (qtokens & _title_query_tokens(h.title.replace("_", " ")))
]
if len(kept) < min_keep:
return hits
if not kept:
return hits[:1] if hits else []
return kept
@ -139,13 +144,42 @@ class _Hit:
chunk_idx: int
def _search_titles(conn, qtokens: list[str], limit: int) -> list[tuple]:
"""SQL LIKE over documents.title — finds the HTTP article that FTS5
misses because list-pages with many URLs have higher 'http' term
frequency than the actual protocol article. Returns rows shaped
to match the FTS5 hit tuple."""
if not qtokens:
return []
clauses = " OR ".join(["LOWER(title) LIKE ?"] * len(qtokens))
params = [f"%{t.lower()}%" for t in qtokens]
params.append(limit)
rows = conn.execute(
f"SELECT document_root, document_uri, title FROM documents "
f"WHERE {clauses} LIMIT ?",
params,
).fetchall()
return rows
def _search_corpus(
shards_dir: Path | None,
single_db: Path | None,
question: str,
over_fetch: int,
) -> list[_Hit]:
"""FTS5 across shards (or single DB). Dedupe by document_root."""
"""Two parallel searches across shards, merged:
- FTS5 body search (BM25 over chunk content).
- SQL title-LIKE search (catches articles whose body is short on the
query terms but whose title is the literal topic — e.g., the HTTP
protocol article doesn't out-frequency-score URL-heavy list pages
but it IS the topic).
Dedupe by document_root. Title hits get a baseline score that
out-ranks FTS5 body hits so the actual topic article rises to the top.
"""
qtokens = _title_query_tokens(question)
paths: list[Path]
if shards_dir is not None:
paths = discover_shards(shards_dir)
@ -159,8 +193,7 @@ def _search_corpus(
conn = connect(p)
try:
backend = FTS5Backend(conn)
hits = backend.search(question, limit=over_fetch)
for h in hits:
for h in backend.search(question, limit=over_fetch):
raw.append(
(
h.score,
@ -171,6 +204,25 @@ def _search_corpus(
str(p.resolve()),
)
)
# Parallel title search. Score is overlap-weighted and
# baseline-elevated so an exact title match (HTTP, Stoicism)
# comes above body BM25.
for r in _search_titles(conn, qtokens, over_fetch):
title_lower = (r["title"] or "").lower().replace("_", " ")
overlap = sum(1 for t in qtokens if t.lower() in title_lower)
if overlap == 0:
continue
title_score = 50.0 + overlap * 10.0
raw.append(
(
title_score,
r["document_root"],
r["document_uri"],
r["title"],
0,
str(p.resolve()),
)
)
finally:
conn.close()
@ -195,7 +247,7 @@ def _search_corpus(
def _rerank(hits: list[_Hit], question: str) -> list[_Hit]:
"""Title-relevance filter, then title-token boost ranking."""
"""Filter out off-topic titles, then boost matches by overlap count."""
hits = _filter_by_title_relevance(hits, question)
return _rerank_by_title(hits, question)