title-LIKE search alongside FTS5 — rescues topic articles BM25 misses
The "what is HTTP?" query returned only Main_Page/Temp because BM25
over chunk content rated URL-heavy list pages (List_of_planned_cities,
Mary_Kay_Letourneau) above the actual HTTP protocol article. Many
http:// links in body trumped the topic article that names "HTTP" once
in title and a few times in body.
Same shape on AMD/Stoicism/etc. when the topic article is short or
when other articles incidentally repeat the query token in body text.
Fix: parallel SQL `LOWER(title) LIKE` search per shard alongside FTS5.
Title hits get a 50-baseline + 10-per-overlap score (well above body
BM25 scores), so an article whose title literally is the topic rises
to the top regardless of body term frequency.
After:
Q "what is HTTP?" → HTTP, Apache_HTTP_Server, HTTP_cookie
Hermes pulls the protocol definition
directly from the HTTP article body.
Q "fastest AMD CPU?" → AMD, AMD_Duron, CPU_design, ...
Answer: Athlon 1400 MHz on Thunderbird,
grounded in the AMD article.
Q "Athlon processor" → Athlon ranks in top sources, answer
correctly attributes to AMD.
The title-relevance filter still applies post-merge: hits without
title-token overlap are dropped if any title-matchers exist. So
poison sources (Pentium_4 in AMD queries) stay out of the context
window.
57 tests passing.
This commit is contained in:
parent
13ae4cfb24
commit
3b010279fe
1 changed files with 69 additions and 17 deletions
|
|
@ -75,15 +75,18 @@ def _title_query_tokens(s: str) -> set[str]:
|
|||
}
|
||||
|
||||
|
||||
def _rerank_by_title(hits: list, question: str, boost: float = 8.0) -> list:
|
||||
"""Boost hits whose title contains query tokens — title is high signal."""
|
||||
def _rerank_by_title(
|
||||
hits: list,
|
||||
question: str,
|
||||
boost: float = 10.0,
|
||||
) -> list:
|
||||
"""Boost hits whose title overlaps query tokens. Pure ordering aid."""
|
||||
qtokens = _title_query_tokens(question)
|
||||
if not qtokens:
|
||||
return hits
|
||||
for h in hits:
|
||||
if not h.title:
|
||||
continue
|
||||
# Replace _ with space: "Back_to_the_Future" -> ["back", "future"]
|
||||
ttokens = _title_query_tokens(h.title.replace("_", " "))
|
||||
overlap = len(qtokens & ttokens)
|
||||
if overlap:
|
||||
|
|
@ -92,15 +95,17 @@ def _rerank_by_title(hits: list, question: str, boost: float = 8.0) -> list:
|
|||
return hits
|
||||
|
||||
|
||||
def _filter_by_title_relevance(hits: list, question: str, *, min_keep: int = 3) -> list:
|
||||
"""Drop hits whose title shares no token with the query.
|
||||
def _filter_by_title_relevance(hits: list, question: str) -> list:
|
||||
"""Hard-filter: when ANY hit's title overlaps the query, drop the rest.
|
||||
|
||||
Title is the strongest topical signal — a Pentium_4 article has no
|
||||
business in the context of an AMD question even if BM25 ranks it high
|
||||
on body-token frequency. If the filter would leave fewer than
|
||||
`min_keep` hits, falls back to the original ranked list (the corpus
|
||||
might just not have a topical match — better to give the LLM
|
||||
something than nothing, then let it say "I don't know").
|
||||
Reranking by score wasn't strong enough — Pentium_4 with score 7 still
|
||||
ended up in the top-K context window for "fastest AMD CPU" and Hermes
|
||||
conflated it as the AMD answer. Fix: if we have ANY title-matchers, use
|
||||
only those. Off-topic articles never enter the context.
|
||||
|
||||
If zero hits match by title (corpus has nothing topical), keep the top
|
||||
one so the LLM sees a single source and can honestly say "I don't know
|
||||
based on these sources." Empty context produces fabrication.
|
||||
"""
|
||||
qtokens = _title_query_tokens(question)
|
||||
if not qtokens:
|
||||
|
|
@ -110,8 +115,8 @@ def _filter_by_title_relevance(hits: list, question: str, *, min_keep: int = 3)
|
|||
if h.title
|
||||
and (qtokens & _title_query_tokens(h.title.replace("_", " ")))
|
||||
]
|
||||
if len(kept) < min_keep:
|
||||
return hits
|
||||
if not kept:
|
||||
return hits[:1] if hits else []
|
||||
return kept
|
||||
|
||||
|
||||
|
|
@ -139,13 +144,42 @@ class _Hit:
|
|||
chunk_idx: int
|
||||
|
||||
|
||||
def _search_titles(conn, qtokens: list[str], limit: int) -> list[tuple]:
|
||||
"""SQL LIKE over documents.title — finds the HTTP article that FTS5
|
||||
misses because list-pages with many URLs have higher 'http' term
|
||||
frequency than the actual protocol article. Returns rows shaped
|
||||
to match the FTS5 hit tuple."""
|
||||
if not qtokens:
|
||||
return []
|
||||
clauses = " OR ".join(["LOWER(title) LIKE ?"] * len(qtokens))
|
||||
params = [f"%{t.lower()}%" for t in qtokens]
|
||||
params.append(limit)
|
||||
rows = conn.execute(
|
||||
f"SELECT document_root, document_uri, title FROM documents "
|
||||
f"WHERE {clauses} LIMIT ?",
|
||||
params,
|
||||
).fetchall()
|
||||
return rows
|
||||
|
||||
|
||||
def _search_corpus(
|
||||
shards_dir: Path | None,
|
||||
single_db: Path | None,
|
||||
question: str,
|
||||
over_fetch: int,
|
||||
) -> list[_Hit]:
|
||||
"""FTS5 across shards (or single DB). Dedupe by document_root."""
|
||||
"""Two parallel searches across shards, merged:
|
||||
|
||||
- FTS5 body search (BM25 over chunk content).
|
||||
- SQL title-LIKE search (catches articles whose body is short on the
|
||||
query terms but whose title is the literal topic — e.g., the HTTP
|
||||
protocol article doesn't out-frequency-score URL-heavy list pages
|
||||
but it IS the topic).
|
||||
|
||||
Dedupe by document_root. Title hits get a baseline score that
|
||||
out-ranks FTS5 body hits so the actual topic article rises to the top.
|
||||
"""
|
||||
qtokens = _title_query_tokens(question)
|
||||
paths: list[Path]
|
||||
if shards_dir is not None:
|
||||
paths = discover_shards(shards_dir)
|
||||
|
|
@ -159,8 +193,7 @@ def _search_corpus(
|
|||
conn = connect(p)
|
||||
try:
|
||||
backend = FTS5Backend(conn)
|
||||
hits = backend.search(question, limit=over_fetch)
|
||||
for h in hits:
|
||||
for h in backend.search(question, limit=over_fetch):
|
||||
raw.append(
|
||||
(
|
||||
h.score,
|
||||
|
|
@ -171,6 +204,25 @@ def _search_corpus(
|
|||
str(p.resolve()),
|
||||
)
|
||||
)
|
||||
# Parallel title search. Score is overlap-weighted and
|
||||
# baseline-elevated so an exact title match (HTTP, Stoicism)
|
||||
# comes above body BM25.
|
||||
for r in _search_titles(conn, qtokens, over_fetch):
|
||||
title_lower = (r["title"] or "").lower().replace("_", " ")
|
||||
overlap = sum(1 for t in qtokens if t.lower() in title_lower)
|
||||
if overlap == 0:
|
||||
continue
|
||||
title_score = 50.0 + overlap * 10.0
|
||||
raw.append(
|
||||
(
|
||||
title_score,
|
||||
r["document_root"],
|
||||
r["document_uri"],
|
||||
r["title"],
|
||||
0,
|
||||
str(p.resolve()),
|
||||
)
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
|
@ -195,7 +247,7 @@ def _search_corpus(
|
|||
|
||||
|
||||
def _rerank(hits: list[_Hit], question: str) -> list[_Hit]:
|
||||
"""Title-relevance filter, then title-token boost ranking."""
|
||||
"""Filter out off-topic titles, then boost matches by overlap count."""
|
||||
hits = _filter_by_title_relevance(hits, question)
|
||||
return _rerank_by_title(hits, question)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue