From 13ae4cfb24766053d7d69d96e9353255d1df6dbf Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Mon, 27 Apr 2026 12:21:35 -0400 Subject: [PATCH] strict retrieval: AND + title-required filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The query "what is the fastest AMD CPU?" was returning Pentium_4 as a top source, and Hermes hallucinated by conflating Intel content with AMD context. Two compounding causes: 1. FTS5 was ORing all content tokens, so any doc with "fastest" OR "CPU" matched. BM25 then ranked Pentium_4 highly because the article is long and discusses CPU clock speed in detail. 2. The reranker only boosted titles that matched — it never penalized titles that didn't match. Pentium_4 had no penalty despite being topically wrong. Two fixes: - FTS5 escape now ANDs content tokens by default. Stopword list expanded to include question-shaping verbs (tell/show/describe/ explain/find) so they don't sneak in as required tokens. If AND returns zero hits, falls back to OR — pragmatic for fuzzy queries the user phrases differently than the article wording. All-stopword inputs still produce a non-empty MATCH so FTS5 doesn't error. - _filter_by_title_relevance hard-drops hits whose title shares no token with the query, before assembling context. If filtering would leave fewer than 3 hits, falls back to the unfiltered set (corpus might genuinely not have a topical match — let the LLM say "I don't know" rather than nothing). Together: AND-mode requires every content token in the body, and title filter requires the doc to actually be about the topic. Pentium articles drop out of AMD queries; Tell_(poker) drops out of plot queries. 57 tests still passing. --- aborist/qa/query.py | 26 +++++++++++- aborist/search/fts5.py | 90 ++++++++++++++++++++++++++---------------- 2 files changed, 82 insertions(+), 34 deletions(-) diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 7e6858f..31d4436 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -92,6 +92,29 @@ def _rerank_by_title(hits: list, question: str, boost: float = 8.0) -> list: return hits +def _filter_by_title_relevance(hits: list, question: str, *, min_keep: int = 3) -> list: + """Drop hits whose title shares no token with the query. + + Title is the strongest topical signal — a Pentium_4 article has no + business in the context of an AMD question even if BM25 ranks it high + on body-token frequency. If the filter would leave fewer than + `min_keep` hits, falls back to the original ranked list (the corpus + might just not have a topical match — better to give the LLM + something than nothing, then let it say "I don't know"). + """ + qtokens = _title_query_tokens(question) + if not qtokens: + return hits + kept = [ + h for h in hits + if h.title + and (qtokens & _title_query_tokens(h.title.replace("_", " "))) + ] + if len(kept) < min_keep: + return hits + return kept + + DEFAULT_QUERY_POLICY = { "system_prompt": ( "You are answering a question using ONLY the sources provided below. " @@ -172,7 +195,8 @@ def _search_corpus( def _rerank(hits: list[_Hit], question: str) -> list[_Hit]: - """Public-private rerank: title-boost. Easy to extend with other signals.""" + """Title-relevance filter, then title-token boost ranking.""" + hits = _filter_by_title_relevance(hits, question) return _rerank_by_title(hits, question) diff --git a/aborist/search/fts5.py b/aborist/search/fts5.py index 6abbac6..8a75f8e 100644 --- a/aborist/search/fts5.py +++ b/aborist/search/fts5.py @@ -12,29 +12,47 @@ _FTS5_STOPWORDS = frozenset( """ the a an is are was were be been being of to in on at for with by from as about into through during - and or but not no nor so yet + and or but not no nor so yet too very also just what who where when why how which this that these those such i you he she it we they me him her us them do does did have has had can could should would will may might + tell show describe explain summarize say give list find make + please """.split() ) -def _escape_fts5(query: str) -> str: - """Build an FTS5 MATCH expression from a free-text query. - - Tokenizes by alpha runs (drops punctuation entirely, including the - `?` that breaks quoted phrases), drops common stopwords, and ORs the - remaining tokens. OR-semantics is intentional for RAG: the LLM sees - BM25-ranked top-K, so partial matches are useful evidence rather than - noise. Stopwords are dropped because they'd otherwise dominate scoring. - """ +def _query_tokens(query: str) -> list[str]: raw = _FTS5_TOKEN_RE.findall(query) - tokens = [t for t in raw if t.lower() not in _FTS5_STOPWORDS and len(t) > 1] + return [t for t in raw if t.lower() not in _FTS5_STOPWORDS and len(t) > 1] + + +def _quote(t: str) -> str: + return '"' + t.replace('"', '""') + '"' + + +def _escape_fts5(query: str, *, mode: str = "and") -> str: + """Build a MATCH expression from a free-text query. + + Tokenizes by alpha runs (so `?` and other punctuation can't break the + quoted phrase), drops question stopwords ('what', 'tell', etc.), then: + + - mode='and' (default): every content token must appear in the doc. + Strict relevance — keeps unrelated docs out of the context window. + - mode='or': any token matches. Used as a fallback when AND returns + zero hits. + + All-stopword queries fall back to OR over the raw tokens so they + still find something instead of crashing FTS5 with an empty MATCH. + """ + tokens = _query_tokens(query) if not tokens: - # Fallback: keep the query alive even if it was all stopwords. + raw = _FTS5_TOKEN_RE.findall(query) tokens = [t for t in raw if len(t) > 1] or ['""'] - return " OR ".join('"' + t.replace('"', '""') + '"' for t in tokens) + sep = " OR " + else: + sep = " AND " if mode == "and" else " OR " + return sep.join(_quote(t) for t in tokens) class FTS5Backend(SearchBackend): @@ -44,32 +62,38 @@ class FTS5Backend(SearchBackend): def search(self, query: str, limit: int = 20) -> list[Hit]: if not query.strip(): return [] - fts_query = _escape_fts5(query) - rows = self.conn.execute( - """ - SELECT - f.document_root, - f.idx, - snippet(chunks_fts, 2, '[', ']', '…', 16) AS snip, - bm25(chunks_fts) AS rank, - d.document_uri, - d.title - FROM chunks_fts AS f - JOIN documents AS d ON d.document_root = f.document_root - WHERE chunks_fts MATCH ? - ORDER BY rank ASC - LIMIT ? - """, - (fts_query, limit), - ).fetchall() - + # Try strict AND first; fall back to OR if it returns nothing. + for mode in ("and", "or"): + fts_query = _escape_fts5(query, mode=mode) + try: + rows = self.conn.execute( + """ + SELECT + f.document_root, + f.idx, + snippet(chunks_fts, 2, '[', ']', '…', 16) AS snip, + bm25(chunks_fts) AS rank, + d.document_uri, + d.title + FROM chunks_fts AS f + JOIN documents AS d ON d.document_root = f.document_root + WHERE chunks_fts MATCH ? + ORDER BY rank ASC + LIMIT ? + """, + (fts_query, limit), + ).fetchall() + except Exception: + rows = [] + if rows: + break return [ Hit( document_root=r["document_root"], document_uri=r["document_uri"], chunk_idx=r["idx"], snippet=r["snip"] or "", - # bm25 returns negative numbers (lower = better); flip sign for sanity. + # bm25 returns negative numbers (lower = better); flip sign. score=-float(r["rank"]) if r["rank"] is not None else 0.0, audit_mode=self.audit_mode, title=r["title"],