strict retrieval: AND + title-required filter

The query "what is the fastest AMD CPU?" was returning Pentium_4 as
a top source, and Hermes hallucinated by conflating Intel content
with AMD context. Two compounding causes:

1. FTS5 was ORing all content tokens, so any doc with "fastest" OR
   "CPU" matched. BM25 then ranked Pentium_4 highly because the
   article is long and discusses CPU clock speed in detail.

2. The reranker only boosted titles that matched — it never
   penalized titles that didn't match. Pentium_4 had no penalty
   despite being topically wrong.

Two fixes:

- FTS5 escape now ANDs content tokens by default. Stopword list
  expanded to include question-shaping verbs (tell/show/describe/
  explain/find) so they don't sneak in as required tokens. If AND
  returns zero hits, falls back to OR — pragmatic for fuzzy queries
  the user phrases differently than the article wording. All-stopword
  inputs still produce a non-empty MATCH so FTS5 doesn't error.

- _filter_by_title_relevance hard-drops hits whose title shares no
  token with the query, before assembling context. If filtering
  would leave fewer than 3 hits, falls back to the unfiltered set
  (corpus might genuinely not have a topical match — let the LLM
  say "I don't know" rather than nothing).

Together: AND-mode requires every content token in the body, and
title filter requires the doc to actually be about the topic. Pentium
articles drop out of AMD queries; Tell_(poker) drops out of plot
queries.

57 tests still passing.

aborist/qa/query.py

@@ -92,6 +92,29 @@ def _rerank_by_title(hits: list, question: str, boost: float = 8.0) -> list:
     return hits
 
 
+def _filter_by_title_relevance(hits: list, question: str, *, min_keep: int = 3) -> list:
+    """Drop hits whose title shares no token with the query.
+
+    Title is the strongest topical signal — a Pentium_4 article has no
+    business in the context of an AMD question even if BM25 ranks it high
+    on body-token frequency. If the filter would leave fewer than
+    `min_keep` hits, falls back to the original ranked list (the corpus
+    might just not have a topical match — better to give the LLM
+    something than nothing, then let it say "I don't know").
+    """
+    qtokens = _title_query_tokens(question)
+    if not qtokens:
+        return hits
+    kept = [
+        h for h in hits
+        if h.title
+        and (qtokens & _title_query_tokens(h.title.replace("_", " ")))
+    ]
+    if len(kept) < min_keep:
+        return hits
+    return kept
+
+
 DEFAULT_QUERY_POLICY = {
     "system_prompt": (
         "You are answering a question using ONLY the sources provided below. "
@@ -172,7 +195,8 @@ def _search_corpus(
 
 
 def _rerank(hits: list[_Hit], question: str) -> list[_Hit]:
-    """Public-private rerank: title-boost. Easy to extend with other signals."""
+    """Title-relevance filter, then title-token boost ranking."""
+    hits = _filter_by_title_relevance(hits, question)
     return _rerank_by_title(hits, question)
 
 

aborist/search/fts5.py

@@ -12,29 +12,47 @@ _FTS5_STOPWORDS = frozenset(
     """
     the a an is are was were be been being
     of to in on at for with by from as about into through during
-    and or but not no nor so yet
+    and or but not no nor so yet too very also just
     what who where when why how which this that these those such
     i you he she it we they me him her us them
     do does did have has had can could should would will may might
+    tell show describe explain summarize say give list find make
+    please
     """.split()
 )
 
 
-def _escape_fts5(query: str) -> str:
-    """Build an FTS5 MATCH expression from a free-text query.
-
-    Tokenizes by alpha runs (drops punctuation entirely, including the
-    `?` that breaks quoted phrases), drops common stopwords, and ORs the
-    remaining tokens. OR-semantics is intentional for RAG: the LLM sees
-    BM25-ranked top-K, so partial matches are useful evidence rather than
-    noise. Stopwords are dropped because they'd otherwise dominate scoring.
-    """
+def _query_tokens(query: str) -> list[str]:
     raw = _FTS5_TOKEN_RE.findall(query)
-    tokens = [t for t in raw if t.lower() not in _FTS5_STOPWORDS and len(t) > 1]
+    return [t for t in raw if t.lower() not in _FTS5_STOPWORDS and len(t) > 1]
+
+
+def _quote(t: str) -> str:
+    return '"' + t.replace('"', '""') + '"'
+
+
+def _escape_fts5(query: str, *, mode: str = "and") -> str:
+    """Build a MATCH expression from a free-text query.
+
+    Tokenizes by alpha runs (so `?` and other punctuation can't break the
+    quoted phrase), drops question stopwords ('what', 'tell', etc.), then:
+
+    - mode='and' (default): every content token must appear in the doc.
+      Strict relevance — keeps unrelated docs out of the context window.
+    - mode='or':  any token matches. Used as a fallback when AND returns
+      zero hits.
+
+    All-stopword queries fall back to OR over the raw tokens so they
+    still find something instead of crashing FTS5 with an empty MATCH.
+    """
+    tokens = _query_tokens(query)
     if not tokens:
-        # Fallback: keep the query alive even if it was all stopwords.
+        raw = _FTS5_TOKEN_RE.findall(query)
         tokens = [t for t in raw if len(t) > 1] or ['""']
-    return " OR ".join('"' + t.replace('"', '""') + '"' for t in tokens)
+        sep = " OR "
+    else:
+        sep = " AND " if mode == "and" else " OR "
+    return sep.join(_quote(t) for t in tokens)
 
 
 class FTS5Backend(SearchBackend):
@@ -44,32 +62,38 @@ class FTS5Backend(SearchBackend):
     def search(self, query: str, limit: int = 20) -> list[Hit]:
         if not query.strip():
             return []
-        fts_query = _escape_fts5(query)
-        rows = self.conn.execute(
-            """
-            SELECT
-                f.document_root,
-                f.idx,
-                snippet(chunks_fts, 2, '[', ']', '…', 16) AS snip,
-                bm25(chunks_fts) AS rank,
-                d.document_uri,
-                d.title
-            FROM chunks_fts AS f
-            JOIN documents  AS d ON d.document_root = f.document_root
-            WHERE chunks_fts MATCH ?
-            ORDER BY rank ASC
-            LIMIT ?
-            """,
-            (fts_query, limit),
-        ).fetchall()
-
+        # Try strict AND first; fall back to OR if it returns nothing.
+        for mode in ("and", "or"):
+            fts_query = _escape_fts5(query, mode=mode)
+            try:
+                rows = self.conn.execute(
+                    """
+                    SELECT
+                        f.document_root,
+                        f.idx,
+                        snippet(chunks_fts, 2, '[', ']', '…', 16) AS snip,
+                        bm25(chunks_fts) AS rank,
+                        d.document_uri,
+                        d.title
+                    FROM chunks_fts AS f
+                    JOIN documents  AS d ON d.document_root = f.document_root
+                    WHERE chunks_fts MATCH ?
+                    ORDER BY rank ASC
+                    LIMIT ?
+                    """,
+                    (fts_query, limit),
+                ).fetchall()
+            except Exception:
+                rows = []
+            if rows:
+                break
         return [
             Hit(
                 document_root=r["document_root"],
                 document_uri=r["document_uri"],
                 chunk_idx=r["idx"],
                 snippet=r["snip"] or "",
-                # bm25 returns negative numbers (lower = better); flip sign for sanity.
+                # bm25 returns negative numbers (lower = better); flip sign.
                 score=-float(r["rank"]) if r["rank"] is not None else 0.0,
                 audit_mode=self.audit_mode,
                 title=r["title"],