From 768a15d5caa8790ff93c42210097e3183ec4969e Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Mon, 1 Jun 2026 13:11:35 -0400 Subject: [PATCH] providence_query: parallel body+title FTS5 retrieval (fixes UNGROUNDED noise) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fox flagged 2026-06-01: `make query Q="when did sesame street first air on television?" LLM=qwen` returned UNGROUNDED with citations to "AC Graph Coloring" and "AC Sudoku Puzzles" textbook chunks. None of the retrieved docs mentioned Sesame Street. Root cause: providence_query was running body-only FTS5 retrieval. Body progressive-AND on a multi-token natural-language query collapses to whichever-token-survives (here "first" or "air"), which matches the textbook crawler shard (`crawl_appliedcombinatorics_org.db`) heavily. The article "Sesame Street" never enters the candidate set. Legacy query() avoids this by running a TITLE FTS5 route in parallel with body — title BM25 over the documents_fts shadow index ranks "Sesame Street" article at rank 1 (bm25 -19) for this query. Fix has two parts: 1) SqliteShardCorpus.fts_body: re-flip FTS5Backend's sign-flipped score back to raw bm25 (more-negative = better). Without the reflip, body returned +0.137 (high = good) and title returned -19 (low = good); the merge couldn't apply a uniform min-bm25 rule. Cloud SidecarBucketCorpus already emits raw bm25; local now matches that convention. 2) providence_query retrieval phase: fan body + title in parallel via ThreadPoolExecutor (max_workers=2), merge by min(score) per document_root, apply_title_boost on the union, take top_k. Body route catches docs with strong body BM25 but no title overlap; title route catches canonical articles whose name matches the question subject. Either side is sufficient to surface the right doc; both together is what legacy does. Measured (live `make query LLM=qwen` on sesame fixture): before: UNGROUNDED, citations to "AC Graph Coloring" / "AC Sudoku Puzzles", primary picked wrong after: STRICT, "Sesame Street" article at primary, Qwen answers "Sesame Street premiered on November 10, 1969" with two STRICT-verified citations to the Sesame Street article and History of Sesame Street article. 8.3s wall time. Top-8 retrieved titles now read: Sesame Street · Sesame Street (Japan) · History of Sesame Street · Sesame Street media · 5, Rue Sésame · Sesame Park · 1, rue Sesame · ... Smoke "test" + Anarchism fixture from test_providence_query also pass (4 tests). 22 corpus + providence tests pass. Progress event surface also gains body= and title= counts on search.done so operators can see which route surfaced what. --- arborist/qa/corpus.py | 12 ++++++--- arborist/qa/providence_query.py | 48 ++++++++++++++++++++++++++------- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/arborist/qa/corpus.py b/arborist/qa/corpus.py index e9708d9..a6b8bb6 100644 --- a/arborist/qa/corpus.py +++ b/arborist/qa/corpus.py @@ -321,15 +321,21 @@ class SqliteShardCorpus: return [] from arborist.search.fts5 import FTS5Backend backend = FTS5Backend(self._conn) - # FTS5Backend.search returns objects with document_root, - # document_uri, title, chunk_idx, score, snippet. + # FTS5Backend.search sign-FLIPS bm25 internally (so higher + # numbers = stronger match in its return shape). Title FTS5 + # via fts_title returns RAW bm25 (more-negative = better). To + # let providence_query merge body+title hits by a single + # min-bm25 rule, re-flip FTS5Backend's positive score back + # to negative here. Net: SqliteShardCorpus.fts_body and + # .fts_title both emit raw bm25 (lower = better), same + # convention as the cloud SidecarBucketCorpus. raw = backend.search(query, limit=limit) return [ Hit( document_root=h.document_root, document_uri=h.document_uri or "", title=h.title or "", - score=h.score, + score=-h.score, shard_id=None, ) for h in raw diff --git a/arborist/qa/providence_query.py b/arborist/qa/providence_query.py index e1e3fbf..7109a04 100644 --- a/arborist/qa/providence_query.py +++ b/arborist/qa/providence_query.py @@ -308,18 +308,47 @@ def providence_query( def emit(self, *a, **kw): pass progress = _NoProgress() - # 1. RETRIEVAL ONLY — fast phase. Run corpus.fts_body + - # apply_title_boost to surface the top-K source docs. Do NOT - # call the LLM yet. Building the cache_key needs to know the - # context_root over retrieved doc roots; we get that here for - # ~sub-second on local shards. + # 1. RETRIEVAL ONLY — fast phase. Body + title FTS5 in parallel, + # merge by best bm25 per doc_root. Body alone misses canonical + # primary articles when high-DF tokens dilute the match (e.g. + # "when did sesame street first air" → body progressive-AND + # surfaces AC textbook chunks because "first"/"air" match too + # broadly; title FTS5 nails "Sesame Street" article at rank 1). + # The merge picks whichever route ranked a doc best. Title-boost + # then runs on the union. from arborist.qa.corpus import NotSupportedError, apply_title_boost + from concurrent.futures import ThreadPoolExecutor progress.emit("search.start", top_k=top_k) t_ret = _time.time() - try: - hits = corpus.fts_body(question, limit=top_k * 4) - except NotSupportedError: - hits = [] + + def _safe(method: str): + try: + fn = getattr(corpus, method, None) + if fn is None: + return [] + return fn(question, limit=top_k * 4) + except NotSupportedError: + return [] + except Exception: + return [] + + with ThreadPoolExecutor(max_workers=2) as ex: + f_body = ex.submit(_safe, "fts_body") + f_title = ex.submit(_safe, "fts_title") + body_hits = f_body.result() + title_hits = f_title.result() + + # Merge by document_root, keeping the best bm25 score across + # routes. FTS5 bm25 is "more negative = better" — title typically + # ranges -10..-20; body can be positive when progressive-AND + # collapsed to a single weak token. min() picks the better. + best: dict = {} + for h in body_hits + title_hits: + cur = best.get(h.document_root) + if cur is None or h.score < cur.score: + best[h.document_root] = h + hits = sorted(best.values(), key=lambda h: h.score) + hits = apply_title_boost( hits, question, higher_is_better=getattr(corpus, "higher_is_better", False), @@ -327,6 +356,7 @@ def providence_query( retrieval_s = _time.time() - t_ret progress.emit( "search.done", hits=len(hits), + body=len(body_hits), title=len(title_hits), ms=int(retrieval_s * 1000), )