diff --git a/arborist/qa/corpus.py b/arborist/qa/corpus.py index e9708d9..a6b8bb6 100644 --- a/arborist/qa/corpus.py +++ b/arborist/qa/corpus.py @@ -321,15 +321,21 @@ class SqliteShardCorpus: return [] from arborist.search.fts5 import FTS5Backend backend = FTS5Backend(self._conn) - # FTS5Backend.search returns objects with document_root, - # document_uri, title, chunk_idx, score, snippet. + # FTS5Backend.search sign-FLIPS bm25 internally (so higher + # numbers = stronger match in its return shape). Title FTS5 + # via fts_title returns RAW bm25 (more-negative = better). To + # let providence_query merge body+title hits by a single + # min-bm25 rule, re-flip FTS5Backend's positive score back + # to negative here. Net: SqliteShardCorpus.fts_body and + # .fts_title both emit raw bm25 (lower = better), same + # convention as the cloud SidecarBucketCorpus. raw = backend.search(query, limit=limit) return [ Hit( document_root=h.document_root, document_uri=h.document_uri or "", title=h.title or "", - score=h.score, + score=-h.score, shard_id=None, ) for h in raw diff --git a/arborist/qa/providence_query.py b/arborist/qa/providence_query.py index e1e3fbf..7109a04 100644 --- a/arborist/qa/providence_query.py +++ b/arborist/qa/providence_query.py @@ -308,18 +308,47 @@ def providence_query( def emit(self, *a, **kw): pass progress = _NoProgress() - # 1. RETRIEVAL ONLY — fast phase. Run corpus.fts_body + - # apply_title_boost to surface the top-K source docs. Do NOT - # call the LLM yet. Building the cache_key needs to know the - # context_root over retrieved doc roots; we get that here for - # ~sub-second on local shards. + # 1. RETRIEVAL ONLY — fast phase. Body + title FTS5 in parallel, + # merge by best bm25 per doc_root. Body alone misses canonical + # primary articles when high-DF tokens dilute the match (e.g. + # "when did sesame street first air" → body progressive-AND + # surfaces AC textbook chunks because "first"/"air" match too + # broadly; title FTS5 nails "Sesame Street" article at rank 1). + # The merge picks whichever route ranked a doc best. Title-boost + # then runs on the union. from arborist.qa.corpus import NotSupportedError, apply_title_boost + from concurrent.futures import ThreadPoolExecutor progress.emit("search.start", top_k=top_k) t_ret = _time.time() - try: - hits = corpus.fts_body(question, limit=top_k * 4) - except NotSupportedError: - hits = [] + + def _safe(method: str): + try: + fn = getattr(corpus, method, None) + if fn is None: + return [] + return fn(question, limit=top_k * 4) + except NotSupportedError: + return [] + except Exception: + return [] + + with ThreadPoolExecutor(max_workers=2) as ex: + f_body = ex.submit(_safe, "fts_body") + f_title = ex.submit(_safe, "fts_title") + body_hits = f_body.result() + title_hits = f_title.result() + + # Merge by document_root, keeping the best bm25 score across + # routes. FTS5 bm25 is "more negative = better" — title typically + # ranges -10..-20; body can be positive when progressive-AND + # collapsed to a single weak token. min() picks the better. + best: dict = {} + for h in body_hits + title_hits: + cur = best.get(h.document_root) + if cur is None or h.score < cur.score: + best[h.document_root] = h + hits = sorted(best.values(), key=lambda h: h.score) + hits = apply_title_boost( hits, question, higher_is_better=getattr(corpus, "higher_is_better", False), @@ -327,6 +356,7 @@ def providence_query( retrieval_s = _time.time() - t_ret progress.emit( "search.done", hits=len(hits), + body=len(body_hits), title=len(title_hits), ms=int(retrieval_s * 1000), )