diff --git a/arborist/qa/corpus.py b/arborist/qa/corpus.py index c527469..e9708d9 100644 --- a/arborist/qa/corpus.py +++ b/arborist/qa/corpus.py @@ -296,31 +296,43 @@ class SqliteShardCorpus: self._conn = conn def fts_body(self, query: str, *, limit: int = 8) -> list[Hit]: - sql = ( - "SELECT d.document_root, d.document_uri, d.title, " - " bm25(chunks_fts) AS score " - "FROM chunks_fts JOIN chunks c ON c.rowid = chunks_fts.rowid " - "JOIN documents d ON d.document_root = c.document_root " - "WHERE chunks_fts MATCH ? " - "ORDER BY score LIMIT ?" - ) - # The caller passes natural-language; sanitize on this side so - # FTS5 doesn't choke on punctuation. Mirrors the bucket-direct - # sanitizer (arborist.wallet.bucket._to_fts5). + """Body-FTS5 via the existing FTS5Backend (progressive-AND + with OR-mode fallback). + + Naive ``MATCH 'tok1 OR tok2 OR ...'`` over a multi-token + natural-language query is catastrophic on a 1.5M-chunk shard + when ANY token has high document-frequency: BM25 has to score + every match in the OR-union, and "muppet show first air + television" pulls ~300k matches in ~30s per shard (×5 shards + = ~150s wall time). FTS5Backend.search runs progressive-AND + first (intersecting posting lists, fast), retries with + shortest-token-dropped on zero results, and only falls back + to OR mode when AND exhausts — clipping the high-DF tail of + tokens that would otherwise dominate. + + Returns the protocol-shaped Hit; FTS5Backend's own Hit lives + in arborist.search.base. + """ + # Short-circuit all-stopword queries — FTS5Backend would + # fall back to a sentinel ``""`` token that matches arbitrary + # docs; we want pure []. from arborist.wallet.bucket import _to_fts5 - match_expr = _to_fts5(query) - if not match_expr.strip(): + if not _to_fts5(query).strip(): return [] - rows = list(self._conn.execute(sql, (match_expr, limit))) + from arborist.search.fts5 import FTS5Backend + backend = FTS5Backend(self._conn) + # FTS5Backend.search returns objects with document_root, + # document_uri, title, chunk_idx, score, snippet. + raw = backend.search(query, limit=limit) return [ Hit( - document_root=r["document_root"], - document_uri=r["document_uri"] or "", - title=r["title"] or "", - score=r["score"], + document_root=h.document_root, + document_uri=h.document_uri or "", + title=h.title or "", + score=h.score, shard_id=None, ) - for r in rows + for h in raw ] def fts_title(self, query: str, *, limit: int = 8) -> list[Hit]: