From 7887a588f945c77d6c37b1cd55508150bed7ab28 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sat, 2 May 2026 06:57:10 -0400 Subject: [PATCH] =?UTF-8?q?qa(retrieval):=20documents=5Ffts=20FTS5=20index?= =?UTF-8?q?=20=E2=80=94=20title=20search=201250=C3=97=20faster?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2 of the holistic-tuning pair. Replaces the un-indexable LOWER(title) LIKE '%tok%' title-search with an FTS5 MATCH-based lookup. The structural fix that was deferred when the >5-token bypass landed (commit 1d70c4f). (1) Schema: new `documents_fts` virtual table over the title column. Contentless mode (same trick as chunks_fts) — stores only the inverted index, not a copy of the title. Joins back to documents via rowid for the post-filter caller. (2) Extractor: backfill_documents_fts (registered as evidence_kind "documents_fts"). Reset-and-rebuild the index from documents in one INSERT...SELECT. Idempotent. Cost: ~2.5s per 870k-doc shard. (3) `_search_titles` rewritten: try FTS5 MATCH first, fall back to LOWER(title) LIKE only on shards lacking documents_fts data (legacy ingest pre-this-commit). MATCH expression OR-joins quoted query tokens; falls through to LIKE form on tokenizer edge cases. (4) Removed the `>5 accept_tokens` bypass in `_search_corpus` that commit 1d70c4f added as a workaround for the LIKE full-scan cost. With FTS5 the title search is sub-second regardless of token count, so synonym-expanded title search is affordable at any query length. Live verified on the 19-token brain-tech query with 50 accept_tokens (post-IDF expansion): Pre-FTS5: ~50s (LIKE '%tok%' × 870k docs × 4 shards) Post-FTS5: ~0.04s ← 1250× speedup End-to-end query: 10.4s (was 11.6-13s; the saved title-search time partially absorbed by Hermes inference variance). Backfill cost: 10.5s wall-clock across 4 wiki shards + 1 crawl shard. Storage: ~30 MB per wiki shard for documents_fts (well inside the 90 MB/shard concept-layer budget). Re-running is idempotent (DELETE FROM ... INSERT INTO ...). Tests: 641 passed (no regression). --- aborist/concepts/extract.py | 41 +++++++++++++++++++++++++ aborist/qa/query.py | 60 ++++++++++++++++++++++++++----------- aborist/store.py | 20 +++++++++++++ 3 files changed, 104 insertions(+), 17 deletions(-) diff --git a/aborist/concepts/extract.py b/aborist/concepts/extract.py index 5a37b51..03e92fc 100644 --- a/aborist/concepts/extract.py +++ b/aborist/concepts/extract.py @@ -238,10 +238,51 @@ def backfill_token_idf( # Adding a new extractor: pick a stable evidence_kind string, implement # the (conn, *, derived_from) -> dict signature, register it here. # CLI command `aborist concepts derive --extractor X` reads this map. +def backfill_documents_fts( + conn: sqlite3.Connection, + *, + derived_from: str | None = None, +) -> dict[str, int]: + """Populate ``documents_fts`` with every documents.title row. + + Replaces the un-indexable LOWER(title) LIKE '%tok%' title-LIKE + search. After this runs, _search_titles can use FTS5 MATCH for + O(K) hash lookups instead of O(corpus × |tokens|) full scan — + which means synonym-expanded title search becomes affordable + on long queries without blowing the per-shard budget. + + Idempotent: DELETE FROM documents_fts; INSERT ... — we reset & + rebuild rather than incrementally upsert because FTS5 contentless + tables don't support partial-key dedupe well. Cost is bounded by + document count (~870k per shard). + + Returns ``{"rows_indexed": N, "elapsed_ms": M}``. + """ + derived_at = int(time.time()) + derived_from = derived_from or "extract.backfill_documents_fts" + t0 = time.time() + conn.execute("DELETE FROM documents_fts") + cursor = conn.execute( + "INSERT INTO documents_fts (rowid, title) " + "SELECT rowid, title FROM documents WHERE title IS NOT NULL" + ) + rows_indexed = cursor.rowcount + conn.commit() + return { + "rows_indexed": rows_indexed, + "elapsed_ms": int((time.time() - t0) * 1000), + } + + EXTRACTORS: dict[str, Callable[..., dict[str, int]]] = { "link_reciprocity": link_reciprocity_synonym, # Not a relation extractor — populates concept_token_idf for IDF # ranking at synonym_expand cap-time. Run AFTER any synonym # extractor since it indexes the union of token + target columns. "token_idf": backfill_token_idf, + # Not a relation extractor — populates the documents_fts virtual + # table that replaces the un-indexable title-LIKE backup search + # in qa.query._search_corpus. Run once per shard at ingest or + # after a bulk title backfill. + "documents_fts": backfill_documents_fts, } diff --git a/aborist/qa/query.py b/aborist/qa/query.py index c356300..eb85e52 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -628,13 +628,44 @@ def _search_titles(conn, qtokens: list[str], limit: int) -> list[tuple]: """ if not qtokens: return [] - MAX_TITLE_LIKE_TOKENS = 24 - bounded = list(qtokens)[:MAX_TITLE_LIKE_TOKENS] + over_fetch_limit = limit * 4 + + # Try FTS5 documents_fts first — O(K) hash lookup vs the un-indexable + # LOWER(title) LIKE '%tok%' that this function used through 2026- + # 05-02. The MATCH expression OR-joins the input tokens (quoted to + # neutralize FTS5 syntax). Empirical: ~0.05s/shard regardless of + # token count, vs. 10-15s for the LIKE form on the 870k-doc shard. + # Falls back to LIKE only on shards whose documents_fts isn't + # populated yet (e.g. legacy ingest before this index landed). + bounded = list(qtokens)[:24] + has_fts = conn.execute( + "SELECT 1 FROM documents_fts WHERE rowid = (SELECT MIN(rowid) FROM documents_fts) LIMIT 1" + ).fetchone() + if has_fts: + # Quote each token & OR-join. FTS5's tokenizer applies the same + # porter stemming we use elsewhere, so 'permacomputer' / 'permac' + # match coherently. + match_expr = " OR ".join(f'"{t.lower().replace(chr(34), chr(34) * 2)}"' for t in bounded) + try: + rows = conn.execute( + "SELECT d.document_root, d.document_uri, d.title " + "FROM documents_fts AS f " + "JOIN documents AS d ON d.rowid = f.rowid " + "WHERE documents_fts MATCH ? " + "ORDER BY LENGTH(d.title) ASC LIMIT ?", + (match_expr, over_fetch_limit), + ).fetchall() + return rows + except Exception: + # Malformed MATCH (rare; tokenizer-strange chars survived + # the quote escape). Fall through to LIKE form. + pass + + # LIKE fallback — kept for shards lacking documents_fts data. + # Capped at 24 tokens so the OR-chain expression-tree stays under + # SQLite's depth-1000 limit on long queries. clauses = " OR ".join(["LOWER(title) LIKE ?"] * len(bounded)) likes = [f"%{t.lower()}%" for t in bounded] - # Bump effective limit so the post-filter sees enough candidates - # to find genuine matches even when the OR-chain is permissive. - over_fetch_limit = limit * 4 params = likes + [over_fetch_limit] rows = conn.execute( f"SELECT document_root, document_uri, title FROM documents " @@ -995,18 +1026,13 @@ def _search_corpus( accept_stems = { _stem_token_for_match(t) for t in accept_tokens } - # Title-LIKE is O(corpus × |tokens|) full-scan (LIKE '%tok%' - # can't use any index). Skip it for long queries (>5 - # tokens) where FTS5 BM25 already returns better candidates - # than title-LIKE could. Title-LIKE remains a backup for - # short focused queries (1-5 tokens) where FTS5 might miss - # the literal-topic article in favor of body-frequency - # noise. The 5-token threshold matches the practical limit - # before title-LIKE blows the per-shard budget at ~1s/token. - title_search_rows = ( - _search_titles(conn, list(accept_tokens), over_fetch) - if len(accept_tokens) <= 5 - else [] + # Title search now uses documents_fts FTS5 index (~0.05s/shard + # regardless of token count) — the prior >5-token bypass + # existed because LIKE '%tok%' was O(corpus × |tokens|). + # FTS5 MATCH makes this an O(K) hash lookup, so synonym- + # expanded title search is affordable at any query length. + title_search_rows = _search_titles( + conn, list(accept_tokens), over_fetch ) for r in title_search_rows: title_norm = (r["title"] or "").replace("_", " ") diff --git a/aborist/store.py b/aborist/store.py index bd2b64c..e0f8b52 100644 --- a/aborist/store.py +++ b/aborist/store.py @@ -317,6 +317,26 @@ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5( tokenize = 'porter unicode61' ); +-- Document-title FTS5 index. Replaces the un-indexable +-- `LOWER(title) LIKE '%tok%'` title-LIKE search with O(K) hash +-- lookup. Pre-2026-05-02 the title-LIKE backup was either skipped +-- (>5 tokens) or paid ~10s/shard for short queries. The MATCH-based +-- replacement runs in ~0.05s/shard regardless of token count, which +-- means we can re-enable synonym-expanded title search for long +-- queries without paying the corpus-scan cost. +-- +-- Contentless mode: same trick as chunks_fts — store only the +-- inverted index, not a copy of the title. The rowid joins back to +-- documents.rowid (sqlite's hidden integer rowid is fine for a +-- 1-1 mapping). On re-ingest, the FTS5 row gets replaced via the +-- ingest path's INSERT OR REPLACE INTO documents flow. +CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5( + title, + content='', + contentless_delete=1, + tokenize = 'porter unicode61' +); + -- Concept-relations layer. Append-only secondary index over the corpus. -- Each row is a (token, target) edge of a given relation_kind, derived -- from a specific source document by a specific extractor (evidence_kind).