diff --git a/aborist/concepts/extract.py b/aborist/concepts/extract.py index 5a37b51..03e92fc 100644 --- a/aborist/concepts/extract.py +++ b/aborist/concepts/extract.py @@ -238,10 +238,51 @@ def backfill_token_idf( # Adding a new extractor: pick a stable evidence_kind string, implement # the (conn, *, derived_from) -> dict signature, register it here. # CLI command `aborist concepts derive --extractor X` reads this map. +def backfill_documents_fts( + conn: sqlite3.Connection, + *, + derived_from: str | None = None, +) -> dict[str, int]: + """Populate ``documents_fts`` with every documents.title row. + + Replaces the un-indexable LOWER(title) LIKE '%tok%' title-LIKE + search. After this runs, _search_titles can use FTS5 MATCH for + O(K) hash lookups instead of O(corpus × |tokens|) full scan — + which means synonym-expanded title search becomes affordable + on long queries without blowing the per-shard budget. + + Idempotent: DELETE FROM documents_fts; INSERT ... — we reset & + rebuild rather than incrementally upsert because FTS5 contentless + tables don't support partial-key dedupe well. Cost is bounded by + document count (~870k per shard). + + Returns ``{"rows_indexed": N, "elapsed_ms": M}``. + """ + derived_at = int(time.time()) + derived_from = derived_from or "extract.backfill_documents_fts" + t0 = time.time() + conn.execute("DELETE FROM documents_fts") + cursor = conn.execute( + "INSERT INTO documents_fts (rowid, title) " + "SELECT rowid, title FROM documents WHERE title IS NOT NULL" + ) + rows_indexed = cursor.rowcount + conn.commit() + return { + "rows_indexed": rows_indexed, + "elapsed_ms": int((time.time() - t0) * 1000), + } + + EXTRACTORS: dict[str, Callable[..., dict[str, int]]] = { "link_reciprocity": link_reciprocity_synonym, # Not a relation extractor — populates concept_token_idf for IDF # ranking at synonym_expand cap-time. Run AFTER any synonym # extractor since it indexes the union of token + target columns. "token_idf": backfill_token_idf, + # Not a relation extractor — populates the documents_fts virtual + # table that replaces the un-indexable title-LIKE backup search + # in qa.query._search_corpus. Run once per shard at ingest or + # after a bulk title backfill. + "documents_fts": backfill_documents_fts, } diff --git a/aborist/qa/query.py b/aborist/qa/query.py index c356300..eb85e52 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -628,13 +628,44 @@ def _search_titles(conn, qtokens: list[str], limit: int) -> list[tuple]: """ if not qtokens: return [] - MAX_TITLE_LIKE_TOKENS = 24 - bounded = list(qtokens)[:MAX_TITLE_LIKE_TOKENS] + over_fetch_limit = limit * 4 + + # Try FTS5 documents_fts first — O(K) hash lookup vs the un-indexable + # LOWER(title) LIKE '%tok%' that this function used through 2026- + # 05-02. The MATCH expression OR-joins the input tokens (quoted to + # neutralize FTS5 syntax). Empirical: ~0.05s/shard regardless of + # token count, vs. 10-15s for the LIKE form on the 870k-doc shard. + # Falls back to LIKE only on shards whose documents_fts isn't + # populated yet (e.g. legacy ingest before this index landed). + bounded = list(qtokens)[:24] + has_fts = conn.execute( + "SELECT 1 FROM documents_fts WHERE rowid = (SELECT MIN(rowid) FROM documents_fts) LIMIT 1" + ).fetchone() + if has_fts: + # Quote each token & OR-join. FTS5's tokenizer applies the same + # porter stemming we use elsewhere, so 'permacomputer' / 'permac' + # match coherently. + match_expr = " OR ".join(f'"{t.lower().replace(chr(34), chr(34) * 2)}"' for t in bounded) + try: + rows = conn.execute( + "SELECT d.document_root, d.document_uri, d.title " + "FROM documents_fts AS f " + "JOIN documents AS d ON d.rowid = f.rowid " + "WHERE documents_fts MATCH ? " + "ORDER BY LENGTH(d.title) ASC LIMIT ?", + (match_expr, over_fetch_limit), + ).fetchall() + return rows + except Exception: + # Malformed MATCH (rare; tokenizer-strange chars survived + # the quote escape). Fall through to LIKE form. + pass + + # LIKE fallback — kept for shards lacking documents_fts data. + # Capped at 24 tokens so the OR-chain expression-tree stays under + # SQLite's depth-1000 limit on long queries. clauses = " OR ".join(["LOWER(title) LIKE ?"] * len(bounded)) likes = [f"%{t.lower()}%" for t in bounded] - # Bump effective limit so the post-filter sees enough candidates - # to find genuine matches even when the OR-chain is permissive. - over_fetch_limit = limit * 4 params = likes + [over_fetch_limit] rows = conn.execute( f"SELECT document_root, document_uri, title FROM documents " @@ -995,18 +1026,13 @@ def _search_corpus( accept_stems = { _stem_token_for_match(t) for t in accept_tokens } - # Title-LIKE is O(corpus × |tokens|) full-scan (LIKE '%tok%' - # can't use any index). Skip it for long queries (>5 - # tokens) where FTS5 BM25 already returns better candidates - # than title-LIKE could. Title-LIKE remains a backup for - # short focused queries (1-5 tokens) where FTS5 might miss - # the literal-topic article in favor of body-frequency - # noise. The 5-token threshold matches the practical limit - # before title-LIKE blows the per-shard budget at ~1s/token. - title_search_rows = ( - _search_titles(conn, list(accept_tokens), over_fetch) - if len(accept_tokens) <= 5 - else [] + # Title search now uses documents_fts FTS5 index (~0.05s/shard + # regardless of token count) — the prior >5-token bypass + # existed because LIKE '%tok%' was O(corpus × |tokens|). + # FTS5 MATCH makes this an O(K) hash lookup, so synonym- + # expanded title search is affordable at any query length. + title_search_rows = _search_titles( + conn, list(accept_tokens), over_fetch ) for r in title_search_rows: title_norm = (r["title"] or "").replace("_", " ") diff --git a/aborist/store.py b/aborist/store.py index bd2b64c..e0f8b52 100644 --- a/aborist/store.py +++ b/aborist/store.py @@ -317,6 +317,26 @@ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5( tokenize = 'porter unicode61' ); +-- Document-title FTS5 index. Replaces the un-indexable +-- `LOWER(title) LIKE '%tok%'` title-LIKE search with O(K) hash +-- lookup. Pre-2026-05-02 the title-LIKE backup was either skipped +-- (>5 tokens) or paid ~10s/shard for short queries. The MATCH-based +-- replacement runs in ~0.05s/shard regardless of token count, which +-- means we can re-enable synonym-expanded title search for long +-- queries without paying the corpus-scan cost. +-- +-- Contentless mode: same trick as chunks_fts — store only the +-- inverted index, not a copy of the title. The rowid joins back to +-- documents.rowid (sqlite's hidden integer rowid is fine for a +-- 1-1 mapping). On re-ingest, the FTS5 row gets replaced via the +-- ingest path's INSERT OR REPLACE INTO documents flow. +CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5( + title, + content='', + contentless_delete=1, + tokenize = 'porter unicode61' +); + -- Concept-relations layer. Append-only secondary index over the corpus. -- Each row is a (token, target) edge of a given relation_kind, derived -- from a specific source document by a specific extractor (evidence_kind).