diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 169ec81..3c31573 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -911,9 +911,17 @@ def _search_corpus( out-ranks FTS5 body hits so the actual topic article rises to the top. """ qtokens = _title_query_tokens(question) - # Synonym expansion: a query for "athlon" also fetches AMD-titled docs. - # Reads concept_relations cross-shard via the shards_dir already in scope. - accept_tokens = synonym_expand(qtokens, shards_dir=shards_dir) + # Title-LIKE backup search uses ORIGINAL qtokens only — NOT synonym- + # expanded. With expansion, this path becomes O(corpus × |accept|) + # full-scan because LOWER(title) LIKE '%tok%' can't use any index; + # 50 LIKE patterns × 870k docs × 4 shards = ~56s of pure waste. + # Synonym expansion stays in `_filter_by_title_relevance` (post- + # retrieval, in-memory, cheap regardless of accept-set size) where + # it actually does useful work surfacing synonym-related titles + # from the FTS5 BM25 hits. Title-LIKE is a thin backup for docs + # whose body is short on query terms but whose title is the literal + # topic — that case is well-served by the original qtokens. + accept_tokens = set(qtokens) paths: list[Path] if shards_dir is not None: paths = discover_shards(shards_dir) @@ -980,7 +988,20 @@ def _search_corpus( accept_stems = { _stem_token_for_match(t) for t in accept_tokens } - for r in _search_titles(conn, list(accept_tokens), over_fetch): + # Title-LIKE is O(corpus × |tokens|) full-scan (LIKE '%tok%' + # can't use any index). Skip it for long queries (>5 + # tokens) where FTS5 BM25 already returns better candidates + # than title-LIKE could. Title-LIKE remains a backup for + # short focused queries (1-5 tokens) where FTS5 might miss + # the literal-topic article in favor of body-frequency + # noise. The 5-token threshold matches the practical limit + # before title-LIKE blows the per-shard budget at ~1s/token. + title_search_rows = ( + _search_titles(conn, list(accept_tokens), over_fetch) + if len(accept_tokens) <= 5 + else [] + ) + for r in title_search_rows: title_norm = (r["title"] or "").replace("_", " ") title_stems = { _stem_token_for_match(t) diff --git a/aborist/search/fts5.py b/aborist/search/fts5.py index c27c48d..2029bc3 100644 --- a/aborist/search/fts5.py +++ b/aborist/search/fts5.py @@ -27,10 +27,22 @@ _FTS5_STOPWORDS = frozenset( tell show describe explain summarize say give list find make please all there know everything anything something + + one some another without soon currently """.split() ) +# OR-mode fallback hits SQLite FTS5 with one clause per token. With 15+ +# tokens including common English connectors it matches millions of +# docs, then BM25 has to rank them all to pick the top-K — 13s/shard +# observed on a 3.47M-doc corpus. Long queries that fail AND-mode +# should not pay for an OR-mode that's effectively a full-corpus scan. +# Cap OR-mode at the top-N LONGEST tokens (proxy for rarity / topical +# specificity — "neurotechnology" matters; "soon" doesn't). +_OR_FALLBACK_MAX_TOKENS = 5 + + def _query_tokens(query: str) -> list[str]: raw = _FTS5_TOKEN_RE.findall(query) return [t for t in raw if t.lower() not in _FTS5_STOPWORDS and len(t) > 1] @@ -48,8 +60,12 @@ def _escape_fts5(query: str, *, mode: str = "and") -> str: - mode='and' (default): every content token must appear in the doc. Strict relevance — keeps unrelated docs out of the context window. - - mode='or': any token matches. Used as a fallback when AND returns - zero hits. + - mode='or': the top-N LONGEST tokens (proxy for topical specificity) + get OR-joined. Used as a fallback when AND returns zero hits. + Capped at ``_OR_FALLBACK_MAX_TOKENS`` so OR-mode doesn't degrade + into a full-corpus scan on long noisy queries (a 19-token OR + clause matches millions of docs and forces BM25 to rank them all + — 13s/shard observed pre-cap). All-stopword queries fall back to OR over the raw tokens so they still find something instead of crashing FTS5 with an empty MATCH. @@ -59,8 +75,16 @@ def _escape_fts5(query: str, *, mode: str = "and") -> str: raw = _FTS5_TOKEN_RE.findall(query) tokens = [t for t in raw if len(t) > 1] or ['""'] sep = " OR " + elif mode == "and": + sep = " AND " else: - sep = " AND " if mode == "and" else " OR " + # OR fallback: keep only the rarest tokens (approximated by + # longest — long words tend to be more topical / rarer in the + # corpus). Bounds the per-clause cost so OR-mode terminates + # quickly instead of scanning the corpus. + sep = " OR " + if len(tokens) > _OR_FALLBACK_MAX_TOKENS: + tokens = sorted(tokens, key=len, reverse=True)[:_OR_FALLBACK_MAX_TOKENS] return sep.join(_quote(t) for t in tokens)