From 1d70c4fd25f3027eac4dc4f9b16cdc84ef8181c9 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Fri, 1 May 2026 22:52:02 -0400 Subject: [PATCH] =?UTF-8?q?qa(retrieval):=20fix=2050=E2=86=92100s=20hangs?= =?UTF-8?q?=20in=20title-LIKE=20&=20FTS5=20OR-fallback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both retrieval phases were doing O(corpus × tokens) full-scan work on long queries — synonym work surfaced the latent issues but they predate it. Hermes is 6-8s; retrieval needs to be sub-second. (1) `_search_titles` (LIKE-based title backup search) is fundamentally O(corpus × |accept_tokens|) — `LOWER(title) LIKE '%tok%'` can't use any index. 50 patterns × 870k docs × 4 shards = ~56s pure full-scan. Skipped for queries with > 5 accept_tokens, where FTS5 BM25 already returns better candidates. Title-LIKE remains a backup for short focused queries (1-5 tokens). (2) FTS5 OR-mode fallback was OR-joining ALL post-stopword query tokens. A 19-token query OR-clause matched millions of docs; BM25 then ranked them all. 13s/shard observed. Capped to top-5 LONGEST tokens (proxy for rarity / topical specificity). Long words like "neurotechnology" matter; short words like "soon" don't. OR-mode now 0.25s/shard. (3) Stopword set extended with 6 connector words (one, some, another, without, soon, currently). Conservative addition — these have zero topical signal. Reduces the AND-mode token count, raising the chance AND succeeds before falling back to OR. Verified live: the 19-token brain-tech query that hung indefinitely now returns in 18s (Hermes + verify + render dominate). Source `_search_corpus` no longer expands synonyms for the title-LIKE call either — synonym expansion stays in `_filter_by_title_relevance` (post-retrieval, in-memory, cheap regardless of accept-set size) where it actually does useful work. Tests: 641 passed (no regression). --- aborist/qa/query.py | 29 +++++++++++++++++++++++++---- aborist/search/fts5.py | 30 +++++++++++++++++++++++++++--- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 169ec81..3c31573 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -911,9 +911,17 @@ def _search_corpus( out-ranks FTS5 body hits so the actual topic article rises to the top. """ qtokens = _title_query_tokens(question) - # Synonym expansion: a query for "athlon" also fetches AMD-titled docs. - # Reads concept_relations cross-shard via the shards_dir already in scope. - accept_tokens = synonym_expand(qtokens, shards_dir=shards_dir) + # Title-LIKE backup search uses ORIGINAL qtokens only — NOT synonym- + # expanded. With expansion, this path becomes O(corpus × |accept|) + # full-scan because LOWER(title) LIKE '%tok%' can't use any index; + # 50 LIKE patterns × 870k docs × 4 shards = ~56s of pure waste. + # Synonym expansion stays in `_filter_by_title_relevance` (post- + # retrieval, in-memory, cheap regardless of accept-set size) where + # it actually does useful work surfacing synonym-related titles + # from the FTS5 BM25 hits. Title-LIKE is a thin backup for docs + # whose body is short on query terms but whose title is the literal + # topic — that case is well-served by the original qtokens. + accept_tokens = set(qtokens) paths: list[Path] if shards_dir is not None: paths = discover_shards(shards_dir) @@ -980,7 +988,20 @@ def _search_corpus( accept_stems = { _stem_token_for_match(t) for t in accept_tokens } - for r in _search_titles(conn, list(accept_tokens), over_fetch): + # Title-LIKE is O(corpus × |tokens|) full-scan (LIKE '%tok%' + # can't use any index). Skip it for long queries (>5 + # tokens) where FTS5 BM25 already returns better candidates + # than title-LIKE could. Title-LIKE remains a backup for + # short focused queries (1-5 tokens) where FTS5 might miss + # the literal-topic article in favor of body-frequency + # noise. The 5-token threshold matches the practical limit + # before title-LIKE blows the per-shard budget at ~1s/token. + title_search_rows = ( + _search_titles(conn, list(accept_tokens), over_fetch) + if len(accept_tokens) <= 5 + else [] + ) + for r in title_search_rows: title_norm = (r["title"] or "").replace("_", " ") title_stems = { _stem_token_for_match(t) diff --git a/aborist/search/fts5.py b/aborist/search/fts5.py index c27c48d..2029bc3 100644 --- a/aborist/search/fts5.py +++ b/aborist/search/fts5.py @@ -27,10 +27,22 @@ _FTS5_STOPWORDS = frozenset( tell show describe explain summarize say give list find make please all there know everything anything something + + one some another without soon currently """.split() ) +# OR-mode fallback hits SQLite FTS5 with one clause per token. With 15+ +# tokens including common English connectors it matches millions of +# docs, then BM25 has to rank them all to pick the top-K — 13s/shard +# observed on a 3.47M-doc corpus. Long queries that fail AND-mode +# should not pay for an OR-mode that's effectively a full-corpus scan. +# Cap OR-mode at the top-N LONGEST tokens (proxy for rarity / topical +# specificity — "neurotechnology" matters; "soon" doesn't). +_OR_FALLBACK_MAX_TOKENS = 5 + + def _query_tokens(query: str) -> list[str]: raw = _FTS5_TOKEN_RE.findall(query) return [t for t in raw if t.lower() not in _FTS5_STOPWORDS and len(t) > 1] @@ -48,8 +60,12 @@ def _escape_fts5(query: str, *, mode: str = "and") -> str: - mode='and' (default): every content token must appear in the doc. Strict relevance — keeps unrelated docs out of the context window. - - mode='or': any token matches. Used as a fallback when AND returns - zero hits. + - mode='or': the top-N LONGEST tokens (proxy for topical specificity) + get OR-joined. Used as a fallback when AND returns zero hits. + Capped at ``_OR_FALLBACK_MAX_TOKENS`` so OR-mode doesn't degrade + into a full-corpus scan on long noisy queries (a 19-token OR + clause matches millions of docs and forces BM25 to rank them all + — 13s/shard observed pre-cap). All-stopword queries fall back to OR over the raw tokens so they still find something instead of crashing FTS5 with an empty MATCH. @@ -59,8 +75,16 @@ def _escape_fts5(query: str, *, mode: str = "and") -> str: raw = _FTS5_TOKEN_RE.findall(query) tokens = [t for t in raw if len(t) > 1] or ['""'] sep = " OR " + elif mode == "and": + sep = " AND " else: - sep = " AND " if mode == "and" else " OR " + # OR fallback: keep only the rarest tokens (approximated by + # longest — long words tend to be more topical / rarer in the + # corpus). Bounds the per-clause cost so OR-mode terminates + # quickly instead of scanning the corpus. + sep = " OR " + if len(tokens) > _OR_FALLBACK_MAX_TOKENS: + tokens = sorted(tokens, key=len, reverse=True)[:_OR_FALLBACK_MAX_TOKENS] return sep.join(_quote(t) for t in tokens)