diff --git a/arborist/qa/query.py b/arborist/qa/query.py index 4f88c0b..f6df80a 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -358,117 +358,11 @@ def _rerank_by_title( return hits -def _filter_by_title_relevance( - hits: list, - question: str, - *, - core_match_roots: set[str] | None = None, - body_density_check: callable | None = None, - phrase_match_roots: set[str] | None = None, - hyphen_fold_anchors: set[str] | None = None, - fallback_top_n: int = 5, - shards_dir=None, -) -> list: - """Concept-aware relevance filter with five accept paths: +# Re-exported from arborist.qa.retrieval_routes so run_query and the +# legacy retrieval pipeline share one implementation. +from arborist.qa.retrieval_routes import filter_by_title_relevance as _filter_by_title_relevance # noqa: E402 - 1. Title-token overlap (after synonym expansion). Strongest signal. - 2. TF-IDF core keyword overlap — `core_match_roots` is a precomputed - set of source document_roots whose TF-IDF cores contain query - tokens. Closes the gap for neologisms like "permacomputer" that - never appear in titles but are distinctive enough to be TF-IDF - keywords of conversation bodies. - 3. Body density — docs mentioning the query token >= N times pass - even without title or core match. Cheap proxy for "actually about - the topic." `body_density_check(hit)` returns bool. - 4. Phrase-match — docs whose body contains a verbatim 4+ token - sequence from the question pass even when title and content - tokens don't overlap. Closes the allusion gap (2026-05-01 - Orwell case): "has oceania always been at war with east asia" - has zero token overlap with the title "Nineteen Eighty-Four" - but the body contains the verbatim phrase "always been at - war" — without this accept path, the phrase-route hit gets - filtered out before it can rerank into the top-K. The - upstream phrase route already gates on 4-token-min sequences - (see _question_phrases) so false-positive risk is low. - 5. Hyphen-fold anchor — when the question has hyphenated runs - (Ticket #000007), `hyphen_fold_anchors` is the joined-form - set ({"bipolar"} for "bi-polar is rare?"). Title-side stem - overlap with this anchor passes the filter even when the - breadth threshold fails. Rescues non-hyphen titles like - `Bipolar disorder` from rejection while leaving non-hyphen - queries (anchors empty) unaffected. - Rivalry exclusion (Intel-titled docs in AMD queries) still applies on - every accept path. - - If all five accept paths together produce nothing, fall back to the - top `fallback_top_n` body-BM25 hits — the LLM gets enough context to - say "I don't know" rather than fabricating from a single tangential - source. - """ - qtokens = _title_query_tokens(question) - if not qtokens: - return hits - qtokens_stem = {_stem_token_for_match(t) for t in qtokens} - accept = synonym_expand(qtokens, shards_dir=shards_dir) - exclude = rivalry_excluded( - qtokens, - compare_phrasing=has_compare_phrasing(question), - shards_dir=shards_dir, - ) - core_roots = core_match_roots or set() - phrase_roots = phrase_match_roots or set() - anchor_stems = ( - {_stem_token_for_match(a) for a in hyphen_fold_anchors} - if hyphen_fold_anchors - else set() - ) - # Title-overlap breadth threshold scales with query length, mirroring - # _body_density_passes: ≤2 tokens require ALL, 3+ require N-1. Without - # this, a 2-token query like "supermans girlfriend" admits docs that - # share only ONE token with the title (e.g. `Girlfriends` the TV show) - # — title-overlap fires first & body-density never gets to reject. - title_breadth = len(qtokens) if len(qtokens) <= 2 else len(qtokens) - 1 - - kept: list = [] - for h in hits: - ttokens = _title_query_tokens(h.title.replace("_", " ")) if h.title else set() - ttokens_stem = {_stem_token_for_match(t) for t in ttokens} - if exclude & ttokens: - continue # rivalry: opposing-side title, drop it - # Direct stem-aware match against query tokens (each qtoken must - # be present, possessive/plural-tolerant). Strict signal. - direct_matches = len(qtokens_stem & ttokens_stem) - if direct_matches >= title_breadth: - kept.append(h) - continue - # Synonym fallback only for 1-token queries — otherwise a single - # synonym hit (e.g. "amd" matching an "intel"-titled doc via the - # Intel/AMD group) would over-recall. - if len(qtokens) == 1 and accept & ttokens: - kept.append(h) - continue - if h.document_root in core_roots: - kept.append(h) - continue - if h.document_root in phrase_roots: - kept.append(h) - continue - # Accept-path 5: hyphen-fold anchor (Ticket #000007). The - # joined-form variant from a hyphenated query token (e.g. - # "bipolar" from "bi-polar") matching the title's stem set - # is enough signal to pass — rescues `Bipolar disorder` from - # the breadth gate when the query was "bi-polar is rare?". - # Empty anchor set on non-hyphen queries — zero side effect. - if anchor_stems and (anchor_stems & ttokens_stem): - kept.append(h) - continue - if body_density_check is not None and body_density_check(h): - kept.append(h) - continue - if not kept: - return hits[: max(1, fallback_top_n)] if hits else [] - return kept DEFAULT_QUERY_POLICY = { diff --git a/arborist/qa/retrieval_routes.py b/arborist/qa/retrieval_routes.py new file mode 100644 index 0000000..b998b29 --- /dev/null +++ b/arborist/qa/retrieval_routes.py @@ -0,0 +1,136 @@ +"""Retrieval orchestration helpers shared between legacy ``query()`` +and the unified ``run_query()`` (Phase 1 step 3 of #53). + +Functions here take pre-retrieved Hits + a question and return filtered +or reranked Hits. They never open a sqlite connection directly — the +caller does the data fetch via the Corpus protocol and hands the +results in. Backwards-compatible with both the legacy ``_Hit`` +dataclass and the protocol ``arborist.qa.corpus.Hit`` since both +expose ``.title`` and ``.document_root``. +""" +from __future__ import annotations + +from typing import Callable, Iterable + +from arborist.qa._text_norm import stem_for_match + + +def filter_by_title_relevance( + hits: list, + question: str, + *, + core_match_roots: set[str] | None = None, + body_density_check: Callable | None = None, + phrase_match_roots: set[str] | None = None, + hyphen_fold_anchors: set[str] | None = None, + fallback_top_n: int = 5, + shards_dir=None, +) -> list: + """Concept-aware relevance filter with five accept paths: + + 1. **Title-token overlap** (after synonym expansion). Strongest + signal. Stem-tolerant via :func:`stem_for_match`. + 2. **TF-IDF core keyword overlap** — ``core_match_roots`` is a + precomputed set of source document_roots whose TF-IDF cores + contain query tokens. Closes the gap for neologisms like + ``permacomputer`` that never appear in titles but are + distinctive enough to be TF-IDF keywords of conversation + bodies. + 3. **Body density** — docs mentioning the query token ≥ N times + pass even without title or core match. Cheap proxy for "actually + about the topic." ``body_density_check(hit)`` returns bool. + 4. **Phrase-match** — docs whose body contains a verbatim 4+ token + sequence from the question pass even when title and content + tokens don't overlap. Closes the allusion gap (2026-05-01 + Orwell case): "has oceania always been at war with east asia" + has zero token overlap with the title "Nineteen Eighty-Four" + but the body contains the verbatim phrase "always been at war" + — without this accept path, the phrase-route hit gets filtered + out before it can rerank into the top-K. The upstream phrase + route already gates on 4-token-min sequences (see + :func:`arborist.qa.query._question_phrases`) so false-positive + risk is low. + 5. **Hyphen-fold anchor** — when the question has hyphenated runs + (Ticket #000007), ``hyphen_fold_anchors`` is the joined-form + set (``{"bipolar"}`` for ``"bi-polar is rare?"``). Title-side + stem overlap with this anchor passes the filter even when the + breadth threshold fails. Rescues non-hyphen titles like + ``Bipolar disorder`` from rejection while leaving non-hyphen + queries (anchors empty) unaffected. + + Rivalry exclusion (Intel-titled docs in AMD queries) still applies + on every accept path. + + If all five accept paths together produce nothing, fall back to the + top ``fallback_top_n`` body-BM25 hits — the LLM gets enough context + to say "I don't know" rather than fabricating from a single + tangential source. + """ + # Lazy imports: _title_query_tokens stays in query.py for now (full + # fold-variants stack hasn't moved); concepts.py is a sibling module + # that imports a bunch of corpus state we'd rather not load eagerly. + from arborist.qa.query import _title_query_tokens + from arborist.concepts import ( + has_compare_phrasing, + rivalry_excluded, + synonym_expand, + ) + + qtokens = _title_query_tokens(question) + if not qtokens: + return hits + qtokens_stem = {stem_for_match(t) for t in qtokens} + accept = synonym_expand(qtokens, shards_dir=shards_dir) + exclude = rivalry_excluded( + qtokens, + compare_phrasing=has_compare_phrasing(question), + shards_dir=shards_dir, + ) + core_roots = core_match_roots or set() + phrase_roots = phrase_match_roots or set() + anchor_stems = ( + {stem_for_match(a) for a in hyphen_fold_anchors} + if hyphen_fold_anchors + else set() + ) + # Title-overlap breadth scales with query length, mirroring + # _body_density_passes: ≤2 tokens require ALL, 3+ require N-1. + # Without this, "supermans girlfriend" admits `Girlfriends` (TV + # show) — title-overlap fires first & body-density never rejects. + title_breadth = len(qtokens) if len(qtokens) <= 2 else len(qtokens) - 1 + + kept: list = [] + for h in hits: + title = getattr(h, "title", "") or "" + ttokens = _title_query_tokens(title.replace("_", " ")) + ttokens_stem = {stem_for_match(t) for t in ttokens} + if exclude & ttokens: + continue # rivalry: opposing-side title, drop it + # Direct stem-aware match against query tokens. Strict signal. + direct_matches = len(qtokens_stem & ttokens_stem) + if direct_matches >= title_breadth: + kept.append(h) + continue + # Synonym fallback only for 1-token queries — multi-token + # synonym match would over-recall (single hit on "amd" matching + # an Intel-titled doc via the rivalry group, say). + if len(qtokens) == 1 and accept & ttokens: + kept.append(h) + continue + droot = getattr(h, "document_root", None) + if droot in core_roots: + kept.append(h) + continue + if droot in phrase_roots: + kept.append(h) + continue + # Accept-path 5: hyphen-fold anchor (Ticket #000007). + if anchor_stems and (anchor_stems & ttokens_stem): + kept.append(h) + continue + if body_density_check is not None and body_density_check(h): + kept.append(h) + continue + if not kept: + return hits[: max(1, fallback_top_n)] if hits else [] + return kept