From 8fb1fe51d7c2aefcf782c4cfcb80d35177f31059 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sat, 2 May 2026 14:12:46 -0400 Subject: [PATCH] =?UTF-8?q?qa(query):=20#000007=20land=20=E2=80=94=20query?= =?UTF-8?q?-layer=20hyphen=20folding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the FTS5 hyphen-tokenization asymmetry: `bi-polar is rare?` retrieved only the Bi-Polar album/disambiguation cluster while the medical-condition cluster (Bipolar disorder, Bipolar I/II disorder, etc.) sat in the same shards untouched. `unicode61` splits hyphens at index AND query time; `Bi-Polar Blues` indexes as [bi, polar, ...] while `Bipolar disorder` indexes as [bipolar] — non-overlapping token sets that never met. Fix is query-layer only — no canonicalization_version bump, no re-index, existing cache_keys stay valid: - _hyphen_fold_variants(s): emit joined-no-hyphen variants for every hyphenated run. - _title_query_tokens(s): additively merges variants symmetrically (queries AND titles when called on either). - _filter_by_title_relevance: accept-path 5 — title stem-overlap with hyphen-fold anchors passes the breadth gate. Rescues `Bipolar disorder` (1-of-N qtoken match) without disrupting non-hyphen queries (anchors empty → zero side effect). - DEFAULT_QUERY_POLICY / DEFAULT_POLICY: hyphen_fold_v1: True marker folds into governance_policy_hash; new records cache-split cleanly from pre-fold records. Live verification on /home/fox/.aborist/shards: same query now retrieves `Bipolar disorder` (#5) and `Bipolar` disambiguation (#7); model cites both, answer reads "Bi-polar disorder is not rare; it affects approximately 2.8% of the U.S. population". EVIDENCE-WARRANTED 2/2, properly grounded. Tests: 4 new (3 unit, 1 integration with regression-pinned Bipolar-disorder retrieval). Full suite 760 passed, 34 skipped. Also: CLAUDE.md gains a close-when-complete hint for tickets — an open ticket whose code already shipped is a stale map. --- CLAUDE.md | 9 +- aborist/qa/query.py | 75 ++++- aborist/qa/runner.py | 3 + docs/TICKETS.md | 3 +- .../ticket-000007-query-layer-hyphen-fold.md | 263 ++++++++++++++++++ tests/test_query.py | 100 +++++++ 6 files changed, 446 insertions(+), 7 deletions(-) create mode 100644 docs/tickets/ticket-000007-query-layer-hyphen-fold.md diff --git a/CLAUDE.md b/CLAUDE.md index cbe8d13..a8e314f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -336,9 +336,12 @@ Architecture / ongoing work: `docs/self-reference-design.md` — recursive distillation. Tickets: `docs/TICKETS.md` is the authoritative index with `Next -ID`. As of 2026-05-02 all five shipped tickets (#000001–#000005) -are closed; closed tickets stay in place as the design log. New -tickets bump `Next ID` atomically. +ID`. Closed tickets stay in place as the design log. New tickets +bump `Next ID` atomically. **Close tickets when the work lands — +flip Status to `closed · landed in commit ` (or +`closed · YYYY-MM-DD`) in the ticket file AND in the index row, in +the same commit as the implementation.** An open ticket whose code +already shipped is a stale map. ## Orientation protocol diff --git a/aborist/qa/query.py b/aborist/qa/query.py index c87b3c3..ec33dde 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -106,6 +106,15 @@ from aborist.store import ( _TITLE_TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z0-9]*") +# Hyphen-joined run of two-or-more word tokens. Used by +# `_hyphen_fold_variants` to emit joined-no-hyphen variants. See +# Ticket #000007 for the FTS5 hyphen-tokenization-asymmetry rationale: +# `Bipolar disorder` indexes as [bipolar], `Bi-Polar (album)` indexes +# as [bi, polar, ...]. Without the fold, query "bi-polar" hits only +# the album cluster. +_HYPHEN_RUN_RE = re.compile( + r"[A-Za-z][A-Za-z0-9]*(?:-[A-Za-z][A-Za-z0-9]*)+" +) # Kept in sync with FTS5 stopwords in aborist.search.fts5 — both filter # the same set of question-shaping words. "tell" leaking into title-LIKE # search caused "tell me about permacomputer" to pull Tell_(poker), the @@ -123,12 +132,40 @@ _TITLE_STOPWORDS = frozenset( ) +def _hyphen_fold_variants(s: str) -> set[str]: + """For each hyphen-joined run of word tokens in ``s``, emit the + joined-no-hyphen form. Lets retrieval reach indexed forms that + survived the FTS5 hyphen-split asymmetry (Ticket #000007): + + "bi-polar is rare?" -> {"bipolar"} + "high-school co-op" -> {"highschool", "coop"} + "plain query" -> set() + + Stopword and length filters mirror ``_title_query_tokens`` so a + junk fold like "of-the" -> "ofthe" never enters the candidate set. + """ + out: set[str] = set() + for run in _HYPHEN_RUN_RE.findall(s): + joined = run.replace("-", "").lower() + if len(joined) > 1 and joined not in _TITLE_STOPWORDS: + out.add(joined) + return out + + def _title_query_tokens(s: str) -> set[str]: - return { + base = { t.lower() for t in _TITLE_TOKEN_RE.findall(s) if t.lower() not in _TITLE_STOPWORDS and len(t) > 1 } + # Hyphen-fold: additively include joined-no-hyphen variants for + # hyphenated runs in `s`. Symmetric — the function is called on + # both queries and titles, and additive fold preserves existing + # match patterns (e.g. `Coca-Cola history` query keeps {coca, + # cola, cocacola, history} so a `Coca-Cola` title still passes + # title-breadth via {coca, cola, cocacola}). See Ticket #000007. + base |= _hyphen_fold_variants(s) + return base def _rerank_by_title( @@ -158,10 +195,11 @@ def _filter_by_title_relevance( core_match_roots: set[str] | None = None, body_density_check: callable | None = None, phrase_match_roots: set[str] | None = None, + hyphen_fold_anchors: set[str] | None = None, fallback_top_n: int = 5, shards_dir=None, ) -> list: - """Concept-aware relevance filter with four accept paths: + """Concept-aware relevance filter with five accept paths: 1. Title-token overlap (after synonym expansion). Strongest signal. 2. TF-IDF core keyword overlap — `core_match_roots` is a precomputed @@ -182,11 +220,18 @@ def _filter_by_title_relevance( filtered out before it can rerank into the top-K. The upstream phrase route already gates on 4-token-min sequences (see _question_phrases) so false-positive risk is low. + 5. Hyphen-fold anchor — when the question has hyphenated runs + (Ticket #000007), `hyphen_fold_anchors` is the joined-form + set ({"bipolar"} for "bi-polar is rare?"). Title-side stem + overlap with this anchor passes the filter even when the + breadth threshold fails. Rescues non-hyphen titles like + `Bipolar disorder` from rejection while leaving non-hyphen + queries (anchors empty) unaffected. Rivalry exclusion (Intel-titled docs in AMD queries) still applies on every accept path. - If all four accept paths together produce nothing, fall back to the + If all five accept paths together produce nothing, fall back to the top `fallback_top_n` body-BM25 hits — the LLM gets enough context to say "I don't know" rather than fabricating from a single tangential source. @@ -203,6 +248,11 @@ def _filter_by_title_relevance( ) core_roots = core_match_roots or set() phrase_roots = phrase_match_roots or set() + anchor_stems = ( + {_stem_token_for_match(a) for a in hyphen_fold_anchors} + if hyphen_fold_anchors + else set() + ) # Title-overlap breadth threshold scales with query length, mirroring # _body_density_passes: ≤2 tokens require ALL, 3+ require N-1. Without # this, a 2-token query like "supermans girlfriend" admits docs that @@ -234,6 +284,15 @@ def _filter_by_title_relevance( if h.document_root in phrase_roots: kept.append(h) continue + # Accept-path 5: hyphen-fold anchor (Ticket #000007). The + # joined-form variant from a hyphenated query token (e.g. + # "bipolar" from "bi-polar") matching the title's stem set + # is enough signal to pass — rescues `Bipolar disorder` from + # the breadth gate when the query was "bi-polar is rare?". + # Empty anchor set on non-hyphen queries — zero side effect. + if anchor_stems and (anchor_stems & ttokens_stem): + kept.append(h) + continue if body_density_check is not None and body_density_check(h): kept.append(h) continue @@ -243,6 +302,13 @@ def _filter_by_title_relevance( DEFAULT_QUERY_POLICY = { + # Ticket #000007 — query-layer hyphen-fold marker. Folds into + # `governance_policy_hash` so records produced under the new + # rule (`Bipolar disorder` reachable from query "bi-polar") + # cache-split cleanly from pre-fold records. Code applies the + # fold unconditionally; this flag exists to make the policy + # transition observable from the cache_key alone. + "hyphen_fold_v1": True, "system_prompt": ( "You are answering a question using ONLY the sources provided below. " "Each source is delimited by '=== Source: ===' headers.\n\n" @@ -1083,6 +1149,7 @@ def _rerank( core_match_roots: set[str] | None = None, body_density_check: callable | None = None, phrase_match_roots: set[str] | None = None, + hyphen_fold_anchors: set[str] | None = None, shards_dir=None, ) -> list[_Hit]: """Filter off-topic, then layer in body-coverage, title-overlap, and @@ -1104,6 +1171,7 @@ def _rerank( core_match_roots=core_match_roots, body_density_check=body_density_check, phrase_match_roots=phrase_match_roots, + hyphen_fold_anchors=hyphen_fold_anchors, shards_dir=shards_dir, ) hits = _rerank_by_body_coverage(hits, question) @@ -1571,6 +1639,7 @@ def query( core_match_roots=core_match_roots, body_density_check=_body_density_check, phrase_match_roots=phrase_match_roots, + hyphen_fold_anchors=_hyphen_fold_variants(retrieval_query), shards_dir=shards_dir, ) search_ms = _ms_since(t_phase) diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index a33c290..feae207 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -68,6 +68,9 @@ except ImportError: # pragma: no cover DEFAULT_POLICY = { + # Ticket #000007 — query-layer hyphen-fold marker. See + # aborist/qa/query.py:DEFAULT_QUERY_POLICY for rationale. + "hyphen_fold_v1": True, "system_prompt": ( "Answer the user's question based ONLY on the document below. " "For EVERY factual claim, include a verbatim quote from the " diff --git a/docs/TICKETS.md b/docs/TICKETS.md index b8547c8..f32d280 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -57,6 +57,7 @@ Newest first. Update on every open/close. | ID | Title | Status | Opened | Directive | |----------|------------------------------------------------|-----------------------|------------|-----------| +| #000007 | Query-layer hyphen folding | closed · 2026-05-02 | 2026-05-02 | — | | #000006 | Bench-emergent findings (first 72 cycles) | open · awaiting tuning| 2026-05-02 | — | | #000005 | Label ladder migration (POINTER-LINKED → …) | closed · 2026-05-02 | 2026-05-01 | D7 | | #000004 | Directive coverage in bench summary | closed · `acd1f9c` | 2026-05-01 | D8 | @@ -66,4 +67,4 @@ Newest first. Update on every open/close. ## Next ID -`000007` +`000008` diff --git a/docs/tickets/ticket-000007-query-layer-hyphen-fold.md b/docs/tickets/ticket-000007-query-layer-hyphen-fold.md new file mode 100644 index 0000000..f1209bc --- /dev/null +++ b/docs/tickets/ticket-000007-query-layer-hyphen-fold.md @@ -0,0 +1,263 @@ +# Ticket #000007 — Query-layer hyphen folding + +**Status:** closed · landed 2026-05-02 +**Opened:** 2026-05-02 +**Closed:** 2026-05-02 +**Scope:** Add hyphen-folded query variants at retrieval time so +`bi-polar` matches both `Bi-Polar Blues` (hyphen-tokenized) and +`Bipolar disorder` (joined-tokenized) without re-indexing the corpus. +**Audience:** fox + future blackops shifts. +**Hard constraint:** No `canonicalization_version`, `chunking_version`, +or schema bump. Existing `cache_key`s and `providence_cache` records +remain valid. Query-layer only. + +--- + +## 1. Problem statement + +Stock FTS5 with `tokenize = 'porter unicode61'` (`store.py:317`, +`store.py:337`) splits on hyphen at index time and at query time. +`_title_query_tokens` (`query.py:108`) uses `[A-Za-z][A-Za-z0-9]*`, +which also splits on hyphen. Net result: + +- Title `Bi-Polar (Chilli album)` indexes as `[bi, polar, chilli, album]`. +- Title `Bipolar disorder` indexes as `[bipolar, disorder]`. +- Query `bi-polar is rare?` produces qtokens `{bi, polar, rare}`. + +Two non-overlapping token sets. The query lands in the album/ +disambiguation cluster and misses the medical-condition cluster +entirely. + +### 1.1 Reproduction (2026-05-02) + +``` +make query Q="bi-polar is rare?" BURN=1 +``` + +returns EVIDENCE-WARRANTED on a claim grounded in the `BI` +disambiguation page, while the corpus contains every medical- +condition article (`Bipolar disorder`, `Bipolar I disorder`, +`Bipolar II disorder`, `Treatment of bipolar disorder`, +`History of bipolar disorder`, `International Society for Bipolar +Disorders`, `List of people affected by bipolar disorder`, +`Bipolar spectrum`). None retrieved. + +The verifier is honest — given that evidence, the claim is grounded. +The defect is upstream of the verifier in the retrieval tokenizer +asymmetry. + +### 1.2 Why existing knobs don't catch it + +- Stem-aware match (`_stem_token_for_match`) folds possessive/plural + only. +- Concept relations are corpus-derived semantic rivalries, not + mechanical orthography. +- Title-relevance Rule 8 passes — `BI` stems to `bi`, claim text + contains `Bi-polar`. Lexically valid, semantically wrong. +- Phrase-pattern route (n=5/n=6) requires longer questions. A four- + word question is below threshold. +- Deflection sidecar passes — subject-anchor `rare` appears in the + answer. + +--- + +## 2. Design choices + +### 2.1 Option A — query-side dual-emit + hyphen-anchor accept path (RECOMMENDED) + +Two-part query-layer change: + +1. `_title_query_tokens` additively emits joined-no-hyphen variants + for every hyphenated run in its input. Applies symmetrically to + queries AND titles (the function is called on both). Hyphenated + query `bi-polar` produces qtokens `{bi, polar, bipolar, rare}`; + hyphenated title `Bi-Polar Blues` produces ttokens `{bi, polar, + blues, bipolar}`; non-hyphen title `Bipolar disorder` is + unchanged at `{bipolar, disorder}`. Fold is purely additive — no + existing call site loses tokens it relied on (preserves the + `Coca-Cola` style 2-token-overlap match). + +2. `_filter_by_title_relevance` gains accept-path 5 — a + `hyphen_fold_anchors` set (the joined-form-only variants from the + query). Any title whose stem set intersects with the anchor set + passes the filter even when title-breadth fails. This rescues + non-hyphen titles like `Bipolar disorder` (which only matches one + of four qtokens after fold) from the breadth gate. Empty when the + query has no hyphens — zero effect on existing queries. + +Without (2), the breadth filter rejects `Bipolar disorder` (1 of 4 +qtokens overlap, breadth threshold = 3) while accepting `Bi-Polar +Blues` (3 of 4 because hyphenated titles double-count via fold). +The accept-path 5 closes the asymmetry without disrupting the +breadth metric for non-hyphen queries. + +**Pros:** +- No re-index, no version bump, no cache invalidation. +- Symmetric: queries hit hyphenated AND non-hyphenated indexed forms. +- Single helper (`_hyphen_fold_variants`) + single new accept path. +- Zero blast radius for non-hyphen queries. + +**Cons:** +- Body-FTS still uses `aborist/search/fts5.py:_query_tokens` (a + separate function) and stays on AND-mode `bi AND polar AND rare`. + Body-FTS won't pull in `Bipolar disorder` directly. Title-route + + accept-path 5 + rerank carries the recall load. Acceptable: title + route is independent and pulls the right doc into top-K via + documents_fts MATCH on `bipolar`. +- One-way: query `bipolar` does NOT also probe `bi+polar` at the + body route. Different problem; no observed demand. +- Slight noise risk on coincidence collisions: query `high-school` + also probes `highschool`. Empirically rare; bench will measure. + +### 2.2 Option B — title-route only fold, no accept path + +Apply hyphen-folding only at `_search_titles`, not at +`_filter_by_title_relevance`. + +**Cons:** Title-FTS pulls `Bipolar disorder` in but the breadth +filter rejects it (1 of N qtoken overlap < threshold). Half-fix. + +### 2.3 Option C — substring `trigram` tokenizer + +Switch to `trigram` for either or both FTS5 indexes. + +**Cons:** BM25 ranking degrades meaningfully; index size inflates. +Forces full re-index; bumps `canonicalization_version`. Out of scope +per ticket's hard constraint. + +### 2.4 Option D — custom tokenizer with hyphen-fold at index time + +Right answer in the long run, but bumps `canonicalization_version` +and stales every prior cache record. Tracked separately as research. + +### 2.5 Recommendation + +**A.** Cheapest, reversible, no migration. Ships as the immediate +fix; index-time work earns a separate research ticket if and when +bench shows it's worth the version-bump cost. + +--- + +## 3. Implementation sketch + +1. **`aborist/qa/query.py`** — new module-level constant + `_HYPHEN_RUN_RE = re.compile(r"[A-Za-z][A-Za-z0-9]*(?:-[A-Za-z][A-Za-z0-9]*)+")` + plus pure helper: + + ```python + def _hyphen_fold_variants(s: str) -> set[str]: + out: set[str] = set() + for run in _HYPHEN_RUN_RE.findall(s): + joined = run.replace("-", "").lower() + if len(joined) > 1 and joined not in _TITLE_STOPWORDS: + out.add(joined) + return out + ``` + +2. **`_title_query_tokens`** — fold the variants into the returned + set: + + ```python + def _title_query_tokens(s: str) -> set[str]: + base = { + t.lower() + for t in _TITLE_TOKEN_RE.findall(s) + if t.lower() not in _TITLE_STOPWORDS and len(t) > 1 + } + base |= _hyphen_fold_variants(s) + return base + ``` + +3. **`_filter_by_title_relevance`** — accept new optional kwarg + `hyphen_fold_anchors: set[str] | None = None`. Add accept-path 5 + inside the per-hit loop: + + ```python + if hyphen_fold_anchors: + anchor_stems = {_stem_token_for_match(a) for a in hyphen_fold_anchors} + if anchor_stems & ttokens_stem: + kept.append(h) + continue + ``` + +4. **`_rerank`** — pass-through `hyphen_fold_anchors` to + `_filter_by_title_relevance`. + +5. **Caller at `query.py:1543` ish** — compute + `hyphen_fold_anchors = _hyphen_fold_variants(retrieval_query)` + and pass to `_rerank`. + +6. **`DEFAULT_QUERY_POLICY`** (and `runner.py:DEFAULT_POLICY`) — + add `"hyphen_fold_v1": True`. Folds into + `governance_policy_hash` automatically; flipping to `False` + later produces a clean cache split. Code unconditionally applies + the fold; the flag is a policy-hash marker so an auditor can + tell which records were produced under the new rule. + +7. **Tests** (`tests/test_query.py`): + - Unit: `_hyphen_fold_variants("bi-polar is rare?")` returns + `{"bipolar"}`. + - Unit: `_hyphen_fold_variants("plain query")` returns `set()`. + - Unit: `_title_query_tokens("bi-polar is rare?")` returns + `{"bi", "polar", "bipolar", "rare"}` (after stopword strip). + - Integration: synthetic shard with one hyphenated-title doc and + one joined-title doc; query the joined form, both surface in + top-K. + - Integration: query `bi-polar` against a fixture whose only + relevant doc is titled `Bipolar disorder`; confirm it lands in + `result["sources"]`. + +8. **Bench**: add four hyphen-stress questions to + `bench/qa_sweep.py` (`bi-polar is rare?`, `co-operative banking`, + `e-mail history`, `re-enter atmosphere`). Baseline + post-patch + strict-rate. + +--- + +## 4. Out of scope + +- Body-FTS hyphen handling (see §2.1 cons). Title-route carries + retrieval; revisit if a bench fixture shows title-route alone is + insufficient. +- Symmetric joined → split variant (`bipolar` query also probing + `bi-polar`). Demand-driven; revisit if a corpus query exhibits + the inverse failure. +- Index-time hyphen handling. Forbidden by hard constraint; future + research ticket. +- Multilingual / compound / diacritic / numeric-internal classes. + Same research-ticket scope. +- Bumping `canonicalization_version`. Forbidden. + +--- + +## 5. Status + +Closed 2026-05-02. Landed Option A as designed: + +- `aborist/qa/query.py` — `_HYPHEN_RUN_RE` constant + + `_hyphen_fold_variants(s)` helper. `_title_query_tokens(s)` + additively merges the variants. `_filter_by_title_relevance` + gains optional `hyphen_fold_anchors` kwarg + accept-path 5 + (title stem-overlap with anchor stems passes the filter even + when title-breadth fails). `_rerank` threads the kwarg. + `_search_corpus` caller computes + `_hyphen_fold_variants(retrieval_query)` and passes through. +- `aborist/qa/query.py:DEFAULT_QUERY_POLICY` and + `aborist/qa/runner.py:DEFAULT_POLICY` — `hyphen_fold_v1: True` + marker. Folds into `governance_policy_hash` so records + produced under the new rule cache-split cleanly. +- `tests/test_query.py` — 4 new tests: + `test_unit_hyphen_fold_variants_emits_joined_form`, + `test_unit_title_query_tokens_includes_hyphen_fold_additively`, + `test_unit_title_query_tokens_no_hyphen_unchanged`, + `test_integration_hyphenated_query_retrieves_joined_title`. + +Full suite: 760 passed, 34 skipped. Live shard repro for +`bi-polar is rare?` now retrieves `Bipolar disorder` (source +#5) and `Bipolar` disambiguation (source #7); answer cites +both: *"Bi-polar disorder is not rare; it affects approximately +2.8% of the U.S. population, or about 5.7 million adults."* +EVIDENCE-WARRANTED, 2/2 claims, properly grounded — defect closed. + +Bench-stress fixtures and the body-FTS hyphen handling stay +deferred per §4. diff --git a/tests/test_query.py b/tests/test_query.py index a2a4194..03e92e7 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1324,3 +1324,103 @@ def test_functional_long_question_returns_sources(tmp_path): assert any("neurotech" in u for u in uris), ( f"neurotech doc missing from sources {uris}" ) + + +# --------------------------------------------------------------------- +# Ticket #000007 — query-layer hyphen folding +# --------------------------------------------------------------------- + + +def test_unit_hyphen_fold_variants_emits_joined_form(): + """`bi-polar` → {"bipolar"}; multiple hyphen runs emit one + joined variant each; non-hyphenated input emits nothing.""" + from aborist.qa.query import _hyphen_fold_variants + + assert _hyphen_fold_variants("bi-polar is rare?") == {"bipolar"} + assert _hyphen_fold_variants("high-school co-op") == { + "highschool", + "coop", + } + assert _hyphen_fold_variants("plain query no hyphens") == set() + # Single-letter pieces are still folded — `X-ray` → `xray`. + assert "xray" in _hyphen_fold_variants("X-ray imaging") + + +def test_unit_title_query_tokens_includes_hyphen_fold_additively(): + """Hyphen-fold is additive — `bi-polar is rare?` produces both + the split forms (bi, polar) AND the joined form (bipolar).""" + from aborist.qa.query import _title_query_tokens + + toks = _title_query_tokens("bi-polar is rare?") + assert "bi" in toks + assert "polar" in toks + assert "bipolar" in toks + assert "rare" in toks + # `is` is a stopword; should not appear. + assert "is" not in toks + + +def test_unit_title_query_tokens_no_hyphen_unchanged(): + """Non-hyphen input behaves exactly as before — pin that the + fold doesn't add spurious tokens for plain queries.""" + from aborist.qa.query import _title_query_tokens + + assert _title_query_tokens("anarchism political philosophy") == { + "anarchism", + "political", + "philosophy", + } + + +def test_integration_hyphenated_query_retrieves_joined_title(tmp_path): + """Ticket #000007 reproduction: a query with a hyphenated form + must reach a corpus document whose title uses the joined form. + + Corpus contains TWO docs: + - `Bipolar disorder` (joined-form title — what we want) + - `Bi-Polar Blues` (hyphenated-form title — irrelevant album) + + Pre-fix: query `bi-polar is rare?` only retrieved `Bi-Polar Blues` + because FTS5 split both query and album-title on the hyphen + while leaving `Bipolar disorder` indexed as a single token. + + Post-fix: the joined-form variant `bipolar` enters the title-FTS + OR-pool; accept-path 5 in `_filter_by_title_relevance` rescues + `Bipolar disorder` from the breadth gate. Both docs surface. + """ + main_db = tmp_path / "corpus.db" + qa_db = tmp_path / "qa.db" + docs = [ + _doc( + "test://bipolar-disorder", + "Bipolar disorder is a mental health condition. " * 20 + + "Bipolar disorder is rare in the elderly population. " * 5, + ), + _doc( + "test://bi-polar-blues", + "Bi-Polar Blues is a 1995 jazz album. " * 20, + ), + ] + # Override default title (last URI segment) so titles match the + # FTS5 hyphen-asymmetry shape we care about. + docs[0].title = "Bipolar disorder" + docs[1].title = "Bi-Polar Blues" + conn = connect(main_db) + try: + ingest_source(conn, FakeSource(docs)) + finally: + conn.close() + + result = query( + question="bi-polar is rare?", + qa_db=qa_db, + chat_client=StubClient(answer="x"), + model_id="m", + single_db=main_db, + top_k=8, + ) + src_uris = [s["document_uri"] for s in result.get("sources") or []] + # `Bipolar disorder` MUST surface — the whole point of the ticket. + assert any("bipolar-disorder" in u for u in src_uris), ( + f"Ticket #000007 regression: bipolar-disorder not in {src_uris}" + )