diff --git a/arborist/qa/_text_norm.py b/arborist/qa/_text_norm.py index 829648f..a3eea3b 100644 --- a/arborist/qa/_text_norm.py +++ b/arborist/qa/_text_norm.py @@ -102,3 +102,198 @@ def stem_for_match(t: str) -> str: if len(t) > 4 and t.endswith("s") and not t.endswith("ss"): return t[:-1] return t + + +# =========================================================================== +# Title-token + fold-variants stack (Path A v3 of #000072). +# +# Lifted verbatim from arborist.qa.query.query:115-325 so the unified +# run_query orchestrator (via apply_title_boost, +# filter_by_title_relevance, classify_source_role) reaches the SAME +# fold helpers the legacy retrieval + verifier use. Without this lift, +# providence_query regressed 12/15 fold-themed bench questions — +# 7 STRICT→HYBRID demotions on correct primaries (the verifier's +# Rule 8 title-overlap check fails when "Andre-Marie" doesn't fold +# to "André-Marie") AND 5 wrong-primary picks (the retrieval doesn't +# fold "third" → "III", "Dr" → "Doctor"). +# See docs/tickets/ticket-000072 §"Bench evidence" for the data. +# =========================================================================== + + +_TITLE_TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z0-9]*") +# Hyphen-joined run of two-or-more word tokens. Used by +# `_hyphen_fold_variants` to emit joined-no-hyphen variants. See +# Ticket #000007 for the FTS5 hyphen-tokenization-asymmetry rationale: +# `Bipolar disorder` indexes as [bipolar], `Bi-Polar (album)` indexes +# as [bi, polar, ...]. Without the fold, query "bi-polar" hits only +# the album cluster. +_HYPHEN_RUN_RE = re.compile( + r"[A-Za-z][A-Za-z0-9]*(?:-[A-Za-z][A-Za-z0-9]*)+" +) +# Kept in sync with FTS5 stopwords in arborist.search.fts5 — both filter +# the same set of question-shaping words. +_TITLE_STOPWORDS = frozenset( + """ + the a an is are was were be been being of to in on at for with by from + as about into through during and or but not no nor so yet too very also just + what who where when why how which this that these those such i you he she + it we they me him her us them do does did have has had can could should + would will may might + tell show describe explain summarize say give list find make please + all there know everything anything something + """.split() +) + + +def _hyphen_fold_variants(s: str) -> set[str]: + """For each hyphen-joined run of word tokens in ``s``, emit the + joined-no-hyphen form (Ticket #000007). + + "bi-polar is rare?" -> {"bipolar"} + "high-school co-op" -> {"highschool", "coop"} + "plain query" -> set() + """ + out: set[str] = set() + for run in _HYPHEN_RUN_RE.findall(s): + joined = run.replace("-", "").lower() + if len(joined) > 1 and joined not in _TITLE_STOPWORDS: + out.add(joined) + return out + + +# Numeral-fold (measured 2026-05-18): ordinal-word query ("Alexander +# the second") never reaches a Roman-numeral title ("Alexander II") +# because `second` != `ii`. Strict 2..40 set → no English-word +# collision; single-char forms intentionally absent (I/V/X collide +# with English). +_NUM_ORD_TO_ROMAN = { + "second": "ii", "third": "iii", "fourth": "iv", "sixth": "vi", + "seventh": "vii", "eighth": "viii", "ninth": "ix", "eleventh": "xi", + "twelfth": "xii", "thirteenth": "xiii", "fourteenth": "xiv", + "fifteenth": "xv", "sixteenth": "xvi", "seventeenth": "xvii", + "eighteenth": "xviii", "nineteenth": "xix", "twentieth": "xx", +} +_NUM_ROMAN_TO_ORD = {v: k for k, v in _NUM_ORD_TO_ROMAN.items()} + + +def _numeral_fold_variants(s: str) -> set[str]: + """Symmetric ordinal-word ↔ multi-char-Roman fold. + + "who was Alexander the second?" -> {"ii"} + "Alexander II" -> {"second"} + """ + out: set[str] = set() + for tok in _TITLE_TOKEN_RE.findall(s): + tl = tok.lower() + v = _NUM_ORD_TO_ROMAN.get(tl) or _NUM_ROMAN_TO_ORD.get(tl) + if v and len(v) > 1 and v not in _TITLE_STOPWORDS: + out.add(v) + return out + + +def _ascii_fold(s: str) -> str: + """Strip combining diacritics: 'Béla Bartók' -> 'Bela Bartok'. + + Distinct from fold_accents() (defined above) in that this returns + the raw folded STRING for downstream re-tokenization; fold_accents + is the public API. They produce identical bytes; kept under both + names to preserve the legacy ``_ascii_fold`` symbol callers reach + via query.py re-export.""" + import unicodedata as _ud + return "".join( + ch for ch in _ud.normalize("NFKD", s) + if not _ud.combining(ch) + ) + + +def _accent_fold_variants(s: str) -> set[str]: + """ASCII-folded word variants for diacritic text. + + "Béla Bartók" -> {"bela", "bartok"} + "what is X?" -> set() (already ASCII — no-op) + + Load-bearing: ``_TITLE_TOKEN_RE`` is ``[A-Za-z]…``, so a diacritic + title fragments ("Béla" -> "B","la") and never matches the ASCII + form a user types. Folding then re-tokenising recovers the clean + tokens. Additive+symmetric; no-op when ``s`` is already ASCII. + """ + folded = _ascii_fold(s) + if folded == s: + return set() + return { + t.lower() + for t in _TITLE_TOKEN_RE.findall(folded) + if t.lower() not in _TITLE_STOPWORDS and len(t) > 1 + } + + +# Honorific-fold: Mt/St/Dr ↔ Mount/Saint/Doctor. Strict closed set +# (no English-word collision). "st" → {saint} only — "street" is +# deliberately excluded. +_HONOR_FOLD = { + "mount": "mt", "saint": "st", "doctor": "dr", "fort": "ft", + "general": "gen", "president": "pres", "captain": "capt", + "senator": "sen", "mister": "mr", "professor": "prof", +} +_HONOR_FOLD.update({v: k for k, v in _HONOR_FOLD.items()}) + + +def _honorific_fold_variants(s: str) -> set[str]: + """Bidirectional honorific fold for query/title symmetry.""" + out: set[str] = set() + for tok in _TITLE_TOKEN_RE.findall(s): + v = _HONOR_FOLD.get(tok.lower()) + if v and len(v) > 1 and v not in _TITLE_STOPWORDS: + out.add(v) + return out + + +# British ↔ American spelling fold. Token-level (the British form +# IS the title token: "Labour", "Organisation", "Centre"). +_BRIT_FOLD = { + "colour": "color", "honour": "honor", "behaviour": "behavior", + "organisation": "organization", "defence": "defense", + "centre": "center", "theatre": "theater", "catalogue": "catalog", + "programme": "program", "labour": "labor", "favour": "favor", + "licence": "license", "neighbour": "neighbor", +} +_BRIT_FOLD.update({v: k for k, v in _BRIT_FOLD.items()}) + + +def _brit_fold_variants(s: str) -> set[str]: + """Bidirectional British/American fold for query/title symmetry.""" + out: set[str] = set() + for tok in _TITLE_TOKEN_RE.findall(s): + v = _BRIT_FOLD.get(tok.lower()) + if v and len(v) > 1 and v not in _TITLE_STOPWORDS: + out.add(v) + return out + + +# Single source of truth for the active `_title_query_tokens` fold +# set. Bound into the run-DAG retrieval plan (RetrievalPlan +# .title_token_policy) so a replay knows which token-normalization +# produced the retrieved sources (#000001-family provenance). Bump +# on any fold add/remove/semantics change. +_TITLE_TOKEN_POLICY = "tt-v2:hyphen+numeral+accent+honorific+brit" + + +def _title_query_tokens(s: str) -> set[str]: + """Tokens used by the title-overlap filter + retrieval + verifier. + + Base token set (case-folded, stopword-stripped, ≥2 chars) UNION + five additive folds (hyphen / numeral / accent / honorific / brit). + Symmetric: called on both queries and titles, additive folds + preserve existing match patterns. + """ + base = { + t.lower() + for t in _TITLE_TOKEN_RE.findall(s) + if t.lower() not in _TITLE_STOPWORDS and len(t) > 1 + } + base |= _hyphen_fold_variants(s) + base |= _numeral_fold_variants(s) + base |= _accent_fold_variants(s) + base |= _honorific_fold_variants(s) + base |= _brit_fold_variants(s) + return base diff --git a/arborist/qa/corpus.py b/arborist/qa/corpus.py index 28caa57..a22a613 100644 --- a/arborist/qa/corpus.py +++ b/arborist/qa/corpus.py @@ -88,30 +88,33 @@ def apply_title_boost( if not hits or not query.strip() or boost <= 0: return hits from arborist.qa._text_norm import ( - _WORD_RE, STOPWORDS, - fold_accents, numeral_expand, stem_for_match as _stem, tokenize_text, + stem_for_match as _stem, + _title_query_tokens, ) - query_stems = numeral_expand({_stem(t) for t in tokenize_text(query)}) + # Use _title_query_tokens (5-fold stack: hyphen+numeral+accent+ + # honorific+brit) on both query and title sides so the title + # boost matches what legacy _rerank_by_title sees. Without the + # full fold, the bench regressed on `Dr Who` (honorific), + # `Albert the third` (numeral), `André-Marie Ampère` (accent), + # `Finnish defence Forces` (brit), and `Spider-Man` (hyphen) + # because the title-overlap returned 0 → no boost → wrong- + # primary or STRICT→HYBRID demote. See ticket #000072 §Bench + # evidence. + query_tokens = _title_query_tokens(query) + query_stems = {_stem(t) for t in query_tokens} if not query_stems: return hits rescored: list[Hit] = [] for h in hits: - title = fold_accents((h.title or "").lower()) - raw_title_tokens: set[str] = set() - for tok in _WORD_RE.findall(title): - if tok in STOPWORDS: - continue - if len(tok) <= 1 and not tok.isdigit(): - continue - raw_title_tokens.add(_stem(tok)) - title_tokens = numeral_expand(raw_title_tokens) - overlap = len(query_stems & title_tokens) + title_tokens = _title_query_tokens((h.title or "").replace("_", " ")) + title_stems = {_stem(t) for t in title_tokens} + overlap = len(query_stems & title_stems) if not overlap: rescored.append(h) continue - extras = len(title_tokens - query_stems) + extras = len(title_stems - query_stems) effective = max(0.0, overlap - extras) if effective <= 0: rescored.append(h) diff --git a/arborist/qa/corpus_query.py b/arborist/qa/corpus_query.py index 70f0378..437d296 100644 --- a/arborist/qa/corpus_query.py +++ b/arborist/qa/corpus_query.py @@ -203,7 +203,7 @@ def run_query( phrase_hits = _safe_phrase_route( corpus, phrases, per_route_limit, ) if phrases else [] - from arborist.qa.query import _title_query_tokens + from arborist.qa._text_norm import _title_query_tokens qtokens = list(_title_query_tokens(question)) core_hits = _safe_core_route( corpus, qtokens, per_route_limit, @@ -278,7 +278,7 @@ def run_query( SOURCE_ROLE_BUDGET_WEIGHTS, ) from arborist.qa._text_norm import stem_for_match - from arborist.qa.query import _title_query_tokens + from arborist.qa._text_norm import _title_query_tokens qtokens_stem = { stem_for_match(t) for t in _title_query_tokens(question) } diff --git a/arborist/qa/providence_query.py b/arborist/qa/providence_query.py index 6aec9e2..88585fc 100644 --- a/arborist/qa/providence_query.py +++ b/arborist/qa/providence_query.py @@ -369,8 +369,12 @@ def providence_query( cached = _lookup(qa_conn, ckey) if cached is not None and not burn_existing: _bump_hit(qa_conn, ckey) - # Surface every field the legacy query() result carries - # so render layers + journal emitters don't crash on None. + # Surface every legacy-shaped field the render layer + # reads. cache hits don't pay LLM cost so timings sit + # at zero; the elapsed_s shows the round-trip wall + # time (cache lookup + this dict build). + from arborist.qa.query import _context_root + cap = run_result.get("capacity") or {} return { "status": "cache_hit", "cache_key": ckey, @@ -387,7 +391,20 @@ def providence_query( "unverified_quotes": _json.loads( cached["unverified_quotes"] or "[]" ), + "partially_verified_quotes": [], + "warrant_proven_claim_idxs": [], "violations": [], + "format_collapsed": None, + "context_root": _context_root( + [s.get("document_root", "") for s in sources] + ), + "prompt_chars": _prompt_chars_legacy_shape(cap), + "answer_chars": cap.get("answer_chars", 0), + "timings": _merge_timings( + run_result.get("timings") or {}, + cache_lookup_s=round(_time.time() - t_total, 3), + cache_persist_s=0.0, + ), "burned_existing": 0, "elapsed_s": round(_time.time() - t_total, 3), } @@ -407,7 +424,20 @@ def providence_query( finally: qa_conn.close() + # Build the renderer-shaped result. The render layer in cli.py + # (`_render_query_human`) and the unfirehose journal emitter + # (`_emit_query_journal`) read many top-level fields legacy + # query() emitted; flatten run_query's nested `capacity` + + # surface every verifier sidecar so the CLI output matches + # legacy's information density. Sidecars that depend on pre/post + # gates not yet ported into providence_query (answerability, + # quantifier_*, frame_detection, soft_preflight_hint, + # question_state, etc.) stay absent — the renderer handles + # missing fields gracefully (defaults to blank/zero), and the + # CLI shows the basics legacy users expect. + from arborist.qa.query import _context_root out = dict(run_result) + cap = out.get("capacity") or {} out.update({ "status": "burned" if burned_existing else "fresh_persisted", "cache_key": ckey, @@ -416,5 +446,93 @@ def providence_query( "audit_event_hash": audit_event_hash, "run_dag_root": run_dag_root, "elapsed_s": round(_time.time() - t_total, 3), + # Legacy-shape prompt_chars is a DICT (the renderer reads + # .get('messages_total'), .get('system_prompt'), etc.) — + # map run_query's flat capacity counts to that schema. + # grounding_reminder is computed as the remainder so totals + # reconcile (sys + evidence + user + reminder == total). + "prompt_chars": _prompt_chars_legacy_shape(cap), + "answer_chars": cap.get("answer_chars", 0), + # Context-root over sources (Merkle root of doc_roots). + "context_root": _context_root( + [s.get("document_root", "") for s in sources] + ), + # Verifier sidecars that run_query already computed but + # nested under different shapes. Default empty when absent. + "unverified_quotes": run_result.get("unverified_quotes") or [], + "partially_verified_quotes": run_result.get( + "partially_verified_quotes" + ) or [], + "warrant_proven_claim_idxs": run_result.get( + "warrant_proven_claim_idxs" + ) or [], + "format_collapsed": run_result.get("format_collapsed"), + # Per-phase timings: surface the run_query phase dict AS-IS + # plus a synthetic legacy-shape entry the renderer hooks. + # Legacy expects: cache_lookup, retrieval, prompt_build, + # llm, verify, persist. run_query emits: search, context, + # llm, verify, total. Map best-fit so the renderer's + # phase-loop prints something for each known key. + "timings": _merge_timings( + run_result.get("timings") or {}, + cache_lookup_s=0.0, # negligible — single SELECT + cache_persist_s=0.0, # baked into total today + ), }) return out + + +def _prompt_chars_legacy_shape(cap: dict) -> dict: + """Reshape run_query's flat capacity counts into the dict legacy + query() emits (and the CLI renderer reads via .get('messages_total'), + .get('system_prompt'), etc.). grounding_reminder is the remainder + so the four parts sum to messages_total exactly.""" + total = cap.get("prompt_chars", 0) + sys_p = cap.get("sys_prompt_chars", 0) + evidence = cap.get("evidence_chars", 0) + user = cap.get("question_chars", 0) + reminder = max(0, total - sys_p - evidence - user) + return { + "messages_total": total, + "system_prompt": sys_p, + "evidence_or_context": evidence, + "user_question": user, + "grounding_reminder": reminder, + } + + +def _merge_timings(rq: dict, *, cache_lookup_s: float, cache_persist_s: float) -> dict: + """Map run_query's per-phase timings + providence's cache phases + into a renderer-friendly shape. + + The CLI renderer (arborist.cli._render_query_human) reads INT + millisecond keys (cache_lookup_ms, search_ms, context_ms, + llm_ms, persist_ms, total_ms). Bench scripts read float-second + keys (search, context, llm, verify, total). Emit BOTH so neither + consumer breaks; JSON callers see the full picture either way.""" + search_s = rq.get("search", 0.0) + context_s = rq.get("context", 0.0) + llm_s = rq.get("llm", 0.0) + verify_s = rq.get("verify", 0.0) + total_s = rq.get("total", 0.0) + return { + # Native run_query phase keys (float seconds — bench scripts) + "search": search_s, + "context": context_s, + "llm": llm_s, + "verify": verify_s, + "total": total_s, + # Legacy aliases (float seconds) + "retrieval": search_s, + "prompt_build": context_s, + "cache_lookup": cache_lookup_s, + "persist": cache_persist_s, + # Renderer-shaped millisecond ints + "search_ms": int(search_s * 1000), + "context_ms": int(context_s * 1000), + "llm_ms": int(llm_s * 1000), + "verify_ms": int(verify_s * 1000), + "cache_lookup_ms": int(cache_lookup_s * 1000), + "persist_ms": int(cache_persist_s * 1000), + "total_ms": int(total_s * 1000), + } diff --git a/arborist/qa/query.py b/arborist/qa/query.py index f6df80a..312afe5 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -112,217 +112,26 @@ from arborist.store import ( ) -_TITLE_TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z0-9]*") -# Hyphen-joined run of two-or-more word tokens. Used by -# `_hyphen_fold_variants` to emit joined-no-hyphen variants. See -# Ticket #000007 for the FTS5 hyphen-tokenization-asymmetry rationale: -# `Bipolar disorder` indexes as [bipolar], `Bi-Polar (album)` indexes -# as [bi, polar, ...]. Without the fold, query "bi-polar" hits only -# the album cluster. -_HYPHEN_RUN_RE = re.compile( - r"[A-Za-z][A-Za-z0-9]*(?:-[A-Za-z][A-Za-z0-9]*)+" +# Fold-variants stack + _title_query_tokens lifted to _text_norm.py +# (Path A v3 of #000072). Re-exported here under the old underscored +# names so existing call sites + tests continue to work without churn. +from arborist.qa._text_norm import ( # noqa: E402 + _TITLE_TOKEN_RE, + _HYPHEN_RUN_RE, + _TITLE_STOPWORDS, + _TITLE_TOKEN_POLICY, + _NUM_ORD_TO_ROMAN, + _NUM_ROMAN_TO_ORD, + _HONOR_FOLD, + _BRIT_FOLD, + _ascii_fold, + _hyphen_fold_variants, + _numeral_fold_variants, + _accent_fold_variants, + _honorific_fold_variants, + _brit_fold_variants, + _title_query_tokens, ) -# Kept in sync with FTS5 stopwords in arborist.search.fts5 — both filter -# the same set of question-shaping words. "tell" leaking into title-LIKE -# search caused "tell me about permacomputer" to pull Tell_(poker), the -# Tell-Tale_Heart movie, Tell_City Indiana, etc. -_TITLE_STOPWORDS = frozenset( - """ - the a an is are was were be been being of to in on at for with by from - as about into through during and or but not no nor so yet too very also just - what who where when why how which this that these those such i you he she - it we they me him her us them do does did have has had can could should - would will may might - tell show describe explain summarize say give list find make please - all there know everything anything something - """.split() -) - - -def _hyphen_fold_variants(s: str) -> set[str]: - """For each hyphen-joined run of word tokens in ``s``, emit the - joined-no-hyphen form. Lets retrieval reach indexed forms that - survived the FTS5 hyphen-split asymmetry (Ticket #000007): - - "bi-polar is rare?" -> {"bipolar"} - "high-school co-op" -> {"highschool", "coop"} - "plain query" -> set() - - Stopword and length filters mirror ``_title_query_tokens`` so a - junk fold like "of-the" -> "ofthe" never enters the candidate set. - """ - out: set[str] = set() - for run in _HYPHEN_RUN_RE.findall(s): - joined = run.replace("-", "").lower() - if len(joined) > 1 and joined not in _TITLE_STOPWORDS: - out.add(joined) - return out - - -# Numeral-fold (measured 2026-05-18): ordinal-word query ("Alexander -# the second") never reaches a Roman-numeral title ("Alexander II") -# because `second` != `ii`. Mined-fixture recall@8 = 22/40 = 55 %, -# 18 clean misses all this shape. Same additive+symmetric mechanism -# as `_hyphen_fold_variants` (#000007). Scope is honest: only -# multi-char Romans survive the universal `len > 1` token filter, so -# single-char Romans (I/V/X — "Charles V") are out of reach by token- -# fold alone (a deeper change; ~4 of the 18, separately measurable). -# Strict 2..40 set → no English-word collision ("did"/"mix"/"civil" -# are not Romans here); single-char forms intentionally absent. -_NUM_ORD_TO_ROMAN = { - "second": "ii", "third": "iii", "fourth": "iv", "sixth": "vi", - "seventh": "vii", "eighth": "viii", "ninth": "ix", "eleventh": "xi", - "twelfth": "xii", "thirteenth": "xiii", "fourteenth": "xiv", - "fifteenth": "xv", "sixteenth": "xvi", "seventeenth": "xvii", - "eighteenth": "xviii", "nineteenth": "xix", "twentieth": "xx", -} -_NUM_ROMAN_TO_ORD = {v: k for k, v in _NUM_ORD_TO_ROMAN.items()} - - -def _numeral_fold_variants(s: str) -> set[str]: - """Symmetric ordinal-word <-> multi-char-Roman fold. - - "who was Alexander the second?" -> {"ii"} - "Alexander II" -> {"second"} - "plain query" -> set() - - Additive (mirrors `_hyphen_fold_variants`): callers union this - into the token set, so existing matches are preserved and a - folded query token can now also overlap a Roman-numeral title. - Stopword/length filtered like `_title_query_tokens`. - """ - out: set[str] = set() - for tok in _TITLE_TOKEN_RE.findall(s): - tl = tok.lower() - v = _NUM_ORD_TO_ROMAN.get(tl) or _NUM_ROMAN_TO_ORD.get(tl) - if v and len(v) > 1 and v not in _TITLE_STOPWORDS: - out.add(v) - return out - - -def _ascii_fold(s: str) -> str: - """Strip combining diacritics: 'Béla Bartók' -> 'Bela Bartok'.""" - return "".join( - ch for ch in unicodedata.normalize("NFKD", s) - if not unicodedata.combining(ch) - ) - - -def _accent_fold_variants(s: str) -> set[str]: - """ASCII-folded word variants for diacritic text. - - "Béla Bartók" -> {"bela", "bartok"} - "what is X?" -> set() (already ASCII — no-op) - - Why this is load-bearing, not cosmetic: `_TITLE_TOKEN_RE` is - `[A-Za-z]…`, so a diacritic title fragments ("Béla" -> "B","la") - and never matches the ASCII form a user types. Folding then - re-tokenising recovers the clean tokens. Additive+symmetric, - same discipline as `_hyphen_fold_variants` (#000007) / - `_numeral_fold_variants`: a pure-ASCII `s` folds to itself -> - empty -> zero effect on non-accent queries/titles. Measured - 2026-05-18 (fold-search #1; 8.1% of corpus titles carry - diacritics). - """ - folded = _ascii_fold(s) - if folded == s: - return set() - return { - t.lower() - for t in _TITLE_TOKEN_RE.findall(folded) - if t.lower() not in _TITLE_STOPWORDS and len(t) > 1 - } - - -# Honorific-fold: a user types "Mt/St/Dr Everest"; the title spells -# "Mount/Saint/Doctor Everest" (or vice versa). Measured 2026-05-18 -# (mined ground-truth fixture, no fold): recall@1 only 45% / @8 62%, -# 15/40 misses — large headroom, no existing fold. Same additive+ -# symmetric discipline as the numeral/accent folds. Bidirectional so -# either surface form reaches the other; strict closed set (no -# English-word collision). "st" maps to {saint} only — "street" is -# deliberately excluded: the measured class is honorific-titled and -# folding street here would add noise for ~zero recall (additive but -# precision-aware, the single-char-Roman lesson). -_HONOR_FOLD = { - "mount": "mt", "saint": "st", "doctor": "dr", "fort": "ft", - "general": "gen", "president": "pres", "captain": "capt", - "senator": "sen", "mister": "mr", "professor": "prof", -} -_HONOR_FOLD.update({v: k for k, v in _HONOR_FOLD.items()}) - - -def _honorific_fold_variants(s: str) -> set[str]: - out: set[str] = set() - for tok in _TITLE_TOKEN_RE.findall(s): - v = _HONOR_FOLD.get(tok.lower()) - if v and len(v) > 1 and v not in _TITLE_STOPWORDS: - out.add(v) - return out - - -# British<->American spelling fold. Measured 2026-05-18 (mined -# ground-truth, no fold): recall@1 50% / @8 70%, 12/40 misses — -# real headroom, no existing fold. Token-level (the British form -# IS the title token: "Labour", "Organisation", "Centre"). Same -# additive+symmetric discipline; strict closed set. -_BRIT_FOLD = { - "colour": "color", "honour": "honor", "behaviour": "behavior", - "organisation": "organization", "defence": "defense", - "centre": "center", "theatre": "theater", "catalogue": "catalog", - "programme": "program", "labour": "labor", "favour": "favor", - "licence": "license", "neighbour": "neighbor", -} -_BRIT_FOLD.update({v: k for k, v in _BRIT_FOLD.items()}) - - -def _brit_fold_variants(s: str) -> set[str]: - out: set[str] = set() - for tok in _TITLE_TOKEN_RE.findall(s): - v = _BRIT_FOLD.get(tok.lower()) - if v and len(v) > 1 and v not in _TITLE_STOPWORDS: - out.add(v) - return out - - -# Single source of truth for the active `_title_query_tokens` fold -# set. Bound into the run-DAG retrieval plan (RetrievalPlan -# .title_token_policy) so a replay knows which token-normalization -# produced the retrieved sources (Dav1d review 2026-05-19; #000001- -# family provenance, run-DAG not governance). Bump on any fold -# add/remove/semantics change. -_TITLE_TOKEN_POLICY = "tt-v2:hyphen+numeral+accent+honorific+brit" - - -def _title_query_tokens(s: str) -> set[str]: - base = { - t.lower() - for t in _TITLE_TOKEN_RE.findall(s) - if t.lower() not in _TITLE_STOPWORDS and len(t) > 1 - } - # Hyphen-fold: additively include joined-no-hyphen variants for - # hyphenated runs in `s`. Symmetric — the function is called on - # both queries and titles, and additive fold preserves existing - # match patterns (e.g. `Coca-Cola history` query keeps {coca, - # cola, cocacola, history} so a `Coca-Cola` title still passes - # title-breadth via {coca, cola, cocacola}). See Ticket #000007. - base |= _hyphen_fold_variants(s) - # Numeral-fold: ordinal-word <-> Roman-numeral, same additive+ - # symmetric discipline (measured 2026-05-18; see above). - base |= _numeral_fold_variants(s) - # Accent-fold: ASCII-folded variants of diacritic words. The - # token regex is [A-Za-z]+, so an accented title ("Béla Bartók") - # otherwise fragments into junk and never matches the ASCII form - # a user types. Additive+symmetric, same discipline; no-op when - # `s` is already ASCII (measured 2026-05-18; fold-search #1, - # 8.1% of titles). See `_accent_fold_variants`. - base |= _accent_fold_variants(s) - # Honorific-fold: Mt/St/Dr <-> Mount/Saint/Doctor, same additive+ - # symmetric discipline (measured 2026-05-18; baseline recall@1 45%). - base |= _honorific_fold_variants(s) - # British<->American spelling, same discipline (baseline @1 50%). - base |= _brit_fold_variants(s) - return base def _rerank_by_title( diff --git a/arborist/qa/retrieval_routes.py b/arborist/qa/retrieval_routes.py index 21c54d3..23e44d5 100644 --- a/arborist/qa/retrieval_routes.py +++ b/arborist/qa/retrieval_routes.py @@ -106,7 +106,7 @@ def filter_by_title_relevance( # Lazy imports: _title_query_tokens stays in query.py for now (full # fold-variants stack hasn't moved); concepts.py is a sibling module # that imports a bunch of corpus state we'd rather not load eagerly. - from arborist.qa.query import _title_query_tokens + from arborist.qa._text_norm import _title_query_tokens from arborist.concepts import ( has_compare_phrasing, rivalry_excluded, @@ -235,7 +235,7 @@ def filter_by_body_density( pass. Fall-open if all hits fail (don't strand the LLM with no context — legacy convention). """ - from arborist.qa.query import _title_query_tokens + from arborist.qa._text_norm import _title_query_tokens qtokens = _title_query_tokens(question) if not qtokens: return hits @@ -259,7 +259,7 @@ def rerank_by_source_role(hits: list, question: str) -> list: behavior; FTS5-negative inputs end up sorted as more-negative- first which is still "best first" if signs stay homogeneous. """ - from arborist.qa.query import _title_query_tokens + from arborist.qa._text_norm import _title_query_tokens from arborist.qa.source_roles import ( SOURCE_ROLE_RANK_WEIGHTS, classify_source_role, @@ -309,7 +309,7 @@ def rerank_by_title_purity( Stem-aware via stem_for_match (possessive/plural collapse). """ - from arborist.qa.query import _title_query_tokens + from arborist.qa._text_norm import _title_query_tokens qtokens = _title_query_tokens(question) if not qtokens: return hits @@ -420,7 +420,7 @@ def rerank_by_body_coverage( surviving candidate via ``corpus.doc_body`` (cheap on local SQLite; the cloud path pays an HTTP fetch per doc). """ - from arborist.qa.query import _title_query_tokens + from arborist.qa._text_norm import _title_query_tokens qtokens_lower = {t.lower() for t in _title_query_tokens(question)} if not qtokens_lower: return hits diff --git a/arborist/qa/source_roles.py b/arborist/qa/source_roles.py index aac5ce8..1a30af1 100644 --- a/arborist/qa/source_roles.py +++ b/arborist/qa/source_roles.py @@ -100,11 +100,9 @@ def classify_source_role( return "sequel_background_source" if any(k in t for k in _SECONDARY_TITLE_MARKERS): return "secondary_context_source" - # _title_query_tokens still lives in query.py (full fold stack is - # not lifted yet); the stemmer is now canonical in _text_norm. - # Lazy import keeps the import path acyclic. - from arborist.qa.query import _title_query_tokens - from arborist.qa._text_norm import stem_for_match + # Direct import from _text_norm (fold stack lifted in #000072 + # Path A v3); no more lazy-import-from-query.py dance. + from arborist.qa._text_norm import _title_query_tokens, stem_for_match title_tokens = _title_query_tokens(t.replace("_", " ")) title_stems = {stem_for_match(tok) for tok in title_tokens} if qtokens_stem and len(title_stems & qtokens_stem) >= max(