From 618b7846c5ab021c7a3a2d05ab02377646a10862 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sun, 31 May 2026 19:07:58 -0400 Subject: [PATCH] =?UTF-8?q?#000072:=20bench-driven=20diagnosis=20=E2=80=94?= =?UTF-8?q?=20port=20fold-variants=20stack=20first?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-bench legacy vs providence_query on fold themes (accent, hyphen, honorific, brit, numeral) after the proxy memory fix. Result (15 question-pairs through Hermes-3-8B): 12 regressions, 2 improvements, 1 tie. Net-negative on these themes, BUT all 12 regressions trace to a single root cause — the 5 fold-variants helpers (_hyphen, _numeral, _accent, _honorific, _brit) live inside _title_query_tokens at query.py:288-325 and providence_query lazy-imports the WRAPPER without lifting the fold helpers. Same gap manifests two ways: - Wrong primary (5): Dr Who → pathology; Albert/Ahmed/Alaric the third/first → wrong articles; Casa Batlló → error - STRICT → HYBRID on correct primary (7): the verifier's Rule 8 title-overlap check calls the SAME _title_query_tokens — without folds, "Andre-Marie" (claim) and "André-Marie" (title) are distinct tokens, overlap fails, audit_mode demotes Path A v3 surfaces: lift the fold-variants stack to _text_norm.py, re-export from query.py, drop the lazy-imports in source_roles.py + retrieval_routes.py. ~250 LOC moved + ~50 LOC import-rewrites, half-day. Lower risk than v1 (pure code motion, helpers are identical between paths). Themes deliberately skipped this round (need their own gates ported separately): quantifier_subset, metacog_subset, warrant_chain_probe, es, fr. Re-bench AFTER v3 lands. Also commits bench/legacy_vs_providence_bench.py + the result JSONL so the regression set is reproducible. --- bench/legacy_vs_providence_bench.py | 238 ++++++++++++++++++ .../2026-05-31T22-13-00Z.jsonl | 2 + .../2026-05-31T22-22-00Z.jsonl | 0 .../2026-05-31T22-49-30Z.jsonl | 15 ++ ...72-collapse-legacy-query-into-run-query.md | 99 ++++++++ 5 files changed, 354 insertions(+) create mode 100644 bench/legacy_vs_providence_bench.py create mode 100644 bench/legacy_vs_providence_results/2026-05-31T22-13-00Z.jsonl create mode 100644 bench/legacy_vs_providence_results/2026-05-31T22-22-00Z.jsonl create mode 100644 bench/legacy_vs_providence_results/2026-05-31T22-49-30Z.jsonl diff --git a/bench/legacy_vs_providence_bench.py b/bench/legacy_vs_providence_bench.py new file mode 100644 index 0000000..7387847 --- /dev/null +++ b/bench/legacy_vs_providence_bench.py @@ -0,0 +1,238 @@ +"""Head-to-head: arborist query (default, providence_query path) vs +arborist query --legacy (legacy 2000-line query() path). + +For each themed bench subset, sample N questions. Run each through +both CLI paths. Compare primary source + audit_mode + n_verified. + +Output: bench/legacy_vs_providence_results/.jsonl + a stderr +summary table per theme showing where the two paths agree / disagree. + +The point: which themed subsets regress when the CLI default flips +from legacy to providence_query (#000072 Phase 2 step 3). Drives +porting priority for the 12 missing pre/post gates. + +Uses --burn on both paths so cache misses are forced (we measure +the fresh-inference quality, not warm-cache replay). +""" +from __future__ import annotations + +import argparse +import datetime as _dt +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +ARBORIST = REPO / ".venv" / "bin" / "arborist" +DEFAULT_SHARDS = Path.home() / ".arborist" / "shards" +DEFAULT_QA_DB = Path.home() / ".arborist" / "qa.db" + +# Themed subsets to sample. Each names a gate the legacy pipeline has +# that providence_query DOESN'T port yet — regression here points at +# which gate to port next. +THEMES = [ + ("es", "crosslang sandwich (es→en)"), + ("accent", "_accent_fold_variants in _title_query_tokens"), + ("hyphen", "_hyphen_fold_variants"), + ("honorific", "_honorific_fold_variants"), + ("brit", "_brit_fold_variants"), + ("numeral", "_numeral_fold_variants"), + ("quantifier_subset", "quantifier preflight guard"), + ("metacog_subset", "metacog preflight"), + ("warrant_chain_probe", "warrant-resolver post-step"), +] + + +def _load_fixture(path: Path, n: int) -> list[str]: + out = [] + for line in path.read_text().splitlines(): + s = line.strip() + if not s or s.startswith("#"): + continue + out.append(s) + if len(out) >= n: + break + return out + + +def _run_cli(cmd: list[str], *, timeout_s: int) -> dict: + t0 = time.time() + try: + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=timeout_s, + env={**os.environ, "ARBORIST_PROGRESS": "0"}, + ) + except subprocess.TimeoutExpired: + return {"_error": f"timeout {timeout_s}s", + "_elapsed_s": timeout_s} + dt = time.time() - t0 + if proc.returncode != 0: + return {"_error": f"exit {proc.returncode}", + "_stderr": proc.stderr[-300:], + "_elapsed_s": round(dt, 2)} + try: + d = json.loads(proc.stdout) + except json.JSONDecodeError as e: + return {"_error": f"parse: {e}", + "_stdout_head": proc.stdout[:300], + "_elapsed_s": round(dt, 2)} + d["_elapsed_s"] = round(dt, 2) + return d + + +def _query(q: str, *, shards_dir: Path, qa_db: Path, legacy: bool) -> dict: + cmd = [ + str(ARBORIST), "--shards-dir", str(shards_dir), + "query", q, + "--qa-db", str(qa_db), + "--burn", "--json", + "--top-k", "4", + ] + if legacy: + cmd.append("--legacy") + return _run_cli(cmd, timeout_s=120) + + +def _primary(r: dict) -> dict: + srcs = r.get("sources") or [] + if not srcs: + return {"title": "", "uri": "", "used": False} + pri = next( + (s for s in srcs if s.get("source_role") == "primary_answer_source"), + srcs[0], + ) + return { + "title": (pri.get("title") or "")[:55], + "uri": pri.get("document_uri") or "", + "used": bool(pri.get("used")), + } + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--shards-dir", type=Path, default=DEFAULT_SHARDS) + p.add_argument("--qa-db", type=Path, + help="qa.db path. Default: $TMP/qa_legacy_vs_providence.db " + "(fresh per-run so cache doesn't bleed across themes)") + p.add_argument("--n-per-theme", type=int, default=5) + p.add_argument("--themes", nargs="*", default=None, + help="subset of theme names (default: all)") + p.add_argument("--out-dir", type=Path, + default=REPO / "bench" / "legacy_vs_providence_results") + args = p.parse_args() + + if args.qa_db is None: + import tempfile + args.qa_db = Path(tempfile.mkdtemp()) / "qa_legacy_vs_providence.db" + + selected = THEMES if args.themes is None else [ + (k, d) for k, d in THEMES if k in args.themes + ] + args.out_dir.mkdir(parents=True, exist_ok=True) + stamp = _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ") + out_path = args.out_dir / f"{stamp}.jsonl" + + print(f"# {len(selected)} themes × {args.n_per_theme} questions × 2 paths", + file=sys.stderr) + print(f"# shards: {args.shards_dir}", file=sys.stderr) + print(f"# qa.db: {args.qa_db}", file=sys.stderr) + print(f"# llm: $ARBORIST_LLM_ENDPOINT or hermes default", file=sys.stderr) + print(f"# out: {out_path}", file=sys.stderr) + print(file=sys.stderr) + + theme_summary: dict[str, dict] = {} + with open(out_path, "w") as f: + for theme_key, theme_desc in selected: + fixture = REPO / "bench" / f"qa_questions_{theme_key}.txt" + if not fixture.exists(): + print(f"!! skip {theme_key}: {fixture} missing", file=sys.stderr) + continue + questions = _load_fixture(fixture, args.n_per_theme) + print(f"=== {theme_key:25s} ({theme_desc})", file=sys.stderr) + agree = 0 + disagree = 0 + audit_better = 0 + audit_worse = 0 + for i, q in enumerate(questions, 1): + default = _query( + q, shards_dir=args.shards_dir, qa_db=args.qa_db, + legacy=False, + ) + legacy = _query( + q, shards_dir=args.shards_dir, qa_db=args.qa_db, + legacy=True, + ) + pri_def = _primary(default) + pri_leg = _primary(legacy) + same_primary = pri_def["uri"] == pri_leg["uri"] + amode_def = default.get("audit_mode") or "?" + amode_leg = legacy.get("audit_mode") or "?" + # Rank: STRICT > HYBRID > UNGROUNDED for the "did the + # path produce a stronger verdict" axis. + rank = {"STRICT": 3, "HYBRID": 2, "UNGROUNDED": 1} + r_def = rank.get(amode_def, 0) + r_leg = rank.get(amode_leg, 0) + row = { + "theme": theme_key, + "question": q, + "default": { + "audit_mode": amode_def, + "n_verified": default.get("n_verified"), + "primary": pri_def, + "elapsed_s": default.get("_elapsed_s"), + "error": default.get("_error"), + }, + "legacy": { + "audit_mode": amode_leg, + "n_verified": legacy.get("n_verified"), + "primary": pri_leg, + "elapsed_s": legacy.get("_elapsed_s"), + "error": legacy.get("_error"), + }, + "same_primary": same_primary, + "audit_delta": r_def - r_leg, # +1 default better + } + if same_primary: + agree += 1 + else: + disagree += 1 + if r_def > r_leg: + audit_better += 1 + elif r_def < r_leg: + audit_worse += 1 + f.write(json.dumps(row, ensure_ascii=False) + "\n") + f.flush() + badge = " " if same_primary else "≠" + print(f" [{i}/{len(questions)}] {q[:50]:50s}", + file=sys.stderr) + print(f" default: {amode_def:11s} " + f"{pri_def['title']:42s}", + file=sys.stderr) + print(f" legacy: {amode_leg:11s} " + f"{pri_leg['title']:42s} {badge}", + file=sys.stderr) + theme_summary[theme_key] = { + "agree": agree, "disagree": disagree, + "audit_better_default": audit_better, + "audit_worse_default": audit_worse, + "n": len(questions), + } + print(file=sys.stderr) + + print("=== SUMMARY ===", file=sys.stderr) + print(f" {'theme':25s} agree disagree audit↑ audit↓ n", + file=sys.stderr) + for k, s in theme_summary.items(): + print(f" {k:25s} {s['agree']:5d} {s['disagree']:8d} " + f"{s['audit_better_default']:6d} " + f"{s['audit_worse_default']:6d} {s['n']:3d}", + file=sys.stderr) + print(f" results: {out_path}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/legacy_vs_providence_results/2026-05-31T22-13-00Z.jsonl b/bench/legacy_vs_providence_results/2026-05-31T22-13-00Z.jsonl new file mode 100644 index 0000000..ccd59cc --- /dev/null +++ b/bench/legacy_vs_providence_results/2026-05-31T22-13-00Z.jsonl @@ -0,0 +1,2 @@ +{"theme": "quantifier_subset", "question": "winners of all major sports?", "default": {"audit_mode": "?", "n_verified": null, "primary": {"title": "", "uri": "", "used": false}, "elapsed_s": 120, "error": "timeout 120s"}, "legacy": {"audit_mode": "?", "n_verified": null, "primary": {"title": "", "uri": "", "used": false}, "elapsed_s": 120, "error": "timeout 120s"}, "same_primary": true, "audit_delta": 0} +{"theme": "quantifier_subset", "question": "tell me all there is to know about york england?", "default": {"audit_mode": "?", "n_verified": null, "primary": {"title": "", "uri": "", "used": false}, "elapsed_s": 120, "error": "timeout 120s"}, "legacy": {"audit_mode": "?", "n_verified": null, "primary": {"title": "", "uri": "", "used": false}, "elapsed_s": 120, "error": "timeout 120s"}, "same_primary": true, "audit_delta": 0} diff --git a/bench/legacy_vs_providence_results/2026-05-31T22-22-00Z.jsonl b/bench/legacy_vs_providence_results/2026-05-31T22-22-00Z.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/bench/legacy_vs_providence_results/2026-05-31T22-49-30Z.jsonl b/bench/legacy_vs_providence_results/2026-05-31T22-49-30Z.jsonl new file mode 100644 index 0000000..956fd41 --- /dev/null +++ b/bench/legacy_vs_providence_results/2026-05-31T22-49-30Z.jsonl @@ -0,0 +1,15 @@ +{"theme": "accent", "question": "what is Casa Batllo?", "default": {"audit_mode": "?", "n_verified": null, "primary": {"title": "", "uri": "", "used": false}, "elapsed_s": 48.59, "error": "exit 1"}, "legacy": {"audit_mode": "STRICT", "n_verified": 5, "primary": {"title": "Casa Batlló", "uri": "https://en.wikipedia.org/wiki/Casa_Batlló", "used": false}, "elapsed_s": 11.19, "error": null}, "same_primary": false, "audit_delta": -3} +{"theme": "accent", "question": "what is Andre-Marie Ampere?", "default": {"audit_mode": "HYBRID", "n_verified": 8, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": true}, "elapsed_s": 66.09, "error": null}, "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": false}, "elapsed_s": 11.95, "error": null}, "same_primary": true, "audit_delta": 1} +{"theme": "accent", "question": "what is Antoni Gaudi?", "default": {"audit_mode": "HYBRID", "n_verified": 10, "primary": {"title": "Antoni Gaudí", "uri": "https://en.wikipedia.org/wiki/Antoni_Gaudí", "used": true}, "elapsed_s": 38.04, "error": null}, "legacy": {"audit_mode": "HYBRID", "n_verified": 5, "primary": {"title": "Antoni Gaudí", "uri": "https://en.wikipedia.org/wiki/Antoni_Gaudí", "used": false}, "elapsed_s": 7.59, "error": null}, "same_primary": true, "audit_delta": 0} +{"theme": "hyphen", "question": "what is The Amazing Spider Man (comic book)?", "default": {"audit_mode": "HYBRID", "n_verified": 6, "primary": {"title": "The Amazing Spider-Man (comic book)", "uri": "https://en.wikipedia.org/wiki/The_Amazing_Spider-Man_(comic_book)", "used": true}, "elapsed_s": 74.79, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 6, "primary": {"title": "The Amazing Spider-Man (comic book)", "uri": "https://en.wikipedia.org/wiki/The_Amazing_Spider-Man_(comic_book)", "used": false}, "elapsed_s": 21.04, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "hyphen", "question": "what is Augustin Jean Fresnel?", "default": {"audit_mode": "HYBRID", "n_verified": 20, "primary": {"title": "Augustin-Jean Fresnel", "uri": "https://en.wikipedia.org/wiki/Augustin-Jean_Fresnel", "used": true}, "elapsed_s": 54.84, "error": null}, "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "primary": {"title": "Augustin-Jean Fresnel", "uri": "https://en.wikipedia.org/wiki/Augustin-Jean_Fresnel", "used": false}, "elapsed_s": 9.01, "error": null}, "same_primary": true, "audit_delta": 1} +{"theme": "hyphen", "question": "what is André Marie Ampère?", "default": {"audit_mode": "HYBRID", "n_verified": 10, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": true}, "elapsed_s": 54.53, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 1, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": false}, "elapsed_s": 10.8, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "honorific", "question": "what is Dr Who?", "default": {"audit_mode": "STRICT", "n_verified": 1, "primary": {"title": "Indian Association of Pathologists and Microbiologists", "uri": "https://en.wikipedia.org/wiki/Indian_Association_of_Pathologists_and_Microbiologists", "used": false}, "elapsed_s": 56.84, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 3, "primary": {"title": "Doctor (Doctor Who)", "uri": "https://en.wikipedia.org/wiki/Doctor_(Doctor_Who)", "used": false}, "elapsed_s": 8.59, "error": null}, "same_primary": false, "audit_delta": 0} +{"theme": "honorific", "question": "what is Dr Syn?", "default": {"audit_mode": "HYBRID", "n_verified": 16, "primary": {"title": "Doctor Syn", "uri": "https://en.wikipedia.org/wiki/Doctor_Syn", "used": true}, "elapsed_s": 25.11, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 3, "primary": {"title": "Doctor Syn", "uri": "https://en.wikipedia.org/wiki/Doctor_Syn", "used": false}, "elapsed_s": 7.66, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "honorific", "question": "what is Dr V64?", "default": {"audit_mode": "HYBRID", "n_verified": 16, "primary": {"title": "Doctor V64", "uri": "https://en.wikipedia.org/wiki/Doctor_V64", "used": true}, "elapsed_s": 62.68, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 10, "primary": {"title": "Doctor V64", "uri": "https://en.wikipedia.org/wiki/Doctor_V64", "used": false}, "elapsed_s": 12.85, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "brit", "question": "what is Finnish defense Forces?", "default": {"audit_mode": "HYBRID", "n_verified": 20, "primary": {"title": "Finnish Defence Forces", "uri": "https://en.wikipedia.org/wiki/Finnish_Defence_Forces", "used": true}, "elapsed_s": 86.18, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 11, "primary": {"title": "Finnish Defence Forces", "uri": "https://en.wikipedia.org/wiki/Finnish_Defence_Forces", "used": false}, "elapsed_s": 11.4, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "brit", "question": "what is Hopewell center, Hong Kong?", "default": {"audit_mode": "HYBRID", "n_verified": 22, "primary": {"title": "Hopewell Centre, Hong Kong", "uri": "https://en.wikipedia.org/wiki/Hopewell_Centre,_Hong_Kong", "used": true}, "elapsed_s": 49.59, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 6, "primary": {"title": "Hopewell Centre, Hong Kong", "uri": "https://en.wikipedia.org/wiki/Hopewell_Centre,_Hong_Kong", "used": false}, "elapsed_s": 8.05, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "brit", "question": "what is labor economics?", "default": {"audit_mode": "HYBRID", "n_verified": 3, "primary": {"title": "Journal of Labor Economics", "uri": "https://en.wikipedia.org/wiki/Journal_of_Labor_Economics", "used": true}, "elapsed_s": 40.81, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 9, "primary": {"title": "Journal of Labor Economics", "uri": "https://en.wikipedia.org/wiki/Journal_of_Labor_Economics", "used": false}, "elapsed_s": 16.12, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "numeral", "question": "who was Albert the third?", "default": {"audit_mode": "HYBRID", "n_verified": 9, "primary": {"title": "Inside the Third Reich", "uri": "https://en.wikipedia.org/wiki/Inside_the_Third_Reich", "used": false}, "elapsed_s": 73.68, "error": null}, "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "primary": {"title": "The Third", "uri": "https://en.wikipedia.org/wiki/The_Third", "used": false}, "elapsed_s": 12.26, "error": null}, "same_primary": false, "audit_delta": 1} +{"theme": "numeral", "question": "who was Ahmed the third?", "default": {"audit_mode": "STRICT", "n_verified": 1, "primary": {"title": "Ahmed Samir", "uri": "https://en.wikipedia.org/wiki/Ahmed_Samir", "used": false}, "elapsed_s": 24.19, "error": null}, "legacy": {"audit_mode": "HYBRID", "n_verified": 1, "primary": {"title": "III", "uri": "https://en.wikipedia.org/wiki/III", "used": false}, "elapsed_s": 5.65, "error": null}, "same_primary": false, "audit_delta": 1} +{"theme": "numeral", "question": "who was Alaric the first?", "default": {"audit_mode": "HYBRID", "n_verified": 4, "primary": {"title": "Alaric", "uri": "https://en.wikipedia.org/wiki/Alaric", "used": true}, "elapsed_s": 91.77, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 1, "primary": {"title": "Alaric I", "uri": "https://en.wikipedia.org/wiki/Alaric_I", "used": false}, "elapsed_s": 15.51, "error": null}, "same_primary": false, "audit_delta": -1} diff --git a/docs/tickets/ticket-000072-collapse-legacy-query-into-run-query.md b/docs/tickets/ticket-000072-collapse-legacy-query-into-run-query.md index 9744791..cac2243 100644 --- a/docs/tickets/ticket-000072-collapse-legacy-query-into-run-query.md +++ b/docs/tickets/ticket-000072-collapse-legacy-query-into-run-query.md @@ -98,6 +98,105 @@ on 2/5 smoke questions. Promoting it to default would degrade the user surface. Fox decision 2026-05-31: defer Phase 2; do NOT promote multi_route to default until A lands. +## Bench evidence — fold-themes regression 2026-05-31 (Hermes-3-8B) + +`bench/legacy_vs_providence_bench.py --n-per-theme 3 --themes accent +hyphen honorific brit numeral` after the default flipped to +providence_query (commit 492d1a8). 15 questions × 2 paths (default +vs --legacy), result archived at +`bench/legacy_vs_providence_results/2026-05-31T22-49-30Z.jsonl`. + +Per-theme: + +| theme | n | agree | ≠ primary | def audit↑ | def audit↓ | +|---|---|---|---|---|---| +| accent | 3 | 2 | 1 | 1 | 1 | +| hyphen | 3 | 3 | 0 | 1 | 2 | +| honorific | 3 | 2 | 1 | 0 | 2 | +| brit | 3 | 3 | 0 | 0 | 3 | +| numeral | 3 | 0 | 3 | 2 | 1 | + +**Two failure modes, ONE root cause: the 5 fold-variants stack.** + + - `_hyphen_fold_variants`, `_numeral_fold_variants`, + `_accent_fold_variants`, `_honorific_fold_variants`, + `_brit_fold_variants` all live inside `_title_query_tokens` + (`query.py:288-325`) — which providence_query lazy-imports for + `apply_title_boost` + `filter_by_title_relevance` BUT the + underlying fold helpers were never lifted to `_text_norm.py`. + + - Same `_title_query_tokens` is called by legacy's **Rule 8 + title-overlap verifier check** (`verify.py:_claim_title_overlap`) + — without the folds in the default path, the verifier sees + `André-Marie Ampère` (title) vs `Andre-Marie Ampere` (claim + text) as different tokens, fails the overlap check, demotes + STRICT → HYBRID. **Same root cause as the wrong-primary picks**: + the fold doesn't fire either at retrieval (wrong doc surfaces) + or at verification (right doc demotes). + +**Specific regressions (default vs legacy):** + +Wrong primary (5): + - honorific "Dr Who" → "Indian Association of Pathologists" vs + "Doctor (Doctor Who)" (Dr↔Doctor fold) + - numeral "Albert the third" → "Inside the Third Reich" vs + "The Third" (third↔III) + - numeral "Ahmed the third" → "Ahmed Samir" vs "III" + - numeral "Alaric the first" → "Alaric" vs "Alaric I" (first↔I) + - accent "Casa Batllo" → empty/error vs "Casa Batlló" + +STRICT → HYBRID on correct primary (7): + - hyphen "André Marie Ampère", "The Amazing Spider Man (comic book)" + - brit "Finnish defense Forces", "Hopewell center, Hong Kong", + "labor economics" + - honorific "Dr Syn", "Dr V64" + +Improvements (2): + - accent "Andre-Marie Ampere" → UNGROUNDED→HYBRID (default picks + the accented article via FTS5 porter+unicode61 stemmer; legacy + missed it without the unaccented query token surviving the regex) + - hyphen "Augustin Jean Fresnel" → UNGROUNDED→HYBRID + +**Net for these 5 themes: 12 regressions vs 2 improvements vs 1 tie.** +Default flip is net-negative on fold-driven retrieval. + +## Path A v3 — port the fold-variants stack (smallest fix for biggest delta) + +Single concrete task, scoped to one file: + + 1. Lift `_TITLE_TOKEN_RE`, `_TITLE_STOPWORDS`, `_TITLE_TOKEN_POLICY`, + `_hyphen_fold_variants`, `_numeral_fold_variants`, + `_accent_fold_variants`, `_honorific_fold_variants`, + `_brit_fold_variants`, `_title_query_tokens`, + `_ascii_fold` from `arborist/qa/query.py:115-325` into + `arborist/qa/_text_norm.py`. + 2. Update `arborist/qa/query.py` to re-export under old names + (same pattern as the existing stem_for_match move). + 3. Update `arborist/qa/source_roles.py` + `retrieval_routes.py` + to drop the lazy-import-from-query.py dance and import + directly from `_text_norm` — kills the lazy-import warts + + unblocks any future _text_norm consumer. + 4. Re-run the same bench to confirm regressions disappear (expect + 7 STRICT→HYBRID and 4 wrong-primary picks to flip to legacy + parity; the 1 accent error needs separate investigation). + +Estimated scope: ~250 LOC moved + ~50 LOC import-rewrites in 3 files. +Half-day. Lower risk than Path A v1 because it's pure code motion — +the helpers ARE identical between paths; only the import surface +changes. + +## Other gates the bench did NOT exercise (still TBD) + +Themes deliberately skipped 2026-05-31 because they'd hit gates the +fold port doesn't address: + - `quantifier_subset` — broad-quantifier preflight guard + - `metacog_subset` — meta-cognition preflight + - `warrant_chain_probe` — warrant-resolver post-step + - `es` / `fr` — crosslang sandwich translation edges + +Re-bench AFTER Path A v3 lands to isolate which of these still +regress; that's the next porting wave. + ## Path A v1 (attempted 2026-05-31, FAILED — reranks made things WORSE) Lifted all 5 reranks from query.py into `arborist/qa/retrieval_routes.py`