#000072: bench-driven diagnosis — port fold-variants stack first
Re-bench legacy vs providence_query on fold themes (accent, hyphen,
honorific, brit, numeral) after the proxy memory fix.
Result (15 question-pairs through Hermes-3-8B): 12 regressions,
2 improvements, 1 tie. Net-negative on these themes, BUT all 12
regressions trace to a single root cause — the 5 fold-variants
helpers (_hyphen, _numeral, _accent, _honorific, _brit) live inside
_title_query_tokens at query.py:288-325 and providence_query
lazy-imports the WRAPPER without lifting the fold helpers.
Same gap manifests two ways:
- Wrong primary (5): Dr Who → pathology; Albert/Ahmed/Alaric the
third/first → wrong articles; Casa Batlló → error
- STRICT → HYBRID on correct primary (7): the verifier's Rule 8
title-overlap check calls the SAME _title_query_tokens —
without folds, "Andre-Marie" (claim) and "André-Marie" (title)
are distinct tokens, overlap fails, audit_mode demotes
Path A v3 surfaces: lift the fold-variants stack to _text_norm.py,
re-export from query.py, drop the lazy-imports in source_roles.py +
retrieval_routes.py. ~250 LOC moved + ~50 LOC import-rewrites,
half-day. Lower risk than v1 (pure code motion, helpers are
identical between paths).
Themes deliberately skipped this round (need their own gates ported
separately): quantifier_subset, metacog_subset, warrant_chain_probe,
es, fr. Re-bench AFTER v3 lands.
Also commits bench/legacy_vs_providence_bench.py + the result JSONL
so the regression set is reproducible.
This commit is contained in:
parent
f7e0a2bd24
commit
618b7846c5
5 changed files with 354 additions and 0 deletions
238
bench/legacy_vs_providence_bench.py
Normal file
238
bench/legacy_vs_providence_bench.py
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
"""Head-to-head: arborist query (default, providence_query path) vs
|
||||
arborist query --legacy (legacy 2000-line query() path).
|
||||
|
||||
For each themed bench subset, sample N questions. Run each through
|
||||
both CLI paths. Compare primary source + audit_mode + n_verified.
|
||||
|
||||
Output: bench/legacy_vs_providence_results/<utc>.jsonl + a stderr
|
||||
summary table per theme showing where the two paths agree / disagree.
|
||||
|
||||
The point: which themed subsets regress when the CLI default flips
|
||||
from legacy to providence_query (#000072 Phase 2 step 3). Drives
|
||||
porting priority for the 12 missing pre/post gates.
|
||||
|
||||
Uses --burn on both paths so cache misses are forced (we measure
|
||||
the fresh-inference quality, not warm-cache replay).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as _dt
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
ARBORIST = REPO / ".venv" / "bin" / "arborist"
|
||||
DEFAULT_SHARDS = Path.home() / ".arborist" / "shards"
|
||||
DEFAULT_QA_DB = Path.home() / ".arborist" / "qa.db"
|
||||
|
||||
# Themed subsets to sample. Each names a gate the legacy pipeline has
|
||||
# that providence_query DOESN'T port yet — regression here points at
|
||||
# which gate to port next.
|
||||
THEMES = [
|
||||
("es", "crosslang sandwich (es→en)"),
|
||||
("accent", "_accent_fold_variants in _title_query_tokens"),
|
||||
("hyphen", "_hyphen_fold_variants"),
|
||||
("honorific", "_honorific_fold_variants"),
|
||||
("brit", "_brit_fold_variants"),
|
||||
("numeral", "_numeral_fold_variants"),
|
||||
("quantifier_subset", "quantifier preflight guard"),
|
||||
("metacog_subset", "metacog preflight"),
|
||||
("warrant_chain_probe", "warrant-resolver post-step"),
|
||||
]
|
||||
|
||||
|
||||
def _load_fixture(path: Path, n: int) -> list[str]:
|
||||
out = []
|
||||
for line in path.read_text().splitlines():
|
||||
s = line.strip()
|
||||
if not s or s.startswith("#"):
|
||||
continue
|
||||
out.append(s)
|
||||
if len(out) >= n:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _run_cli(cmd: list[str], *, timeout_s: int) -> dict:
|
||||
t0 = time.time()
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
cmd, capture_output=True, text=True, timeout=timeout_s,
|
||||
env={**os.environ, "ARBORIST_PROGRESS": "0"},
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"_error": f"timeout {timeout_s}s",
|
||||
"_elapsed_s": timeout_s}
|
||||
dt = time.time() - t0
|
||||
if proc.returncode != 0:
|
||||
return {"_error": f"exit {proc.returncode}",
|
||||
"_stderr": proc.stderr[-300:],
|
||||
"_elapsed_s": round(dt, 2)}
|
||||
try:
|
||||
d = json.loads(proc.stdout)
|
||||
except json.JSONDecodeError as e:
|
||||
return {"_error": f"parse: {e}",
|
||||
"_stdout_head": proc.stdout[:300],
|
||||
"_elapsed_s": round(dt, 2)}
|
||||
d["_elapsed_s"] = round(dt, 2)
|
||||
return d
|
||||
|
||||
|
||||
def _query(q: str, *, shards_dir: Path, qa_db: Path, legacy: bool) -> dict:
|
||||
cmd = [
|
||||
str(ARBORIST), "--shards-dir", str(shards_dir),
|
||||
"query", q,
|
||||
"--qa-db", str(qa_db),
|
||||
"--burn", "--json",
|
||||
"--top-k", "4",
|
||||
]
|
||||
if legacy:
|
||||
cmd.append("--legacy")
|
||||
return _run_cli(cmd, timeout_s=120)
|
||||
|
||||
|
||||
def _primary(r: dict) -> dict:
|
||||
srcs = r.get("sources") or []
|
||||
if not srcs:
|
||||
return {"title": "", "uri": "", "used": False}
|
||||
pri = next(
|
||||
(s for s in srcs if s.get("source_role") == "primary_answer_source"),
|
||||
srcs[0],
|
||||
)
|
||||
return {
|
||||
"title": (pri.get("title") or "")[:55],
|
||||
"uri": pri.get("document_uri") or "",
|
||||
"used": bool(pri.get("used")),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--shards-dir", type=Path, default=DEFAULT_SHARDS)
|
||||
p.add_argument("--qa-db", type=Path,
|
||||
help="qa.db path. Default: $TMP/qa_legacy_vs_providence.db "
|
||||
"(fresh per-run so cache doesn't bleed across themes)")
|
||||
p.add_argument("--n-per-theme", type=int, default=5)
|
||||
p.add_argument("--themes", nargs="*", default=None,
|
||||
help="subset of theme names (default: all)")
|
||||
p.add_argument("--out-dir", type=Path,
|
||||
default=REPO / "bench" / "legacy_vs_providence_results")
|
||||
args = p.parse_args()
|
||||
|
||||
if args.qa_db is None:
|
||||
import tempfile
|
||||
args.qa_db = Path(tempfile.mkdtemp()) / "qa_legacy_vs_providence.db"
|
||||
|
||||
selected = THEMES if args.themes is None else [
|
||||
(k, d) for k, d in THEMES if k in args.themes
|
||||
]
|
||||
args.out_dir.mkdir(parents=True, exist_ok=True)
|
||||
stamp = _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ")
|
||||
out_path = args.out_dir / f"{stamp}.jsonl"
|
||||
|
||||
print(f"# {len(selected)} themes × {args.n_per_theme} questions × 2 paths",
|
||||
file=sys.stderr)
|
||||
print(f"# shards: {args.shards_dir}", file=sys.stderr)
|
||||
print(f"# qa.db: {args.qa_db}", file=sys.stderr)
|
||||
print(f"# llm: $ARBORIST_LLM_ENDPOINT or hermes default", file=sys.stderr)
|
||||
print(f"# out: {out_path}", file=sys.stderr)
|
||||
print(file=sys.stderr)
|
||||
|
||||
theme_summary: dict[str, dict] = {}
|
||||
with open(out_path, "w") as f:
|
||||
for theme_key, theme_desc in selected:
|
||||
fixture = REPO / "bench" / f"qa_questions_{theme_key}.txt"
|
||||
if not fixture.exists():
|
||||
print(f"!! skip {theme_key}: {fixture} missing", file=sys.stderr)
|
||||
continue
|
||||
questions = _load_fixture(fixture, args.n_per_theme)
|
||||
print(f"=== {theme_key:25s} ({theme_desc})", file=sys.stderr)
|
||||
agree = 0
|
||||
disagree = 0
|
||||
audit_better = 0
|
||||
audit_worse = 0
|
||||
for i, q in enumerate(questions, 1):
|
||||
default = _query(
|
||||
q, shards_dir=args.shards_dir, qa_db=args.qa_db,
|
||||
legacy=False,
|
||||
)
|
||||
legacy = _query(
|
||||
q, shards_dir=args.shards_dir, qa_db=args.qa_db,
|
||||
legacy=True,
|
||||
)
|
||||
pri_def = _primary(default)
|
||||
pri_leg = _primary(legacy)
|
||||
same_primary = pri_def["uri"] == pri_leg["uri"]
|
||||
amode_def = default.get("audit_mode") or "?"
|
||||
amode_leg = legacy.get("audit_mode") or "?"
|
||||
# Rank: STRICT > HYBRID > UNGROUNDED for the "did the
|
||||
# path produce a stronger verdict" axis.
|
||||
rank = {"STRICT": 3, "HYBRID": 2, "UNGROUNDED": 1}
|
||||
r_def = rank.get(amode_def, 0)
|
||||
r_leg = rank.get(amode_leg, 0)
|
||||
row = {
|
||||
"theme": theme_key,
|
||||
"question": q,
|
||||
"default": {
|
||||
"audit_mode": amode_def,
|
||||
"n_verified": default.get("n_verified"),
|
||||
"primary": pri_def,
|
||||
"elapsed_s": default.get("_elapsed_s"),
|
||||
"error": default.get("_error"),
|
||||
},
|
||||
"legacy": {
|
||||
"audit_mode": amode_leg,
|
||||
"n_verified": legacy.get("n_verified"),
|
||||
"primary": pri_leg,
|
||||
"elapsed_s": legacy.get("_elapsed_s"),
|
||||
"error": legacy.get("_error"),
|
||||
},
|
||||
"same_primary": same_primary,
|
||||
"audit_delta": r_def - r_leg, # +1 default better
|
||||
}
|
||||
if same_primary:
|
||||
agree += 1
|
||||
else:
|
||||
disagree += 1
|
||||
if r_def > r_leg:
|
||||
audit_better += 1
|
||||
elif r_def < r_leg:
|
||||
audit_worse += 1
|
||||
f.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||||
f.flush()
|
||||
badge = " " if same_primary else "≠"
|
||||
print(f" [{i}/{len(questions)}] {q[:50]:50s}",
|
||||
file=sys.stderr)
|
||||
print(f" default: {amode_def:11s} "
|
||||
f"{pri_def['title']:42s}",
|
||||
file=sys.stderr)
|
||||
print(f" legacy: {amode_leg:11s} "
|
||||
f"{pri_leg['title']:42s} {badge}",
|
||||
file=sys.stderr)
|
||||
theme_summary[theme_key] = {
|
||||
"agree": agree, "disagree": disagree,
|
||||
"audit_better_default": audit_better,
|
||||
"audit_worse_default": audit_worse,
|
||||
"n": len(questions),
|
||||
}
|
||||
print(file=sys.stderr)
|
||||
|
||||
print("=== SUMMARY ===", file=sys.stderr)
|
||||
print(f" {'theme':25s} agree disagree audit↑ audit↓ n",
|
||||
file=sys.stderr)
|
||||
for k, s in theme_summary.items():
|
||||
print(f" {k:25s} {s['agree']:5d} {s['disagree']:8d} "
|
||||
f"{s['audit_better_default']:6d} "
|
||||
f"{s['audit_worse_default']:6d} {s['n']:3d}",
|
||||
file=sys.stderr)
|
||||
print(f" results: {out_path}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
{"theme": "quantifier_subset", "question": "winners of all major sports?", "default": {"audit_mode": "?", "n_verified": null, "primary": {"title": "", "uri": "", "used": false}, "elapsed_s": 120, "error": "timeout 120s"}, "legacy": {"audit_mode": "?", "n_verified": null, "primary": {"title": "", "uri": "", "used": false}, "elapsed_s": 120, "error": "timeout 120s"}, "same_primary": true, "audit_delta": 0}
|
||||
{"theme": "quantifier_subset", "question": "tell me all there is to know about york england?", "default": {"audit_mode": "?", "n_verified": null, "primary": {"title": "", "uri": "", "used": false}, "elapsed_s": 120, "error": "timeout 120s"}, "legacy": {"audit_mode": "?", "n_verified": null, "primary": {"title": "", "uri": "", "used": false}, "elapsed_s": 120, "error": "timeout 120s"}, "same_primary": true, "audit_delta": 0}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
{"theme": "accent", "question": "what is Casa Batllo?", "default": {"audit_mode": "?", "n_verified": null, "primary": {"title": "", "uri": "", "used": false}, "elapsed_s": 48.59, "error": "exit 1"}, "legacy": {"audit_mode": "STRICT", "n_verified": 5, "primary": {"title": "Casa Batlló", "uri": "https://en.wikipedia.org/wiki/Casa_Batlló", "used": false}, "elapsed_s": 11.19, "error": null}, "same_primary": false, "audit_delta": -3}
|
||||
{"theme": "accent", "question": "what is Andre-Marie Ampere?", "default": {"audit_mode": "HYBRID", "n_verified": 8, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": true}, "elapsed_s": 66.09, "error": null}, "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": false}, "elapsed_s": 11.95, "error": null}, "same_primary": true, "audit_delta": 1}
|
||||
{"theme": "accent", "question": "what is Antoni Gaudi?", "default": {"audit_mode": "HYBRID", "n_verified": 10, "primary": {"title": "Antoni Gaudí", "uri": "https://en.wikipedia.org/wiki/Antoni_Gaudí", "used": true}, "elapsed_s": 38.04, "error": null}, "legacy": {"audit_mode": "HYBRID", "n_verified": 5, "primary": {"title": "Antoni Gaudí", "uri": "https://en.wikipedia.org/wiki/Antoni_Gaudí", "used": false}, "elapsed_s": 7.59, "error": null}, "same_primary": true, "audit_delta": 0}
|
||||
{"theme": "hyphen", "question": "what is The Amazing Spider Man (comic book)?", "default": {"audit_mode": "HYBRID", "n_verified": 6, "primary": {"title": "The Amazing Spider-Man (comic book)", "uri": "https://en.wikipedia.org/wiki/The_Amazing_Spider-Man_(comic_book)", "used": true}, "elapsed_s": 74.79, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 6, "primary": {"title": "The Amazing Spider-Man (comic book)", "uri": "https://en.wikipedia.org/wiki/The_Amazing_Spider-Man_(comic_book)", "used": false}, "elapsed_s": 21.04, "error": null}, "same_primary": true, "audit_delta": -1}
|
||||
{"theme": "hyphen", "question": "what is Augustin Jean Fresnel?", "default": {"audit_mode": "HYBRID", "n_verified": 20, "primary": {"title": "Augustin-Jean Fresnel", "uri": "https://en.wikipedia.org/wiki/Augustin-Jean_Fresnel", "used": true}, "elapsed_s": 54.84, "error": null}, "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "primary": {"title": "Augustin-Jean Fresnel", "uri": "https://en.wikipedia.org/wiki/Augustin-Jean_Fresnel", "used": false}, "elapsed_s": 9.01, "error": null}, "same_primary": true, "audit_delta": 1}
|
||||
{"theme": "hyphen", "question": "what is André Marie Ampère?", "default": {"audit_mode": "HYBRID", "n_verified": 10, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": true}, "elapsed_s": 54.53, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 1, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": false}, "elapsed_s": 10.8, "error": null}, "same_primary": true, "audit_delta": -1}
|
||||
{"theme": "honorific", "question": "what is Dr Who?", "default": {"audit_mode": "STRICT", "n_verified": 1, "primary": {"title": "Indian Association of Pathologists and Microbiologists", "uri": "https://en.wikipedia.org/wiki/Indian_Association_of_Pathologists_and_Microbiologists", "used": false}, "elapsed_s": 56.84, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 3, "primary": {"title": "Doctor (Doctor Who)", "uri": "https://en.wikipedia.org/wiki/Doctor_(Doctor_Who)", "used": false}, "elapsed_s": 8.59, "error": null}, "same_primary": false, "audit_delta": 0}
|
||||
{"theme": "honorific", "question": "what is Dr Syn?", "default": {"audit_mode": "HYBRID", "n_verified": 16, "primary": {"title": "Doctor Syn", "uri": "https://en.wikipedia.org/wiki/Doctor_Syn", "used": true}, "elapsed_s": 25.11, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 3, "primary": {"title": "Doctor Syn", "uri": "https://en.wikipedia.org/wiki/Doctor_Syn", "used": false}, "elapsed_s": 7.66, "error": null}, "same_primary": true, "audit_delta": -1}
|
||||
{"theme": "honorific", "question": "what is Dr V64?", "default": {"audit_mode": "HYBRID", "n_verified": 16, "primary": {"title": "Doctor V64", "uri": "https://en.wikipedia.org/wiki/Doctor_V64", "used": true}, "elapsed_s": 62.68, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 10, "primary": {"title": "Doctor V64", "uri": "https://en.wikipedia.org/wiki/Doctor_V64", "used": false}, "elapsed_s": 12.85, "error": null}, "same_primary": true, "audit_delta": -1}
|
||||
{"theme": "brit", "question": "what is Finnish defense Forces?", "default": {"audit_mode": "HYBRID", "n_verified": 20, "primary": {"title": "Finnish Defence Forces", "uri": "https://en.wikipedia.org/wiki/Finnish_Defence_Forces", "used": true}, "elapsed_s": 86.18, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 11, "primary": {"title": "Finnish Defence Forces", "uri": "https://en.wikipedia.org/wiki/Finnish_Defence_Forces", "used": false}, "elapsed_s": 11.4, "error": null}, "same_primary": true, "audit_delta": -1}
|
||||
{"theme": "brit", "question": "what is Hopewell center, Hong Kong?", "default": {"audit_mode": "HYBRID", "n_verified": 22, "primary": {"title": "Hopewell Centre, Hong Kong", "uri": "https://en.wikipedia.org/wiki/Hopewell_Centre,_Hong_Kong", "used": true}, "elapsed_s": 49.59, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 6, "primary": {"title": "Hopewell Centre, Hong Kong", "uri": "https://en.wikipedia.org/wiki/Hopewell_Centre,_Hong_Kong", "used": false}, "elapsed_s": 8.05, "error": null}, "same_primary": true, "audit_delta": -1}
|
||||
{"theme": "brit", "question": "what is labor economics?", "default": {"audit_mode": "HYBRID", "n_verified": 3, "primary": {"title": "Journal of Labor Economics", "uri": "https://en.wikipedia.org/wiki/Journal_of_Labor_Economics", "used": true}, "elapsed_s": 40.81, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 9, "primary": {"title": "Journal of Labor Economics", "uri": "https://en.wikipedia.org/wiki/Journal_of_Labor_Economics", "used": false}, "elapsed_s": 16.12, "error": null}, "same_primary": true, "audit_delta": -1}
|
||||
{"theme": "numeral", "question": "who was Albert the third?", "default": {"audit_mode": "HYBRID", "n_verified": 9, "primary": {"title": "Inside the Third Reich", "uri": "https://en.wikipedia.org/wiki/Inside_the_Third_Reich", "used": false}, "elapsed_s": 73.68, "error": null}, "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "primary": {"title": "The Third", "uri": "https://en.wikipedia.org/wiki/The_Third", "used": false}, "elapsed_s": 12.26, "error": null}, "same_primary": false, "audit_delta": 1}
|
||||
{"theme": "numeral", "question": "who was Ahmed the third?", "default": {"audit_mode": "STRICT", "n_verified": 1, "primary": {"title": "Ahmed Samir", "uri": "https://en.wikipedia.org/wiki/Ahmed_Samir", "used": false}, "elapsed_s": 24.19, "error": null}, "legacy": {"audit_mode": "HYBRID", "n_verified": 1, "primary": {"title": "III", "uri": "https://en.wikipedia.org/wiki/III", "used": false}, "elapsed_s": 5.65, "error": null}, "same_primary": false, "audit_delta": 1}
|
||||
{"theme": "numeral", "question": "who was Alaric the first?", "default": {"audit_mode": "HYBRID", "n_verified": 4, "primary": {"title": "Alaric", "uri": "https://en.wikipedia.org/wiki/Alaric", "used": true}, "elapsed_s": 91.77, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 1, "primary": {"title": "Alaric I", "uri": "https://en.wikipedia.org/wiki/Alaric_I", "used": false}, "elapsed_s": 15.51, "error": null}, "same_primary": false, "audit_delta": -1}
|
||||
|
|
@ -98,6 +98,105 @@ on 2/5 smoke questions. Promoting it to default would degrade the
|
|||
user surface. Fox decision 2026-05-31: defer Phase 2; do NOT promote
|
||||
multi_route to default until A lands.
|
||||
|
||||
## Bench evidence — fold-themes regression 2026-05-31 (Hermes-3-8B)
|
||||
|
||||
`bench/legacy_vs_providence_bench.py --n-per-theme 3 --themes accent
|
||||
hyphen honorific brit numeral` after the default flipped to
|
||||
providence_query (commit 492d1a8). 15 questions × 2 paths (default
|
||||
vs --legacy), result archived at
|
||||
`bench/legacy_vs_providence_results/2026-05-31T22-49-30Z.jsonl`.
|
||||
|
||||
Per-theme:
|
||||
|
||||
| theme | n | agree | ≠ primary | def audit↑ | def audit↓ |
|
||||
|---|---|---|---|---|---|
|
||||
| accent | 3 | 2 | 1 | 1 | 1 |
|
||||
| hyphen | 3 | 3 | 0 | 1 | 2 |
|
||||
| honorific | 3 | 2 | 1 | 0 | 2 |
|
||||
| brit | 3 | 3 | 0 | 0 | 3 |
|
||||
| numeral | 3 | 0 | 3 | 2 | 1 |
|
||||
|
||||
**Two failure modes, ONE root cause: the 5 fold-variants stack.**
|
||||
|
||||
- `_hyphen_fold_variants`, `_numeral_fold_variants`,
|
||||
`_accent_fold_variants`, `_honorific_fold_variants`,
|
||||
`_brit_fold_variants` all live inside `_title_query_tokens`
|
||||
(`query.py:288-325`) — which providence_query lazy-imports for
|
||||
`apply_title_boost` + `filter_by_title_relevance` BUT the
|
||||
underlying fold helpers were never lifted to `_text_norm.py`.
|
||||
|
||||
- Same `_title_query_tokens` is called by legacy's **Rule 8
|
||||
title-overlap verifier check** (`verify.py:_claim_title_overlap`)
|
||||
— without the folds in the default path, the verifier sees
|
||||
`André-Marie Ampère` (title) vs `Andre-Marie Ampere` (claim
|
||||
text) as different tokens, fails the overlap check, demotes
|
||||
STRICT → HYBRID. **Same root cause as the wrong-primary picks**:
|
||||
the fold doesn't fire either at retrieval (wrong doc surfaces)
|
||||
or at verification (right doc demotes).
|
||||
|
||||
**Specific regressions (default vs legacy):**
|
||||
|
||||
Wrong primary (5):
|
||||
- honorific "Dr Who" → "Indian Association of Pathologists" vs
|
||||
"Doctor (Doctor Who)" (Dr↔Doctor fold)
|
||||
- numeral "Albert the third" → "Inside the Third Reich" vs
|
||||
"The Third" (third↔III)
|
||||
- numeral "Ahmed the third" → "Ahmed Samir" vs "III"
|
||||
- numeral "Alaric the first" → "Alaric" vs "Alaric I" (first↔I)
|
||||
- accent "Casa Batllo" → empty/error vs "Casa Batlló"
|
||||
|
||||
STRICT → HYBRID on correct primary (7):
|
||||
- hyphen "André Marie Ampère", "The Amazing Spider Man (comic book)"
|
||||
- brit "Finnish defense Forces", "Hopewell center, Hong Kong",
|
||||
"labor economics"
|
||||
- honorific "Dr Syn", "Dr V64"
|
||||
|
||||
Improvements (2):
|
||||
- accent "Andre-Marie Ampere" → UNGROUNDED→HYBRID (default picks
|
||||
the accented article via FTS5 porter+unicode61 stemmer; legacy
|
||||
missed it without the unaccented query token surviving the regex)
|
||||
- hyphen "Augustin Jean Fresnel" → UNGROUNDED→HYBRID
|
||||
|
||||
**Net for these 5 themes: 12 regressions vs 2 improvements vs 1 tie.**
|
||||
Default flip is net-negative on fold-driven retrieval.
|
||||
|
||||
## Path A v3 — port the fold-variants stack (smallest fix for biggest delta)
|
||||
|
||||
Single concrete task, scoped to one file:
|
||||
|
||||
1. Lift `_TITLE_TOKEN_RE`, `_TITLE_STOPWORDS`, `_TITLE_TOKEN_POLICY`,
|
||||
`_hyphen_fold_variants`, `_numeral_fold_variants`,
|
||||
`_accent_fold_variants`, `_honorific_fold_variants`,
|
||||
`_brit_fold_variants`, `_title_query_tokens`,
|
||||
`_ascii_fold` from `arborist/qa/query.py:115-325` into
|
||||
`arborist/qa/_text_norm.py`.
|
||||
2. Update `arborist/qa/query.py` to re-export under old names
|
||||
(same pattern as the existing stem_for_match move).
|
||||
3. Update `arborist/qa/source_roles.py` + `retrieval_routes.py`
|
||||
to drop the lazy-import-from-query.py dance and import
|
||||
directly from `_text_norm` — kills the lazy-import warts +
|
||||
unblocks any future _text_norm consumer.
|
||||
4. Re-run the same bench to confirm regressions disappear (expect
|
||||
7 STRICT→HYBRID and 4 wrong-primary picks to flip to legacy
|
||||
parity; the 1 accent error needs separate investigation).
|
||||
|
||||
Estimated scope: ~250 LOC moved + ~50 LOC import-rewrites in 3 files.
|
||||
Half-day. Lower risk than Path A v1 because it's pure code motion —
|
||||
the helpers ARE identical between paths; only the import surface
|
||||
changes.
|
||||
|
||||
## Other gates the bench did NOT exercise (still TBD)
|
||||
|
||||
Themes deliberately skipped 2026-05-31 because they'd hit gates the
|
||||
fold port doesn't address:
|
||||
- `quantifier_subset` — broad-quantifier preflight guard
|
||||
- `metacog_subset` — meta-cognition preflight
|
||||
- `warrant_chain_probe` — warrant-resolver post-step
|
||||
- `es` / `fr` — crosslang sandwich translation edges
|
||||
|
||||
Re-bench AFTER Path A v3 lands to isolate which of these still
|
||||
regress; that's the next porting wave.
|
||||
|
||||
## Path A v1 (attempted 2026-05-31, FAILED — reranks made things WORSE)
|
||||
|
||||
Lifted all 5 reranks from query.py into `arborist/qa/retrieval_routes.py`
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue