diff --git a/bench/legacy_vs_providence_results/2026-05-31T23-48-44Z.jsonl b/bench/legacy_vs_providence_results/2026-05-31T23-48-44Z.jsonl new file mode 100644 index 0000000..e55d689 --- /dev/null +++ b/bench/legacy_vs_providence_results/2026-05-31T23-48-44Z.jsonl @@ -0,0 +1,15 @@ +{"theme": "accent", "question": "what is Casa Batllo?", "default": {"audit_mode": "?", "n_verified": null, "primary": {"title": "", "uri": "", "used": false}, "elapsed_s": 43.45, "error": "exit 1"}, "legacy": {"audit_mode": "STRICT", "n_verified": 4, "primary": {"title": "Casa Batlló", "uri": "https://en.wikipedia.org/wiki/Casa_Batlló", "used": false}, "elapsed_s": 12.92, "error": null}, "same_primary": false, "audit_delta": -3} +{"theme": "accent", "question": "what is Andre-Marie Ampere?", "default": {"audit_mode": "HYBRID", "n_verified": 4, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": true}, "elapsed_s": 50.67, "error": null}, "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": false}, "elapsed_s": 6.14, "error": null}, "same_primary": true, "audit_delta": 1} +{"theme": "accent", "question": "what is Antoni Gaudi?", "default": {"audit_mode": "HYBRID", "n_verified": 8, "primary": {"title": "Antoni Gaudí", "uri": "https://en.wikipedia.org/wiki/Antoni_Gaudí", "used": true}, "elapsed_s": 23.6, "error": null}, "legacy": {"audit_mode": "HYBRID", "n_verified": 5, "primary": {"title": "Antoni Gaudí", "uri": "https://en.wikipedia.org/wiki/Antoni_Gaudí", "used": false}, "elapsed_s": 6.3, "error": null}, "same_primary": true, "audit_delta": 0} +{"theme": "hyphen", "question": "what is The Amazing Spider Man (comic book)?", "default": {"audit_mode": "HYBRID", "n_verified": 4, "primary": {"title": "The Amazing Spider-Man (comic book)", "uri": "https://en.wikipedia.org/wiki/The_Amazing_Spider-Man_(comic_book)", "used": true}, "elapsed_s": 83.7, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 6, "primary": {"title": "The Amazing Spider-Man (comic book)", "uri": "https://en.wikipedia.org/wiki/The_Amazing_Spider-Man_(comic_book)", "used": false}, "elapsed_s": 16.64, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "hyphen", "question": "what is Augustin Jean Fresnel?", "default": {"audit_mode": "HYBRID", "n_verified": 10, "primary": {"title": "Augustin-Jean Fresnel", "uri": "https://en.wikipedia.org/wiki/Augustin-Jean_Fresnel", "used": true}, "elapsed_s": 48.68, "error": null}, "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "primary": {"title": "Augustin-Jean Fresnel", "uri": "https://en.wikipedia.org/wiki/Augustin-Jean_Fresnel", "used": false}, "elapsed_s": 10.12, "error": null}, "same_primary": true, "audit_delta": 1} +{"theme": "hyphen", "question": "what is André Marie Ampère?", "default": {"audit_mode": "HYBRID", "n_verified": 4, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": true}, "elapsed_s": 47.45, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 2, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": false}, "elapsed_s": 6.13, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "honorific", "question": "what is Dr Who?", "default": {"audit_mode": "STRICT", "n_verified": 1, "primary": {"title": "Indian Association of Pathologists and Microbiologists", "uri": "https://en.wikipedia.org/wiki/Indian_Association_of_Pathologists_and_Microbiologists", "used": false}, "elapsed_s": 24.85, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 3, "primary": {"title": "Doctor (Doctor Who)", "uri": "https://en.wikipedia.org/wiki/Doctor_(Doctor_Who)", "used": false}, "elapsed_s": 12.14, "error": null}, "same_primary": false, "audit_delta": 0} +{"theme": "honorific", "question": "what is Dr Syn?", "default": {"audit_mode": "HYBRID", "n_verified": 20, "primary": {"title": "Doctor Syn", "uri": "https://en.wikipedia.org/wiki/Doctor_Syn", "used": true}, "elapsed_s": 51.77, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 2, "primary": {"title": "Doctor Syn", "uri": "https://en.wikipedia.org/wiki/Doctor_Syn", "used": false}, "elapsed_s": 11.2, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "honorific", "question": "what is Dr V64?", "default": {"audit_mode": "HYBRID", "n_verified": 16, "primary": {"title": "Doctor V64", "uri": "https://en.wikipedia.org/wiki/Doctor_V64", "used": true}, "elapsed_s": 52.09, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 33, "primary": {"title": "Doctor V64", "uri": "https://en.wikipedia.org/wiki/Doctor_V64", "used": false}, "elapsed_s": 20.89, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "brit", "question": "what is Finnish defense Forces?", "default": {"audit_mode": "HYBRID", "n_verified": 16, "primary": {"title": "Finnish Defence Forces", "uri": "https://en.wikipedia.org/wiki/Finnish_Defence_Forces", "used": true}, "elapsed_s": 58.6, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 11, "primary": {"title": "Finnish Defence Forces", "uri": "https://en.wikipedia.org/wiki/Finnish_Defence_Forces", "used": false}, "elapsed_s": 10.01, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "brit", "question": "what is Hopewell center, Hong Kong?", "default": {"audit_mode": "HYBRID", "n_verified": 20, "primary": {"title": "Hopewell Centre, Hong Kong", "uri": "https://en.wikipedia.org/wiki/Hopewell_Centre,_Hong_Kong", "used": true}, "elapsed_s": 49.19, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 6, "primary": {"title": "Hopewell Centre, Hong Kong", "uri": "https://en.wikipedia.org/wiki/Hopewell_Centre,_Hong_Kong", "used": false}, "elapsed_s": 15.23, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "brit", "question": "what is labor economics?", "default": {"audit_mode": "HYBRID", "n_verified": 2, "primary": {"title": "Journal of Labor Economics", "uri": "https://en.wikipedia.org/wiki/Journal_of_Labor_Economics", "used": true}, "elapsed_s": 49.84, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 6, "primary": {"title": "Journal of Labor Economics", "uri": "https://en.wikipedia.org/wiki/Journal_of_Labor_Economics", "used": false}, "elapsed_s": 15.4, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "numeral", "question": "who was Albert the third?", "default": {"audit_mode": "HYBRID", "n_verified": 2, "primary": {"title": "The Third Clue", "uri": "https://en.wikipedia.org/wiki/The_Third_Clue", "used": false}, "elapsed_s": 67.08, "error": null}, "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "primary": {"title": "The Third", "uri": "https://en.wikipedia.org/wiki/The_Third", "used": false}, "elapsed_s": 17.99, "error": null}, "same_primary": false, "audit_delta": 1} +{"theme": "numeral", "question": "who was Ahmed the third?", "default": {"audit_mode": "STRICT", "n_verified": 1, "primary": {"title": "Ahmed Samir", "uri": "https://en.wikipedia.org/wiki/Ahmed_Samir", "used": false}, "elapsed_s": 27.98, "error": null}, "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "primary": {"title": "III", "uri": "https://en.wikipedia.org/wiki/III", "used": false}, "elapsed_s": 6.21, "error": null}, "same_primary": false, "audit_delta": 2} +{"theme": "numeral", "question": "who was Alaric the first?", "default": {"audit_mode": "HYBRID", "n_verified": 4, "primary": {"title": "Alaric", "uri": "https://en.wikipedia.org/wiki/Alaric", "used": true}, "elapsed_s": 111.21, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 3, "primary": {"title": "Alaric I", "uri": "https://en.wikipedia.org/wiki/Alaric_I", "used": false}, "elapsed_s": 17.49, "error": null}, "same_primary": false, "audit_delta": -1} diff --git a/docs/tickets/ticket-000072-collapse-legacy-query-into-run-query.md b/docs/tickets/ticket-000072-collapse-legacy-query-into-run-query.md index cac2243..f00c495 100644 --- a/docs/tickets/ticket-000072-collapse-legacy-query-into-run-query.md +++ b/docs/tickets/ticket-000072-collapse-legacy-query-into-run-query.md @@ -197,6 +197,116 @@ fold port doesn't address: Re-bench AFTER Path A v3 lands to isolate which of these still regress; that's the next porting wave. +## Path A v3 verdict (2026-05-31, FAILED — wire-up didn't move the bench) + +Two-stage attempt after the original Path A v3 plan: + + - **Stage 1** (commit pending in 971fb58 batch): lifted the + fold-variants stack to `_text_norm.py`, re-exported from + query.py, dropped lazy imports. + - **Stage 2** (same commit): wired `apply_title_boost` to use + `_title_query_tokens` (full 5-fold) instead of + `tokenize_text + numeral_expand`. + +Bench v5 against the same fold-themed fixture (Hermes-3-8B, 5 themes +× 3 questions × 2 paths, result at +`bench/legacy_vs_providence_results/2026-05-31T23-48-44Z.jsonl`): + +| theme | n | agree | ≠ primary | def audit↑ | def audit↓ | +|---|---|---|---|---|---| +| accent | 3 | 2 | 1 | 1 | 1 | +| hyphen | 3 | 3 | 0 | 1 | 2 | +| honorific | 3 | 2 | 1 | 0 | 2 | +| brit | 3 | 3 | 0 | 0 | 3 | +| numeral | 3 | 0 | 3 | 2 | 1 | + +**Identical to the pre-wire-up baseline.** The fold-aware +apply_title_boost can only rerank docs ALREADY in the candidate +set. For "Dr Who" the body BM25 doesn't surface "Doctor (Doctor +Who)" at all — so no fold-aware rerank can promote it. Legacy +surfaces it via the title route (one of its 4 parallel routes). +The default path runs body-only retrieval (multi_route=False); +the right candidate never enters the top-K. + +So Path A v3 was right in diagnosis but wrong in remedy: the +folds need to fire in RETRIEVAL, not just RERANK. The +SqliteShardCorpus.fts_title route (Corpus protocol method, shipped +step 4 / e322bbd) doesn't apply the folds — it just runs SQLite +FTS5 MATCH on `documents_fts` with the raw query. FTS5's porter +stemmer handles plurals/possessives, NOT Dr→Doctor or third→III. + +## The actual story + +Path A across v1, v2 (deferred), v3 stages 1+2 has now proven: + + 1. The fold helpers exist in tree and are importable + (`_text_norm.py`). + 2. They are NOT a drop-in replacement for legacy's retrieval- + time fan-out because the folds need to feed RETRIEVAL token + generation, not just rank-time. + 3. Legacy's 4-route fan-out (body + title-LIKE + phrase + core- + keyword) is the actual mechanism that surfaces fold-needing + docs; lifting the helpers without lifting the fan-out shape + keeps us in the same basin. + 4. Forcing the fan-out on (commit 5fdd573 multi_route flag) + regressed primary picks 2/5 on the smoke fixture — different + basin, also broken. + 5. Adding the 5 reranks on top (Path A v1, commit d099995) + regressed primary picks 4/5 on smoke — yet another basin. + +Net: legacy `query()`'s 2200-line shape is genuinely a tuned +manifold, not a library of multipliers. Reproducing it without +reproducing the topology gives strictly-worse results in every +basin we've measured. + +## What we DID ship from this work + +The Phase 1 foundation (10 commits 9ba6317..20faae0) + Phase 2 +cache wrapper (17b9622) + CLI default flip (492d1a8) + bench +infrastructure (618b784) + Path A v3 (971fb58) are all in tree +and useful EVEN THOUGH the legacy-collapse goal didn't land: + + - `arborist cloud query` + `arborist corpus-query` keep using + the unified pipeline with no regression (already proved 5/5 + on the smoke fixture) + - `providence_query` is a usable standalone cache-aware Q&A + orchestrator (4 tests, cache hit/miss/burn/audit-chain all + pass) for any caller that doesn't need the pre/post gates + - The byte-identity test (1822741) keeps cache_key drift + detectable for any future Phase 2 retry + - The fold helpers live in `_text_norm.py` available to ANY + consumer that wants them at retrieval-token-gen time + - The bench script (`bench/legacy_vs_providence_bench.py`) + drives head-to-head comparisons against any fixture set + +## What this ticket recommends next + +**Stop chasing decomposition.** The 5 Path A attempts have shown +that legacy `query()` doesn't decompose cleanly. Either: + + A. **Leave the default at providence_query.** The dinosaur Q5 + bug fixes itself (proven on cloud), the fold themes regress + (proven here). Accept the trade. Users who hit a fold- + regression bench failure can `--legacy` per-call or set + `ARBORIST_LEGACY_QUERY=1` env-wide. + + B. **Flip the default BACK to legacy.** Treat #000072 Phase 1+2 + work as INFRASTRUCTURE for `arborist cloud query` and + `arborist corpus-query` (where it works) and stop trying to + replace `arborist query`'s legacy path. + + C. **Keep both alive long-term.** `arborist query` = legacy + (rich, slow, well-tuned), `arborist query2` = providence + (fast, lean, opt-in). The dual-CLI ceremony I initially + hedged toward in this session was actually right; the + forcing-function flip was wrong. + +Fox decides. The bench evidence is what it is — no amount of +additional porting is likely to close the regression without +just copying legacy's full 4-route + filter + 5-rerank topology +verbatim into providence_query, at which point we've gained +NOTHING from the refactor. + ## Path A v1 (attempted 2026-05-31, FAILED — reranks made things WORSE) Lifted all 5 reranks from query.py into `arborist/qa/retrieval_routes.py`