engineering/arborist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

#000052 §3.2.1 round-2: add L-2 (~30MB) + L-4 (~50MB) + bge-reranker-large (~560MB) — extend the size spectrum from 3.5× to ~20× to test 'specific checkpoint > size' more thoroughly

Browse source
This commit is contained in:
russell@unturf.com 2026-05-13 09:58:42 -04:00
parent aeeef603a1
commit 9ad224695e
No known key found for this signature in database
1 changed files with 6 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

9
arborist/qa/relevance/manifest.json
View file

@ -12,9 +12,12 @@
"demote_below_score": null,
"demote_below_score_comment": "the demote threshold MUST be set by a shadow sweep against bench-qa traffic — DO NOT default to a literature number. The #000049 §7 #18→#27 arc proved repeatedly that clean-eval thresholds don't transfer to real-pipeline data; recommended_operating_point is left null until a candidate-bench + a bench-qa shadow sweep produces a θ with the FP discipline §3.2 inherits from #000049 (zero false-demote on the pooled-808 STRICT denominator at the chosen θ).",
"alternates": [
{"relevance_model_version": "relevance-shadow-v1-bge-reranker-base", "hf_repo": "BAAI/bge-reranker-base", "license": "MIT", "approx_mb": 280, "candidate_bench_margin": 3.076, "note": "BAAI general-purpose reranker; second-best candidate-bench margin after electra-base"},
{"relevance_model_version": "relevance-shadow-v1-ms-marco-MiniLM-L-6-v2", "hf_repo": "cross-encoder/ms-marco-MiniLM-L-6-v2", "license": "Apache-2.0", "approx_mb": 80, "candidate_bench_margin": 3.053, "note": "the original primary before §3.2.1 candidate-bench; smallest/cheapest, comparable margin to BGE on the 26-pair candidate-bench"},
{"relevance_model_version": "relevance-shadow-v1-ms-marco-MiniLM-L-12-v2", "hf_repo": "cross-encoder/ms-marco-MiniLM-L-12-v2", "license": "Apache-2.0", "approx_mb": 130, "candidate_bench_margin": 2.663, "note": "deeper L-12 MiniLM; worst margin of the 4 on the candidate-bench — deeper ≠ better here, same pattern as #000049 §7 #24's 'bart-large worse than deberta-large'"}
{"relevance_model_version": "relevance-shadow-v1-bge-reranker-large", "hf_repo": "BAAI/bge-reranker-large", "license": "MIT", "approx_mb": 560, "note": "LARGE end of the spectrum — added §3.2.1 round-2 (does bigger BGE beat bge-base?)"},
{"relevance_model_version": "relevance-shadow-v1-bge-reranker-base", "hf_repo": "BAAI/bge-reranker-base", "license": "MIT", "approx_mb": 280, "candidate_bench_margin": 3.076, "note": "BAAI general-purpose reranker; second-best candidate-bench margin after electra-base (round 1)"},
{"relevance_model_version": "relevance-shadow-v1-ms-marco-MiniLM-L-6-v2", "hf_repo": "cross-encoder/ms-marco-MiniLM-L-6-v2", "license": "Apache-2.0", "approx_mb": 80, "candidate_bench_margin": 3.053, "note": "the original primary before §3.2.1 candidate-bench"},
{"relevance_model_version": "relevance-shadow-v1-ms-marco-MiniLM-L-12-v2", "hf_repo": "cross-encoder/ms-marco-MiniLM-L-12-v2", "license": "Apache-2.0", "approx_mb": 130, "candidate_bench_margin": 2.663, "note": "deeper L-12 MiniLM; round-1 worst — deeper ≠ better here"},
{"relevance_model_version": "relevance-shadow-v1-ms-marco-MiniLM-L-4-v2", "hf_repo": "cross-encoder/ms-marco-MiniLM-L-4-v2", "license": "Apache-2.0", "approx_mb": 50, "note": "SMALL end — added §3.2.1 round-2 (does L-4 still separate the candidate-bench?)"},
{"relevance_model_version": "relevance-shadow-v1-ms-marco-MiniLM-L-2-v2", "hf_repo": "cross-encoder/ms-marco-MiniLM-L-2-v2", "license": "Apache-2.0", "approx_mb": 30, "note": "TINY end — added §3.2.1 round-2 (the smallest reranker; floor of the size spectrum)"}
],
"candidate_bench_results_2026_05_13": {
"fixtures": "bench/fixtures/5f/relevance-aboutness-v1.jsonl (14 POS + 12 NEG)",