From 9ad224695e712011ca75536665ba9b598e6726ea Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 13 May 2026 09:58:42 -0400 Subject: [PATCH] =?UTF-8?q?#000052=20=C2=A73.2.1=20round-2:=20add=20L-2=20?= =?UTF-8?q?(~30MB)=20+=20L-4=20(~50MB)=20+=20bge-reranker-large=20(~560MB)?= =?UTF-8?q?=20=E2=80=94=20extend=20the=20size=20spectrum=20from=203.5?= =?UTF-8?q?=C3=97=20to=20~20=C3=97=20to=20test=20'specific=20checkpoint=20?= =?UTF-8?q?>=20size'=20more=20thoroughly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- arborist/qa/relevance/manifest.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/arborist/qa/relevance/manifest.json b/arborist/qa/relevance/manifest.json index 329dbe1..7e1922f 100644 --- a/arborist/qa/relevance/manifest.json +++ b/arborist/qa/relevance/manifest.json @@ -12,9 +12,12 @@ "demote_below_score": null, "demote_below_score_comment": "the demote threshold MUST be set by a shadow sweep against bench-qa traffic — DO NOT default to a literature number. The #000049 §7 #18→#27 arc proved repeatedly that clean-eval thresholds don't transfer to real-pipeline data; recommended_operating_point is left null until a candidate-bench + a bench-qa shadow sweep produces a θ with the FP discipline §3.2 inherits from #000049 (zero false-demote on the pooled-808 STRICT denominator at the chosen θ).", "alternates": [ - {"relevance_model_version": "relevance-shadow-v1-bge-reranker-base", "hf_repo": "BAAI/bge-reranker-base", "license": "MIT", "approx_mb": 280, "candidate_bench_margin": 3.076, "note": "BAAI general-purpose reranker; second-best candidate-bench margin after electra-base"}, - {"relevance_model_version": "relevance-shadow-v1-ms-marco-MiniLM-L-6-v2", "hf_repo": "cross-encoder/ms-marco-MiniLM-L-6-v2", "license": "Apache-2.0", "approx_mb": 80, "candidate_bench_margin": 3.053, "note": "the original primary before §3.2.1 candidate-bench; smallest/cheapest, comparable margin to BGE on the 26-pair candidate-bench"}, - {"relevance_model_version": "relevance-shadow-v1-ms-marco-MiniLM-L-12-v2", "hf_repo": "cross-encoder/ms-marco-MiniLM-L-12-v2", "license": "Apache-2.0", "approx_mb": 130, "candidate_bench_margin": 2.663, "note": "deeper L-12 MiniLM; worst margin of the 4 on the candidate-bench — deeper ≠ better here, same pattern as #000049 §7 #24's 'bart-large worse than deberta-large'"} + {"relevance_model_version": "relevance-shadow-v1-bge-reranker-large", "hf_repo": "BAAI/bge-reranker-large", "license": "MIT", "approx_mb": 560, "note": "LARGE end of the spectrum — added §3.2.1 round-2 (does bigger BGE beat bge-base?)"}, + {"relevance_model_version": "relevance-shadow-v1-bge-reranker-base", "hf_repo": "BAAI/bge-reranker-base", "license": "MIT", "approx_mb": 280, "candidate_bench_margin": 3.076, "note": "BAAI general-purpose reranker; second-best candidate-bench margin after electra-base (round 1)"}, + {"relevance_model_version": "relevance-shadow-v1-ms-marco-MiniLM-L-6-v2", "hf_repo": "cross-encoder/ms-marco-MiniLM-L-6-v2", "license": "Apache-2.0", "approx_mb": 80, "candidate_bench_margin": 3.053, "note": "the original primary before §3.2.1 candidate-bench"}, + {"relevance_model_version": "relevance-shadow-v1-ms-marco-MiniLM-L-12-v2", "hf_repo": "cross-encoder/ms-marco-MiniLM-L-12-v2", "license": "Apache-2.0", "approx_mb": 130, "candidate_bench_margin": 2.663, "note": "deeper L-12 MiniLM; round-1 worst — deeper ≠ better here"}, + {"relevance_model_version": "relevance-shadow-v1-ms-marco-MiniLM-L-4-v2", "hf_repo": "cross-encoder/ms-marco-MiniLM-L-4-v2", "license": "Apache-2.0", "approx_mb": 50, "note": "SMALL end — added §3.2.1 round-2 (does L-4 still separate the candidate-bench?)"}, + {"relevance_model_version": "relevance-shadow-v1-ms-marco-MiniLM-L-2-v2", "hf_repo": "cross-encoder/ms-marco-MiniLM-L-2-v2", "license": "Apache-2.0", "approx_mb": 30, "note": "TINY end — added §3.2.1 round-2 (the smallest reranker; floor of the size spectrum)"} ], "candidate_bench_results_2026_05_13": { "fixtures": "bench/fixtures/5f/relevance-aboutness-v1.jsonl (14 POS + 12 NEG)",