russell@unturf.com
9fc951592d
All 6 swept rerankers (bge-large/base, MiniLM-L-2/L-4/L-6/L-12) false-fire on 4.5–21.8% of real bench-qa STRICT at their candidate-bench fp=0 θ. The smallest θ that yields fp=0 on real STRICT is BELOW the candidate-bench NEG max for every model — meaning at the runtime-safe θ, catch on the 12 candidate-bench NEG = 0/12 across the board. Structural reason: real bench-qa STRICT answers have a much wider score distribution (bge-large STRICT: min -2.20, p10 +2.99, p50 +5.80, p90 +7.54) than the tight contrived candidate-bench POS band. The bottom 10% of legitimate STRICT score below where the candidate-bench NEG cases sat. Distributions overlap heavily; no threshold separates them. This is the §3.2 mirror of #000049 §7 #27's recall-side walk-back — clean candidate-bench → fails real-pipeline gate. Same diagnosis: lexical-candidate selection + cross-encoder scoring + hard threshold doesn't survive real-pipeline heterogeneity. Verdict: relevance reranker CANNOT be promoted to a runtime demotion-only veto on this design. demote_below_score stays null; manifest gains runtime_viability block documenting the negative result + the still-viable advisory soft-signal uses (render-tail, multi-signal advisory, contrastive Q→A vs claim→source delta). bench/scripts/relevance_shadow_sweep.py + bench/results/ relevance-shadow-sweep-pooled808.json committed. Production verifier unchanged; advisory only; no audit_mode effect.
221 lines
No EOL
5.8 KiB
JSON
221 lines
No EOL
5.8 KiB
JSON
{
|
|
"generated_at": "2026-05-13T17:40:47Z",
|
|
"strict_pool_files": [
|
|
"/home/fox/git/arborist/bench/qa_results/2026-05-12T20-53-11Z.jsonl",
|
|
"/home/fox/git/arborist/bench/qa_results/2026-05-12T21-58-58Z.jsonl",
|
|
"/home/fox/git/arborist/bench/qa_results/2026-05-12T22-44-30Z.jsonl"
|
|
],
|
|
"n_strict_pairs": 808,
|
|
"models": {
|
|
"relevance-shadow-v1-bge-reranker-large": {
|
|
"hf_repo": "BAAI/bge-reranker-large",
|
|
"backend": "torch",
|
|
"device": "cpu",
|
|
"n_pairs": 808,
|
|
"infer_seconds": 863.3,
|
|
"score_quantiles_on_strict": {
|
|
"p0": -2.1998,
|
|
"p10": 2.9896,
|
|
"p50": 5.8016,
|
|
"p90": 7.54,
|
|
"p100": 8.2654
|
|
},
|
|
"candidate_bench_theta": 4.643,
|
|
"fp_rate_at_candidate_bench_theta": 0.21782178217821782,
|
|
"smallest_theta_for_zero_fp_on_pool": -2.1998,
|
|
"headroom_table": [
|
|
{
|
|
"pct_strict_below": 0.0,
|
|
"theta_at_that_cut": -2.1998
|
|
},
|
|
{
|
|
"pct_strict_below": 1.0,
|
|
"theta_at_that_cut": -0.2684
|
|
},
|
|
{
|
|
"pct_strict_below": 5.0,
|
|
"theta_at_that_cut": 2.0856
|
|
},
|
|
{
|
|
"pct_strict_below": 10.0,
|
|
"theta_at_that_cut": 2.9896
|
|
}
|
|
]
|
|
},
|
|
"relevance-shadow-v1-bge-reranker-base": {
|
|
"hf_repo": "BAAI/bge-reranker-base",
|
|
"backend": "torch",
|
|
"device": "cpu",
|
|
"n_pairs": 808,
|
|
"infer_seconds": 240.16,
|
|
"score_quantiles_on_strict": {
|
|
"p0": -3.6162,
|
|
"p10": 3.2004,
|
|
"p50": 6.8232,
|
|
"p90": 8.1698,
|
|
"p100": 9.6886
|
|
},
|
|
"candidate_bench_theta": 2.395,
|
|
"fp_rate_at_candidate_bench_theta": 0.0952970297029703,
|
|
"smallest_theta_for_zero_fp_on_pool": -3.6162,
|
|
"headroom_table": [
|
|
{
|
|
"pct_strict_below": 0.0,
|
|
"theta_at_that_cut": -3.6162
|
|
},
|
|
{
|
|
"pct_strict_below": 1.0,
|
|
"theta_at_that_cut": -1.637
|
|
},
|
|
{
|
|
"pct_strict_below": 5.0,
|
|
"theta_at_that_cut": 1.2898
|
|
},
|
|
{
|
|
"pct_strict_below": 10.0,
|
|
"theta_at_that_cut": 3.2004
|
|
}
|
|
]
|
|
},
|
|
"relevance-shadow-v1-ms-marco-MiniLM-L-6-v2": {
|
|
"hf_repo": "cross-encoder/ms-marco-MiniLM-L-6-v2",
|
|
"backend": "torch",
|
|
"device": "cpu",
|
|
"n_pairs": 808,
|
|
"infer_seconds": 37.98,
|
|
"score_quantiles_on_strict": {
|
|
"p0": -7.0632,
|
|
"p10": 6.3215,
|
|
"p50": 8.7989,
|
|
"p90": 10.4243,
|
|
"p100": 11.0746
|
|
},
|
|
"candidate_bench_theta": 6.754,
|
|
"fp_rate_at_candidate_bench_theta": 0.11633663366336634,
|
|
"smallest_theta_for_zero_fp_on_pool": -7.0632,
|
|
"headroom_table": [
|
|
{
|
|
"pct_strict_below": 0.0,
|
|
"theta_at_that_cut": -7.0632
|
|
},
|
|
{
|
|
"pct_strict_below": 1.0,
|
|
"theta_at_that_cut": 4.0411
|
|
},
|
|
{
|
|
"pct_strict_below": 5.0,
|
|
"theta_at_that_cut": 5.2801
|
|
},
|
|
{
|
|
"pct_strict_below": 10.0,
|
|
"theta_at_that_cut": 6.3215
|
|
}
|
|
]
|
|
},
|
|
"relevance-shadow-v1-ms-marco-MiniLM-L-12-v2": {
|
|
"hf_repo": "cross-encoder/ms-marco-MiniLM-L-12-v2",
|
|
"backend": "torch",
|
|
"device": "cpu",
|
|
"n_pairs": 808,
|
|
"infer_seconds": 72.27,
|
|
"score_quantiles_on_strict": {
|
|
"p0": -4.9336,
|
|
"p10": 7.0023,
|
|
"p50": 8.8233,
|
|
"p90": 10.2894,
|
|
"p100": 10.8656
|
|
},
|
|
"candidate_bench_theta": 7.138,
|
|
"fp_rate_at_candidate_bench_theta": 0.11014851485148515,
|
|
"smallest_theta_for_zero_fp_on_pool": -4.9336,
|
|
"headroom_table": [
|
|
{
|
|
"pct_strict_below": 0.0,
|
|
"theta_at_that_cut": -4.9336
|
|
},
|
|
{
|
|
"pct_strict_below": 1.0,
|
|
"theta_at_that_cut": 4.3478
|
|
},
|
|
{
|
|
"pct_strict_below": 5.0,
|
|
"theta_at_that_cut": 5.8487
|
|
},
|
|
{
|
|
"pct_strict_below": 10.0,
|
|
"theta_at_that_cut": 7.0023
|
|
}
|
|
]
|
|
},
|
|
"relevance-shadow-v1-ms-marco-MiniLM-L-4-v2": {
|
|
"hf_repo": "cross-encoder/ms-marco-MiniLM-L-4-v2",
|
|
"backend": "torch",
|
|
"device": "cpu",
|
|
"n_pairs": 808,
|
|
"infer_seconds": 28.22,
|
|
"score_quantiles_on_strict": {
|
|
"p0": -5.5123,
|
|
"p10": 5.8181,
|
|
"p50": 8.3106,
|
|
"p90": 9.9864,
|
|
"p100": 10.7486
|
|
},
|
|
"candidate_bench_theta": 6.736,
|
|
"fp_rate_at_candidate_bench_theta": 0.14975247524752475,
|
|
"smallest_theta_for_zero_fp_on_pool": -5.5123,
|
|
"headroom_table": [
|
|
{
|
|
"pct_strict_below": 0.0,
|
|
"theta_at_that_cut": -5.5123
|
|
},
|
|
{
|
|
"pct_strict_below": 1.0,
|
|
"theta_at_that_cut": 3.9381
|
|
},
|
|
{
|
|
"pct_strict_below": 5.0,
|
|
"theta_at_that_cut": 4.8423
|
|
},
|
|
{
|
|
"pct_strict_below": 10.0,
|
|
"theta_at_that_cut": 5.8181
|
|
}
|
|
]
|
|
},
|
|
"relevance-shadow-v1-ms-marco-MiniLM-L-2-v2": {
|
|
"hf_repo": "cross-encoder/ms-marco-MiniLM-L-2-v2",
|
|
"backend": "torch",
|
|
"device": "cpu",
|
|
"n_pairs": 808,
|
|
"infer_seconds": 11.24,
|
|
"score_quantiles_on_strict": {
|
|
"p0": -6.8554,
|
|
"p10": 4.0983,
|
|
"p50": 7.7531,
|
|
"p90": 9.6393,
|
|
"p100": 10.4551
|
|
},
|
|
"candidate_bench_theta": 3.371,
|
|
"fp_rate_at_candidate_bench_theta": 0.04455445544554455,
|
|
"smallest_theta_for_zero_fp_on_pool": -6.8554,
|
|
"headroom_table": [
|
|
{
|
|
"pct_strict_below": 0.0,
|
|
"theta_at_that_cut": -6.8554
|
|
},
|
|
{
|
|
"pct_strict_below": 1.0,
|
|
"theta_at_that_cut": 1.1027
|
|
},
|
|
{
|
|
"pct_strict_below": 5.0,
|
|
"theta_at_that_cut": 3.5032
|
|
},
|
|
{
|
|
"pct_strict_below": 10.0,
|
|
"theta_at_that_cut": 4.0983
|
|
}
|
|
]
|
|
}
|
|
}
|
|
} |