russell@unturf.com
da5af2dfaa
7 models on the 4090 against the 26-pair candidate-bench: bge-reranker-large 560MB +6.285 ← new primary (most cushion) ms-marco-electra-base 110MB +5.349 (cost alternate — 5× smaller, 87% the margin) MiniLM-L-4-v2 50MB +3.187 bge-reranker-base 280MB +3.076 MiniLM-L-6-v2 80MB +3.053 (was primary before §3.2.1) MiniLM-L-12-v2 130MB +2.663 (deeper ≠ better — same pattern as §7 #24) MiniLM-L-2-v2 30MB -1.970 ← capacity floor (NOT separable, 10/12 catch) Findings: - Biggest is best WITHIN a family (bge-large > bge-base, 2× size → 2× margin). - Across families: electra-base (110MB) beats bge-base (280MB) — 60% smaller, 75% more margin. Architecture/training corpus > parameter count. - Depth non-monotonic within MS-MARCO MiniLM: L-4 > L-6 > L-12. - Capacity floor between L-2 (30MB, fails) and L-4 (50MB, separates). - 6 of 7 clean-separate the candidate-bench. The 'reranker catches Zionist-style mis-cites' claim is a property of competent rerankers as a class — once above the floor. Manifest primary moves to bge-reranker-large for cushion. demote_below_score STAYS null — clean candidate-bench doesn't predict real-pipeline behavior (§7 #18→#27 = 6 verdict flips). Expect a walk-back at §3.2.2 step 2.
377 lines
No EOL
9.7 KiB
JSON
377 lines
No EOL
9.7 KiB
JSON
{
|
|
"generated_at": "2026-05-13T13:58:45Z",
|
|
"fixtures_file": "/home/fox/nli-data/relevance-aboutness-v1.jsonl",
|
|
"n_pos": 14,
|
|
"n_neg": 12,
|
|
"models": {
|
|
"relevance-shadow-v1-ms-marco-electra-base": {
|
|
"hf_repo": "cross-encoder/ms-marco-electra-base",
|
|
"backend": "torch",
|
|
"device": "cuda",
|
|
"n_pairs": 26,
|
|
"infer_seconds": 0.14,
|
|
"pos_scores": {
|
|
"pos-qa-001": 3.448,
|
|
"pos-qa-002": 2.3775,
|
|
"pos-qa-003": 3.8591,
|
|
"pos-qa-004": 1.7753,
|
|
"pos-qa-005": 3.4742,
|
|
"pos-qa-006": 1.5494,
|
|
"pos-qa-007": 3.3069,
|
|
"pos-cs-001": 0.0497,
|
|
"pos-cs-002": 2.3681,
|
|
"pos-cs-003": 4.3109,
|
|
"pos-cs-004": 2.4378,
|
|
"pos-cs-005": 2.0715,
|
|
"pos-cs-006": 5.0473,
|
|
"pos-cs-007": 4.9582
|
|
},
|
|
"neg_scores": {
|
|
"neg-qa-001": -5.7487,
|
|
"neg-qa-002": -8.6513,
|
|
"neg-qa-003": -6.7965,
|
|
"neg-qa-004": -10.2494,
|
|
"neg-qa-005": -10.6582,
|
|
"neg-qa-006": -10.3879,
|
|
"neg-cs-001": -10.4413,
|
|
"neg-cs-002": -5.9059,
|
|
"neg-cs-003": -5.2999,
|
|
"neg-cs-004": -10.5911,
|
|
"neg-cs-005": -9.6781,
|
|
"neg-cs-006": -10.8435
|
|
},
|
|
"frontier": {
|
|
"separation_margin": 5.3495,
|
|
"fp0_best": {
|
|
"theta": 0.0497,
|
|
"catch_neg": 1.0,
|
|
"fp_on_pos": 0.0
|
|
},
|
|
"pareto": [
|
|
{
|
|
"theta": 0.0497,
|
|
"catch_neg": 1.0,
|
|
"fp_on_pos": 0.0
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"relevance-shadow-v1-bge-reranker-large": {
|
|
"hf_repo": "BAAI/bge-reranker-large",
|
|
"backend": "torch",
|
|
"device": "cuda",
|
|
"n_pairs": 26,
|
|
"infer_seconds": 0.13,
|
|
"pos_scores": {
|
|
"pos-qa-001": 6.6537,
|
|
"pos-qa-002": 6.5452,
|
|
"pos-qa-003": 6.0493,
|
|
"pos-qa-004": 5.2356,
|
|
"pos-qa-005": 7.0453,
|
|
"pos-qa-006": 7.1394,
|
|
"pos-qa-007": 5.8083,
|
|
"pos-cs-001": 4.6429,
|
|
"pos-cs-002": 5.5175,
|
|
"pos-cs-003": 6.8305,
|
|
"pos-cs-004": 7.1814,
|
|
"pos-cs-005": 7.122,
|
|
"pos-cs-006": 9.3351,
|
|
"pos-cs-007": 6.0121
|
|
},
|
|
"neg_scores": {
|
|
"neg-qa-001": -2.8404,
|
|
"neg-qa-002": -4.4815,
|
|
"neg-qa-003": -3.3392,
|
|
"neg-qa-004": -1.6424,
|
|
"neg-qa-005": -7.585,
|
|
"neg-qa-006": -5.5973,
|
|
"neg-cs-001": -7.0224,
|
|
"neg-cs-002": -5.5504,
|
|
"neg-cs-003": -5.5888,
|
|
"neg-cs-004": -5.1806,
|
|
"neg-cs-005": -6.5948,
|
|
"neg-cs-006": -6.2856
|
|
},
|
|
"frontier": {
|
|
"separation_margin": 6.2853,
|
|
"fp0_best": {
|
|
"theta": 4.6429,
|
|
"catch_neg": 1.0,
|
|
"fp_on_pos": 0.0
|
|
},
|
|
"pareto": [
|
|
{
|
|
"theta": 4.6429,
|
|
"catch_neg": 1.0,
|
|
"fp_on_pos": 0.0
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"relevance-shadow-v1-bge-reranker-base": {
|
|
"hf_repo": "BAAI/bge-reranker-base",
|
|
"backend": "torch",
|
|
"device": "cuda",
|
|
"n_pairs": 26,
|
|
"infer_seconds": 0.05,
|
|
"pos_scores": {
|
|
"pos-qa-001": 8.0599,
|
|
"pos-qa-002": 7.8094,
|
|
"pos-qa-003": 6.7217,
|
|
"pos-qa-004": 7.1199,
|
|
"pos-qa-005": 8.3108,
|
|
"pos-qa-006": 7.723,
|
|
"pos-qa-007": 6.2936,
|
|
"pos-cs-001": 2.395,
|
|
"pos-cs-002": 2.7944,
|
|
"pos-cs-003": 4.9885,
|
|
"pos-cs-004": 4.7798,
|
|
"pos-cs-005": 6.3818,
|
|
"pos-cs-006": 8.417,
|
|
"pos-cs-007": 5.2747
|
|
},
|
|
"neg_scores": {
|
|
"neg-qa-001": -0.7671,
|
|
"neg-qa-002": -3.387,
|
|
"neg-qa-003": -1.9937,
|
|
"neg-qa-004": -0.6807,
|
|
"neg-qa-005": -6.0444,
|
|
"neg-qa-006": -7.8323,
|
|
"neg-cs-001": -6.4084,
|
|
"neg-cs-002": -3.9995,
|
|
"neg-cs-003": -5.247,
|
|
"neg-cs-004": -4.4321,
|
|
"neg-cs-005": -7.8685,
|
|
"neg-cs-006": -6.9767
|
|
},
|
|
"frontier": {
|
|
"separation_margin": 3.0757,
|
|
"fp0_best": {
|
|
"theta": 2.395,
|
|
"catch_neg": 1.0,
|
|
"fp_on_pos": 0.0
|
|
},
|
|
"pareto": [
|
|
{
|
|
"theta": 2.395,
|
|
"catch_neg": 1.0,
|
|
"fp_on_pos": 0.0
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"relevance-shadow-v1-ms-marco-MiniLM-L-6-v2": {
|
|
"hf_repo": "cross-encoder/ms-marco-MiniLM-L-6-v2",
|
|
"backend": "torch",
|
|
"device": "cuda",
|
|
"n_pairs": 26,
|
|
"infer_seconds": 0.03,
|
|
"pos_scores": {
|
|
"pos-qa-001": 10.3561,
|
|
"pos-qa-002": 7.6135,
|
|
"pos-qa-003": 10.2615,
|
|
"pos-qa-004": 9.6113,
|
|
"pos-qa-005": 10.761,
|
|
"pos-qa-006": 10.6969,
|
|
"pos-qa-007": 10.189,
|
|
"pos-cs-001": 8.6417,
|
|
"pos-cs-002": 6.754,
|
|
"pos-cs-003": 10.2053,
|
|
"pos-cs-004": 9.1802,
|
|
"pos-cs-005": 10.58,
|
|
"pos-cs-006": 9.8569,
|
|
"pos-cs-007": 9.986
|
|
},
|
|
"neg_scores": {
|
|
"neg-qa-001": 3.7006,
|
|
"neg-qa-002": -0.0128,
|
|
"neg-qa-003": 3.1369,
|
|
"neg-qa-004": 0.2058,
|
|
"neg-qa-005": -5.2781,
|
|
"neg-qa-006": -9.0515,
|
|
"neg-cs-001": -8.1389,
|
|
"neg-cs-002": -2.9362,
|
|
"neg-cs-003": -5.963,
|
|
"neg-cs-004": -6.7567,
|
|
"neg-cs-005": -5.9802,
|
|
"neg-cs-006": -9.8485
|
|
},
|
|
"frontier": {
|
|
"separation_margin": 3.0533,
|
|
"fp0_best": {
|
|
"theta": 6.754,
|
|
"catch_neg": 1.0,
|
|
"fp_on_pos": 0.0
|
|
},
|
|
"pareto": [
|
|
{
|
|
"theta": 6.754,
|
|
"catch_neg": 1.0,
|
|
"fp_on_pos": 0.0
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"relevance-shadow-v1-ms-marco-MiniLM-L-12-v2": {
|
|
"hf_repo": "cross-encoder/ms-marco-MiniLM-L-12-v2",
|
|
"backend": "torch",
|
|
"device": "cuda",
|
|
"n_pairs": 26,
|
|
"infer_seconds": 0.05,
|
|
"pos_scores": {
|
|
"pos-qa-001": 10.1692,
|
|
"pos-qa-002": 8.139,
|
|
"pos-qa-003": 10.1767,
|
|
"pos-qa-004": 9.3511,
|
|
"pos-qa-005": 10.5464,
|
|
"pos-qa-006": 10.0394,
|
|
"pos-qa-007": 9.4536,
|
|
"pos-cs-001": 8.6255,
|
|
"pos-cs-002": 7.1379,
|
|
"pos-cs-003": 9.4853,
|
|
"pos-cs-004": 9.6927,
|
|
"pos-cs-005": 10.1209,
|
|
"pos-cs-006": 9.9942,
|
|
"pos-cs-007": 10.2065
|
|
},
|
|
"neg_scores": {
|
|
"neg-qa-001": 4.4752,
|
|
"neg-qa-002": -0.2735,
|
|
"neg-qa-003": 3.1724,
|
|
"neg-qa-004": -0.3941,
|
|
"neg-qa-005": -5.2772,
|
|
"neg-qa-006": -10.3666,
|
|
"neg-cs-001": -6.9534,
|
|
"neg-cs-002": -2.7428,
|
|
"neg-cs-003": -6.2688,
|
|
"neg-cs-004": -6.3582,
|
|
"neg-cs-005": -7.7344,
|
|
"neg-cs-006": -10.1564
|
|
},
|
|
"frontier": {
|
|
"separation_margin": 2.6627,
|
|
"fp0_best": {
|
|
"theta": 7.1379,
|
|
"catch_neg": 1.0,
|
|
"fp_on_pos": 0.0
|
|
},
|
|
"pareto": [
|
|
{
|
|
"theta": 7.1379,
|
|
"catch_neg": 1.0,
|
|
"fp_on_pos": 0.0
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"relevance-shadow-v1-ms-marco-MiniLM-L-4-v2": {
|
|
"hf_repo": "cross-encoder/ms-marco-MiniLM-L-4-v2",
|
|
"backend": "torch",
|
|
"device": "cuda",
|
|
"n_pairs": 26,
|
|
"infer_seconds": 0.02,
|
|
"pos_scores": {
|
|
"pos-qa-001": 10.4093,
|
|
"pos-qa-002": 8.3202,
|
|
"pos-qa-003": 9.753,
|
|
"pos-qa-004": 8.5922,
|
|
"pos-qa-005": 10.3538,
|
|
"pos-qa-006": 9.6334,
|
|
"pos-qa-007": 8.9049,
|
|
"pos-cs-001": 7.4864,
|
|
"pos-cs-002": 6.736,
|
|
"pos-cs-003": 9.3345,
|
|
"pos-cs-004": 9.2342,
|
|
"pos-cs-005": 10.1494,
|
|
"pos-cs-006": 9.879,
|
|
"pos-cs-007": 9.8858
|
|
},
|
|
"neg_scores": {
|
|
"neg-qa-001": 3.5486,
|
|
"neg-qa-002": -0.0804,
|
|
"neg-qa-003": 1.3674,
|
|
"neg-qa-004": -1.9597,
|
|
"neg-qa-005": -7.1108,
|
|
"neg-qa-006": -10.4729,
|
|
"neg-cs-001": -10.0115,
|
|
"neg-cs-002": -5.1121,
|
|
"neg-cs-003": -6.8413,
|
|
"neg-cs-004": -7.3554,
|
|
"neg-cs-005": -8.4252,
|
|
"neg-cs-006": -10.3512
|
|
},
|
|
"frontier": {
|
|
"separation_margin": 3.1874,
|
|
"fp0_best": {
|
|
"theta": 6.736,
|
|
"catch_neg": 1.0,
|
|
"fp_on_pos": 0.0
|
|
},
|
|
"pareto": [
|
|
{
|
|
"theta": 6.736,
|
|
"catch_neg": 1.0,
|
|
"fp_on_pos": 0.0
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"relevance-shadow-v1-ms-marco-MiniLM-L-2-v2": {
|
|
"hf_repo": "cross-encoder/ms-marco-MiniLM-L-2-v2",
|
|
"backend": "torch",
|
|
"device": "cuda",
|
|
"n_pairs": 26,
|
|
"infer_seconds": 0.02,
|
|
"pos_scores": {
|
|
"pos-qa-001": 10.1813,
|
|
"pos-qa-002": 7.5824,
|
|
"pos-qa-003": 10.007,
|
|
"pos-qa-004": 9.9509,
|
|
"pos-qa-005": 8.9993,
|
|
"pos-qa-006": 10.4307,
|
|
"pos-qa-007": 9.0209,
|
|
"pos-cs-001": 7.6376,
|
|
"pos-cs-002": 3.3707,
|
|
"pos-cs-003": 9.1434,
|
|
"pos-cs-004": 9.0668,
|
|
"pos-cs-005": 9.4988,
|
|
"pos-cs-006": 8.6096,
|
|
"pos-cs-007": 9.6396
|
|
},
|
|
"neg_scores": {
|
|
"neg-qa-001": 5.3408,
|
|
"neg-qa-002": -0.2153,
|
|
"neg-qa-003": 4.4499,
|
|
"neg-qa-004": -1.3149,
|
|
"neg-qa-005": -6.088,
|
|
"neg-qa-006": -11.2537,
|
|
"neg-cs-001": -11.3311,
|
|
"neg-cs-002": -9.3536,
|
|
"neg-cs-003": -8.6388,
|
|
"neg-cs-004": -11.478,
|
|
"neg-cs-005": -9.13,
|
|
"neg-cs-006": -9.0799
|
|
},
|
|
"frontier": {
|
|
"separation_margin": -1.97,
|
|
"fp0_best": {
|
|
"theta": 3.3707,
|
|
"catch_neg": 0.8333,
|
|
"fp_on_pos": 0.0
|
|
},
|
|
"pareto": [
|
|
{
|
|
"theta": 3.3707,
|
|
"catch_neg": 0.8333,
|
|
"fp_on_pos": 0.0
|
|
},
|
|
{
|
|
"theta": 7.5824,
|
|
"catch_neg": 1.0,
|
|
"fp_on_pos": 0.0714
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
} |