{
  "generated_at": "2026-05-13T17:40:47Z",
  "strict_pool_files": [
    "/home/fox/git/arborist/bench/qa_results/2026-05-12T20-53-11Z.jsonl",
    "/home/fox/git/arborist/bench/qa_results/2026-05-12T21-58-58Z.jsonl",
    "/home/fox/git/arborist/bench/qa_results/2026-05-12T22-44-30Z.jsonl"
  ],
  "n_strict_pairs": 808,
  "models": {
    "relevance-shadow-v1-bge-reranker-large": {
      "hf_repo": "BAAI/bge-reranker-large",
      "backend": "torch",
      "device": "cpu",
      "n_pairs": 808,
      "infer_seconds": 863.3,
      "score_quantiles_on_strict": {
        "p0": -2.1998,
        "p10": 2.9896,
        "p50": 5.8016,
        "p90": 7.54,
        "p100": 8.2654
      },
      "candidate_bench_theta": 4.643,
      "fp_rate_at_candidate_bench_theta": 0.21782178217821782,
      "smallest_theta_for_zero_fp_on_pool": -2.1998,
      "headroom_table": [
        {
          "pct_strict_below": 0.0,
          "theta_at_that_cut": -2.1998
        },
        {
          "pct_strict_below": 1.0,
          "theta_at_that_cut": -0.2684
        },
        {
          "pct_strict_below": 5.0,
          "theta_at_that_cut": 2.0856
        },
        {
          "pct_strict_below": 10.0,
          "theta_at_that_cut": 2.9896
        }
      ]
    },
    "relevance-shadow-v1-bge-reranker-base": {
      "hf_repo": "BAAI/bge-reranker-base",
      "backend": "torch",
      "device": "cpu",
      "n_pairs": 808,
      "infer_seconds": 240.16,
      "score_quantiles_on_strict": {
        "p0": -3.6162,
        "p10": 3.2004,
        "p50": 6.8232,
        "p90": 8.1698,
        "p100": 9.6886
      },
      "candidate_bench_theta": 2.395,
      "fp_rate_at_candidate_bench_theta": 0.0952970297029703,
      "smallest_theta_for_zero_fp_on_pool": -3.6162,
      "headroom_table": [
        {
          "pct_strict_below": 0.0,
          "theta_at_that_cut": -3.6162
        },
        {
          "pct_strict_below": 1.0,
          "theta_at_that_cut": -1.637
        },
        {
          "pct_strict_below": 5.0,
          "theta_at_that_cut": 1.2898
        },
        {
          "pct_strict_below": 10.0,
          "theta_at_that_cut": 3.2004
        }
      ]
    },
    "relevance-shadow-v1-ms-marco-MiniLM-L-6-v2": {
      "hf_repo": "cross-encoder/ms-marco-MiniLM-L-6-v2",
      "backend": "torch",
      "device": "cpu",
      "n_pairs": 808,
      "infer_seconds": 37.98,
      "score_quantiles_on_strict": {
        "p0": -7.0632,
        "p10": 6.3215,
        "p50": 8.7989,
        "p90": 10.4243,
        "p100": 11.0746
      },
      "candidate_bench_theta": 6.754,
      "fp_rate_at_candidate_bench_theta": 0.11633663366336634,
      "smallest_theta_for_zero_fp_on_pool": -7.0632,
      "headroom_table": [
        {
          "pct_strict_below": 0.0,
          "theta_at_that_cut": -7.0632
        },
        {
          "pct_strict_below": 1.0,
          "theta_at_that_cut": 4.0411
        },
        {
          "pct_strict_below": 5.0,
          "theta_at_that_cut": 5.2801
        },
        {
          "pct_strict_below": 10.0,
          "theta_at_that_cut": 6.3215
        }
      ]
    },
    "relevance-shadow-v1-ms-marco-MiniLM-L-12-v2": {
      "hf_repo": "cross-encoder/ms-marco-MiniLM-L-12-v2",
      "backend": "torch",
      "device": "cpu",
      "n_pairs": 808,
      "infer_seconds": 72.27,
      "score_quantiles_on_strict": {
        "p0": -4.9336,
        "p10": 7.0023,
        "p50": 8.8233,
        "p90": 10.2894,
        "p100": 10.8656
      },
      "candidate_bench_theta": 7.138,
      "fp_rate_at_candidate_bench_theta": 0.11014851485148515,
      "smallest_theta_for_zero_fp_on_pool": -4.9336,
      "headroom_table": [
        {
          "pct_strict_below": 0.0,
          "theta_at_that_cut": -4.9336
        },
        {
          "pct_strict_below": 1.0,
          "theta_at_that_cut": 4.3478
        },
        {
          "pct_strict_below": 5.0,
          "theta_at_that_cut": 5.8487
        },
        {
          "pct_strict_below": 10.0,
          "theta_at_that_cut": 7.0023
        }
      ]
    },
    "relevance-shadow-v1-ms-marco-MiniLM-L-4-v2": {
      "hf_repo": "cross-encoder/ms-marco-MiniLM-L-4-v2",
      "backend": "torch",
      "device": "cpu",
      "n_pairs": 808,
      "infer_seconds": 28.22,
      "score_quantiles_on_strict": {
        "p0": -5.5123,
        "p10": 5.8181,
        "p50": 8.3106,
        "p90": 9.9864,
        "p100": 10.7486
      },
      "candidate_bench_theta": 6.736,
      "fp_rate_at_candidate_bench_theta": 0.14975247524752475,
      "smallest_theta_for_zero_fp_on_pool": -5.5123,
      "headroom_table": [
        {
          "pct_strict_below": 0.0,
          "theta_at_that_cut": -5.5123
        },
        {
          "pct_strict_below": 1.0,
          "theta_at_that_cut": 3.9381
        },
        {
          "pct_strict_below": 5.0,
          "theta_at_that_cut": 4.8423
        },
        {
          "pct_strict_below": 10.0,
          "theta_at_that_cut": 5.8181
        }
      ]
    },
    "relevance-shadow-v1-ms-marco-MiniLM-L-2-v2": {
      "hf_repo": "cross-encoder/ms-marco-MiniLM-L-2-v2",
      "backend": "torch",
      "device": "cpu",
      "n_pairs": 808,
      "infer_seconds": 11.24,
      "score_quantiles_on_strict": {
        "p0": -6.8554,
        "p10": 4.0983,
        "p50": 7.7531,
        "p90": 9.6393,
        "p100": 10.4551
      },
      "candidate_bench_theta": 3.371,
      "fp_rate_at_candidate_bench_theta": 0.04455445544554455,
      "smallest_theta_for_zero_fp_on_pool": -6.8554,
      "headroom_table": [
        {
          "pct_strict_below": 0.0,
          "theta_at_that_cut": -6.8554
        },
        {
          "pct_strict_below": 1.0,
          "theta_at_that_cut": 1.1027
        },
        {
          "pct_strict_below": 5.0,
          "theta_at_that_cut": 3.5032
        },
        {
          "pct_strict_below": 10.0,
          "theta_at_that_cut": 4.0983
        }
      ]
    }
  }
}