{
  "generated_at": "2026-05-13T18:33:03Z",
  "strict_pool_files": [
    "/home/fox/git/arborist/bench/qa_results/2026-05-12T20-53-11Z.jsonl",
    "/home/fox/git/arborist/bench/qa_results/2026-05-12T21-58-58Z.jsonl",
    "/home/fox/git/arborist/bench/qa_results/2026-05-12T22-44-30Z.jsonl"
  ],
  "n_strict_pairs": 808,
  "models": {
    "relevance-shadow-v1-bge-reranker-large": {
      "hf_repo": "BAAI/bge-reranker-large",
      "backend": "torch",
      "device": "cpu",
      "n_pairs": 808,
      "infer_seconds": 678.5,
      "score_quantiles_on_strict": {
        "p0": -2.4174,
        "p10": 3.5623,
        "p50": 6.4753,
        "p90": 7.9334,
        "p100": 8.7043
      },
      "candidate_bench_theta": 4.643,
      "fp_rate_at_candidate_bench_theta": 0.18564356435643564,
      "smallest_theta_for_zero_fp_on_pool": -2.4174,
      "headroom_table": [
        {
          "pct_strict_below": 0.0,
          "theta_at_that_cut": -2.4174
        },
        {
          "pct_strict_below": 1.0,
          "theta_at_that_cut": -0.2684
        },
        {
          "pct_strict_below": 5.0,
          "theta_at_that_cut": 2.217
        },
        {
          "pct_strict_below": 10.0,
          "theta_at_that_cut": 3.5623
        }
      ]
    },
    "relevance-shadow-v1-bge-reranker-base": {
      "hf_repo": "BAAI/bge-reranker-base",
      "backend": "torch",
      "device": "cpu",
      "n_pairs": 808,
      "infer_seconds": 195.39,
      "score_quantiles_on_strict": {
        "p0": -3.6162,
        "p10": 3.2004,
        "p50": 7.6752,
        "p90": 8.8818,
        "p100": 10.0034
      },
      "candidate_bench_theta": 2.395,
      "fp_rate_at_candidate_bench_theta": 0.09034653465346534,
      "smallest_theta_for_zero_fp_on_pool": -3.6162,
      "headroom_table": [
        {
          "pct_strict_below": 0.0,
          "theta_at_that_cut": -3.6162
        },
        {
          "pct_strict_below": 1.0,
          "theta_at_that_cut": -1.637
        },
        {
          "pct_strict_below": 5.0,
          "theta_at_that_cut": 1.2898
        },
        {
          "pct_strict_below": 10.0,
          "theta_at_that_cut": 3.2004
        }
      ]
    },
    "relevance-shadow-v1-ms-marco-MiniLM-L-6-v2": {
      "hf_repo": "cross-encoder/ms-marco-MiniLM-L-6-v2",
      "backend": "torch",
      "device": "cpu",
      "n_pairs": 808,
      "infer_seconds": 25.76,
      "score_quantiles_on_strict": {
        "p0": -5.2893,
        "p10": 6.9279,
        "p50": 9.3245,
        "p90": 10.5692,
        "p100": 11.0746
      },
      "candidate_bench_theta": 6.754,
      "fp_rate_at_candidate_bench_theta": 0.09034653465346534,
      "smallest_theta_for_zero_fp_on_pool": -5.2893,
      "headroom_table": [
        {
          "pct_strict_below": 0.0,
          "theta_at_that_cut": -5.2893
        },
        {
          "pct_strict_below": 1.0,
          "theta_at_that_cut": 4.7953
        },
        {
          "pct_strict_below": 5.0,
          "theta_at_that_cut": 5.6997
        },
        {
          "pct_strict_below": 10.0,
          "theta_at_that_cut": 6.9279
        }
      ]
    },
    "relevance-shadow-v1-ms-marco-MiniLM-L-12-v2": {
      "hf_repo": "cross-encoder/ms-marco-MiniLM-L-12-v2",
      "backend": "torch",
      "device": "cpu",
      "n_pairs": 808,
      "infer_seconds": 55.14,
      "score_quantiles_on_strict": {
        "p0": -3.9779,
        "p10": 7.5669,
        "p50": 9.2423,
        "p90": 10.4212,
        "p100": 10.8656
      },
      "candidate_bench_theta": 7.138,
      "fp_rate_at_candidate_bench_theta": 0.08044554455445545,
      "smallest_theta_for_zero_fp_on_pool": -3.9779,
      "headroom_table": [
        {
          "pct_strict_below": 0.0,
          "theta_at_that_cut": -3.9779
        },
        {
          "pct_strict_below": 1.0,
          "theta_at_that_cut": 4.9686
        },
        {
          "pct_strict_below": 5.0,
          "theta_at_that_cut": 5.8329
        },
        {
          "pct_strict_below": 10.0,
          "theta_at_that_cut": 7.5669
        }
      ]
    },
    "relevance-shadow-v1-ms-marco-MiniLM-L-4-v2": {
      "hf_repo": "cross-encoder/ms-marco-MiniLM-L-4-v2",
      "backend": "torch",
      "device": "cpu",
      "n_pairs": 808,
      "infer_seconds": 19.14,
      "score_quantiles_on_strict": {
        "p0": -5.9961,
        "p10": 6.8021,
        "p50": 8.8673,
        "p90": 10.3022,
        "p100": 10.7486
      },
      "candidate_bench_theta": 6.736,
      "fp_rate_at_candidate_bench_theta": 0.09405940594059406,
      "smallest_theta_for_zero_fp_on_pool": -5.9961,
      "headroom_table": [
        {
          "pct_strict_below": 0.0,
          "theta_at_that_cut": -5.9961
        },
        {
          "pct_strict_below": 1.0,
          "theta_at_that_cut": 4.3229
        },
        {
          "pct_strict_below": 5.0,
          "theta_at_that_cut": 5.3019
        },
        {
          "pct_strict_below": 10.0,
          "theta_at_that_cut": 6.8021
        }
      ]
    },
    "relevance-shadow-v1-ms-marco-MiniLM-L-2-v2": {
      "hf_repo": "cross-encoder/ms-marco-MiniLM-L-2-v2",
      "backend": "torch",
      "device": "cpu",
      "n_pairs": 808,
      "infer_seconds": 9.8,
      "score_quantiles_on_strict": {
        "p0": -5.9918,
        "p10": 5.2108,
        "p50": 8.5422,
        "p90": 10.1706,
        "p100": 10.8884
      },
      "candidate_bench_theta": 3.371,
      "fp_rate_at_candidate_bench_theta": 0.01485148514851485,
      "smallest_theta_for_zero_fp_on_pool": -5.9918,
      "headroom_table": [
        {
          "pct_strict_below": 0.0,
          "theta_at_that_cut": -5.9918
        },
        {
          "pct_strict_below": 1.0,
          "theta_at_that_cut": 1.1027
        },
        {
          "pct_strict_below": 5.0,
          "theta_at_that_cut": 4.1674
        },
        {
          "pct_strict_below": 10.0,
          "theta_at_that_cut": 5.2108
        }
      ]
    }
  }
}