russell@unturf.com
d57e5c1435
(a) Mined the pooled n=1+3+5 bench-qa runs (808 distinct STRICT answers) for natural recombinations at lowered θc≥0.7 → 37 would-fires, ALL token-collision FPs on inspection (Mount Kenya pulled into a Kilimanjaro answer, Dalí into da Vinci, Donovan into Superman). ZERO genuine recombination errors — the boundary is theoretical-in-practice; the failure mode is the candidate selector (top-k by token overlap) pulling different-entity same-subject-area clauses. (b) Re-ran the grid against the 808-cell pooled STRICT set: the §7 #25 'max@0.96' was itself a small-sample artifact — the n=5 444-cell set lacked the high-confidence spurious hits the pooled set has. On 808 cells θc goes back to ~0.995, and at θc=0.995 only agg=margin still catches 28/28 (max gets 27/28). microsoft/deberta-large-mnli / k=3 / agg=margin / θc=0.995 → 28/28 synthetic recombinations · 0/808 pooled real STRICT FP · 0/26 synthetic legit — the ONLY config in the 7-model×full-grid sweep that hits 1.0/0.0 on 808 cells, held at n=3 too. recommended_operating_point reverted to margin@0.995. Realistic next check: ~20-30 hand-built synthetic-recombination-vs- real-bench-qa-context fixtures (real haystack, deliberate re-attribution). Still SHADOW; runtime promotion fox+dav1d-decides. Production verifier unchanged; falsification-hard stays 10/12. Meta-lesson instance five: a bigger sample can vindicate a config a smaller one made look unnecessary — re-confirm the config choice (not just the threshold) each time the denominator grows.
8597 lines
No EOL
198 KiB
JSON
8597 lines
No EOL
198 KiB
JSON
{
|
|
"generated_at": "2026-05-13T01:01:09Z",
|
|
"should_demote_file": "/home/fox/nli-data/recombination.jsonl",
|
|
"n_should_demote": 28,
|
|
"should_not_demote_file": "/home/fox/nli-data/benchqa-pooled-strict.jsonl",
|
|
"n_should_not_demote": 808,
|
|
"k_grid": [
|
|
1,
|
|
2,
|
|
3,
|
|
4,
|
|
6,
|
|
8,
|
|
12
|
|
],
|
|
"aggs": [
|
|
"max",
|
|
"mean",
|
|
"top2",
|
|
"top3",
|
|
"meanTop2",
|
|
"meanTop3",
|
|
"margin"
|
|
],
|
|
"guards": [
|
|
"max_entail",
|
|
"paired_entail"
|
|
],
|
|
"theta_c_grid": [
|
|
0.3,
|
|
0.4,
|
|
0.5,
|
|
0.6,
|
|
0.7,
|
|
0.8,
|
|
0.85,
|
|
0.9,
|
|
0.92,
|
|
0.94,
|
|
0.95,
|
|
0.96,
|
|
0.97,
|
|
0.975,
|
|
0.98,
|
|
0.985,
|
|
0.99,
|
|
0.995,
|
|
0.998,
|
|
0.999
|
|
],
|
|
"theta_e_grid": [
|
|
0.3,
|
|
0.5,
|
|
0.7,
|
|
0.9,
|
|
0.95,
|
|
1.01
|
|
],
|
|
"kmax": 12,
|
|
"models": {
|
|
"nli-shadow-v1-minilm2-l6-h768": {
|
|
"hf_repo": "cross-encoder/nli-MiniLM2-L6-H768",
|
|
"backend": "torch",
|
|
"device": "cuda",
|
|
"n_nli_pairs": 9749,
|
|
"infer_seconds": 9.29,
|
|
"global_best": {
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.96,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5,
|
|
"fp": 0.0
|
|
},
|
|
"pareto": [
|
|
{
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.96,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5,
|
|
"fp": 0.0
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 1.01,
|
|
"catch": 0.6429,
|
|
"fp": 0.0012
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.98,
|
|
"theta_e": 0.9,
|
|
"catch": 0.8214,
|
|
"fp": 0.0025
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.95,
|
|
"theta_e": 0.9,
|
|
"catch": 0.8571,
|
|
"fp": 0.0062
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.94,
|
|
"theta_e": 0.9,
|
|
"catch": 0.9286,
|
|
"fp": 0.0297
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.94,
|
|
"theta_e": 0.9,
|
|
"catch": 0.9643,
|
|
"fp": 0.0557
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.7,
|
|
"theta_e": 0.9,
|
|
"catch": 1.0,
|
|
"fp": 0.1002
|
|
}
|
|
],
|
|
"configs_ranked": [
|
|
{
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.96,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.96,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.97,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.97,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4286,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4286,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.98,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4286,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.98,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4286,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 1.01,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 1.01,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 1.01,
|
|
"catch": 0.0357,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 1.01,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 1.01,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 1.01,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 1.01,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
}
|
|
]
|
|
},
|
|
"nli-shadow-v1-minilm-l6-mnli": {
|
|
"hf_repo": "cross-encoder/nli-MiniLM2-L6-H768",
|
|
"backend": "torch",
|
|
"device": "cuda",
|
|
"n_nli_pairs": 9749,
|
|
"infer_seconds": 9.41,
|
|
"global_best": {
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.96,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5,
|
|
"fp": 0.0
|
|
},
|
|
"pareto": [
|
|
{
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.96,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5,
|
|
"fp": 0.0
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 1.01,
|
|
"catch": 0.6429,
|
|
"fp": 0.0012
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.98,
|
|
"theta_e": 0.9,
|
|
"catch": 0.8214,
|
|
"fp": 0.0025
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.95,
|
|
"theta_e": 0.9,
|
|
"catch": 0.8571,
|
|
"fp": 0.0062
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.94,
|
|
"theta_e": 0.9,
|
|
"catch": 0.9286,
|
|
"fp": 0.0297
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.94,
|
|
"theta_e": 0.9,
|
|
"catch": 0.9643,
|
|
"fp": 0.0557
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.7,
|
|
"theta_e": 0.9,
|
|
"catch": 1.0,
|
|
"fp": 0.1002
|
|
}
|
|
],
|
|
"configs_ranked": [
|
|
{
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.96,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.96,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.97,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.97,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4286,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4286,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.98,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4286,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.98,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4286,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 1.01,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 1.01,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.2143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 1.01,
|
|
"catch": 0.0357,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 1.01,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 1.01,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 1.01,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 1.01,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
}
|
|
]
|
|
},
|
|
"nli-shadow-v1-deberta-v3-base": {
|
|
"hf_repo": "MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli",
|
|
"backend": "torch",
|
|
"device": "cuda",
|
|
"n_nli_pairs": 9749,
|
|
"infer_seconds": 24.14,
|
|
"global_best": {
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"pareto": [
|
|
{
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.9,
|
|
"catch": 0.9643,
|
|
"fp": 0.0012
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.9,
|
|
"catch": 1.0,
|
|
"fp": 0.005
|
|
}
|
|
],
|
|
"configs_ranked": [
|
|
{
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0357,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0357,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0357,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0357,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0357,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0357,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.9,
|
|
"theta_e": 0.3,
|
|
"catch": 1.0,
|
|
"fp": 0.0347
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.97,
|
|
"theta_e": 0.3,
|
|
"catch": 1.0,
|
|
"fp": 0.0371
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.97,
|
|
"theta_e": 1.01,
|
|
"catch": 1.0,
|
|
"fp": 0.0371
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.97,
|
|
"theta_e": 0.9,
|
|
"catch": 1.0,
|
|
"fp": 0.0371
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.85,
|
|
"theta_e": 1.01,
|
|
"catch": 1.0,
|
|
"fp": 0.0408
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.96,
|
|
"theta_e": 0.9,
|
|
"catch": 1.0,
|
|
"fp": 0.0458
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.97,
|
|
"theta_e": 0.3,
|
|
"catch": 1.0,
|
|
"fp": 0.047
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.97,
|
|
"theta_e": 1.01,
|
|
"catch": 1.0,
|
|
"fp": 0.047
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.8,
|
|
"theta_e": 0.9,
|
|
"catch": 1.0,
|
|
"fp": 0.0483
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.96,
|
|
"theta_e": 0.9,
|
|
"catch": 1.0,
|
|
"fp": 0.0483
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.96,
|
|
"theta_e": 0.3,
|
|
"catch": 1.0,
|
|
"fp": 0.0495
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.95,
|
|
"theta_e": 1.01,
|
|
"catch": 1.0,
|
|
"fp": 0.0495
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.97,
|
|
"theta_e": 0.9,
|
|
"catch": 1.0,
|
|
"fp": 0.0495
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.9643,
|
|
"fp": 0.0421
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 1.01,
|
|
"catch": 0.9643,
|
|
"fp": 0.0421
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.9,
|
|
"theta_e": 0.3,
|
|
"catch": 0.9643,
|
|
"fp": 0.0483
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 1.01,
|
|
"catch": 0.9286,
|
|
"fp": 0.0186
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.9286,
|
|
"fp": 0.0198
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.9286,
|
|
"fp": 0.0198
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.9286,
|
|
"fp": 0.0198
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.9286,
|
|
"fp": 0.0198
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.9286,
|
|
"fp": 0.0198
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.9286,
|
|
"fp": 0.0198
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.9286,
|
|
"fp": 0.0198
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.9286,
|
|
"fp": 0.0198
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 1.01,
|
|
"catch": 0.8929,
|
|
"fp": 0.0421
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.8929,
|
|
"fp": 0.0446
|
|
},
|
|
"best_kind": "fp<=0.05"
|
|
}
|
|
]
|
|
},
|
|
"nli-shadow-v1-bart-large-mnli": {
|
|
"hf_repo": "facebook/bart-large-mnli",
|
|
"backend": "torch",
|
|
"device": "cuda",
|
|
"n_nli_pairs": 9749,
|
|
"infer_seconds": 50.11,
|
|
"global_best": {
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.94,
|
|
"theta_e": 0.3,
|
|
"catch": 0.7143,
|
|
"fp": 0.0
|
|
},
|
|
"pareto": [
|
|
{
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.94,
|
|
"theta_e": 0.3,
|
|
"catch": 0.7143,
|
|
"fp": 0.0
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0111
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 1.01,
|
|
"catch": 0.9286,
|
|
"fp": 0.0186
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 1.01,
|
|
"catch": 0.9643,
|
|
"fp": 0.0297
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.6,
|
|
"theta_e": 1.01,
|
|
"catch": 1.0,
|
|
"fp": 0.1275
|
|
}
|
|
],
|
|
"configs_ranked": [
|
|
{
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.94,
|
|
"theta_e": 0.3,
|
|
"catch": 0.7143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.94,
|
|
"theta_e": 0.3,
|
|
"catch": 0.7143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.94,
|
|
"theta_e": 0.3,
|
|
"catch": 0.7143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.94,
|
|
"theta_e": 0.3,
|
|
"catch": 0.7143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.95,
|
|
"theta_e": 0.3,
|
|
"catch": 0.7143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.95,
|
|
"theta_e": 0.3,
|
|
"catch": 0.7143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.97,
|
|
"theta_e": 0.3,
|
|
"catch": 0.7143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.97,
|
|
"theta_e": 0.3,
|
|
"catch": 0.7143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.7143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.7143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.98,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.98,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5714,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5357,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5357,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 1.01,
|
|
"catch": 0.5357,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 1.01,
|
|
"catch": 0.5357,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 1.01,
|
|
"catch": 0.5357,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 1.01,
|
|
"catch": 0.5357,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 1.01,
|
|
"catch": 0.5357,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 1.01,
|
|
"catch": 0.5,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3214,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3214,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3214,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3214,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3214,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3214,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3214,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3214,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3214,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3214,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3214,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3214,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3214,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3214,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 1.01,
|
|
"catch": 0.2857,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.85,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.85,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
}
|
|
]
|
|
},
|
|
"extra:roberta-large-mnli": {
|
|
"hf_repo": "roberta-large-mnli",
|
|
"backend": "torch",
|
|
"device": "cuda",
|
|
"n_nli_pairs": 9749,
|
|
"infer_seconds": 41.57,
|
|
"global_best": {
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.7,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"pareto": [
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.7,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.95,
|
|
"theta_e": 0.9,
|
|
"catch": 1.0,
|
|
"fp": 0.0012
|
|
}
|
|
],
|
|
"configs_ranked": [
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.7,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.7,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.7,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.7,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.7,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.7,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 1.01,
|
|
"catch": 0.6429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 1.01,
|
|
"catch": 0.6429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 1.01,
|
|
"catch": 0.6429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 1.01,
|
|
"catch": 0.6429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 1.01,
|
|
"catch": 0.6429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 1.01,
|
|
"catch": 0.6429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.4643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 1.01,
|
|
"catch": 0.4286,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3929,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
}
|
|
]
|
|
},
|
|
"extra:deberta-large-mnli": {
|
|
"hf_repo": "microsoft/deberta-large-mnli",
|
|
"backend": "torch",
|
|
"device": "cuda",
|
|
"n_nli_pairs": 9749,
|
|
"infer_seconds": 68.78,
|
|
"global_best": {
|
|
"k": 3,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 1.01,
|
|
"catch": 1.0,
|
|
"fp": 0.0
|
|
},
|
|
"pareto": [
|
|
{
|
|
"k": 3,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 1.01,
|
|
"catch": 1.0,
|
|
"fp": 0.0
|
|
}
|
|
],
|
|
"configs_ranked": [
|
|
{
|
|
"k": 3,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 1.01,
|
|
"catch": 1.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 1.01,
|
|
"catch": 1.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.9,
|
|
"catch": 0.9643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.9643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 1.01,
|
|
"catch": 0.9643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.5,
|
|
"catch": 0.9643,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 1.01,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.9,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.9,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.75,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 1.01,
|
|
"catch": 0.7143,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.98,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.98,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.975,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "mean",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.99,
|
|
"theta_e": 0.3,
|
|
"catch": 0.6071,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.5,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.3571,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "max",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "meanTop3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.25,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.1786,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 1.01,
|
|
"catch": 0.1429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "margin",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.999,
|
|
"theta_e": 1.01,
|
|
"catch": 0.1429,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top2",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 1,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 2,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.3,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 3,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 4,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.985,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 6,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.995,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 8,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "max_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
},
|
|
{
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"best": {
|
|
"k": 12,
|
|
"agg": "top3",
|
|
"guard": "paired_entail",
|
|
"theta_c": 0.998,
|
|
"theta_e": 0.3,
|
|
"catch": 0.0,
|
|
"fp": 0.0
|
|
},
|
|
"best_kind": "fp=0"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
} |