diff --git a/arborist/qa/relevance/manifest.json b/arborist/qa/relevance/manifest.json index 2d12f3d..1ace17d 100644 --- a/arborist/qa/relevance/manifest.json +++ b/arborist/qa/relevance/manifest.json @@ -29,7 +29,7 @@ ], "step3_finding_miscite_vs_deflection": "Mis-cite is structurally much easier than deflection — bge-large mis-cite scores range -9.37 to -3.81 (max), strictly below STRICT min (-2.42). Deflection range -5.57 to +5.01 overlaps STRICT bottom. So the runtime veto is FULLY effective on mis-cite (the Zionist failure mode) and PARTIALLY effective on deflection. That's appropriate to the structural difference: mis-cite is 'wrong topic entirely', deflection is 'right topic, not answering'.", "as_advisory_soft_signal": "viable — surface the raw score on the render tail or as bench telemetry; no threshold needed", - "as_multi_signal_factor": "viable — combine with title-relevance, deflection, coherence sidecars for a weighted advisory", + "as_multi_signal_factor": "TESTED 2026-05-13 (§3.2.2 step 3 part C) — DOES NOT LIFT precision/recall on the failure shapes §3.2 covers. diagnose_deflection fires on 0/40 of the deflection+miscite NEG fixtures because the fixtures are token-overlap-correct (which is the whole point — they're the cases the lexical sidecar misses); diagnose_coherence fires on 0/40 because the fixtures are well-formed. So UNION = relevance alone, INTERSECT = 0. The architectural finding is positive in a different way: the lexical sidecars and the relevance reranker cover NON-OVERLAPPING failure shapes (structural vs subject-anchor vs aboutness). Each signal owns its own slice cleanly; combining them is redundant on these cases.", "as_contrastive_signal": "TESTED 2026-05-13 — DOES NOT SEPARATE on real bench-qa data: 97% of STRICT-fires at cb-θ have BOTH Q→A and A→ctx_lead low (co-varying); only 3% match the 'high A→ctx, low Q→A' deflection signature. Contrastive rescue path closed — but it's no longer needed since real-context deflections (step 3) DID separate cleanly with absolute thresholds.", "preprocessing_cleaning_is_free_improvement": "claim-lattice metadata [E\\d+ | … | hash: '…'] depresses scores by ~3-6 pts on cleaned vs raw; clean_for_relevance is baked into check_* by default. Universal win across all 6 swept models (-0.5 to -5.6 pts FP)." }, diff --git a/bench/results/nli-shadow-grid-n1-minilm.json b/bench/results/nli-shadow-grid-n1-minilm.json deleted file mode 100644 index f87d586..0000000 --- a/bench/results/nli-shadow-grid-n1-minilm.json +++ /dev/null @@ -1,250 +0,0 @@ -{ - "generated_at": "2026-05-12T22:00:11Z", - "should_demote_file": "/home/fox/git/arborist/bench/fixtures/5f/falsification-hard-v1.jsonl", - "n_should_demote": 12, - "should_not_demote_file": "bench/qa_results/2026-05-12T20-53-11Z.jsonl", - "n_should_not_demote": 89, - "k_grid": [ - 1, - 2, - 3, - 4, - 6, - 12 - ], - "aggs": [ - "max", - "mean", - "top2" - ], - "theta_c_grid": [ - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, - 0.8, - 0.85, - 0.9, - 0.93, - 0.95, - 0.97, - 0.99 - ], - "theta_e_grid": [ - 0.5, - 0.7, - 0.9, - 1.01 - ], - "kmax": 12, - "models": { - "nli-shadow-v1-minilm2-l6-h768": { - "hf_repo": "cross-encoder/nli-MiniLM2-L6-H768", - "backend": "onnx-int8", - "device": "cpu", - "n_nli_pairs": 1085, - "infer_seconds": 184.74, - "configs_ranked": [ - { - "k": 1, - "agg": "max", - "best": { - "theta_c": 0.93, - "theta_e": 0.5, - "catch": 0.6667, - "fp": 0.0 - }, - "best_kind": "fp=0" - }, - { - "k": 1, - "agg": "mean", - "best": { - "theta_c": 0.93, - "theta_e": 0.5, - "catch": 0.6667, - "fp": 0.0 - }, - "best_kind": "fp=0" - }, - { - "k": 2, - "agg": "max", - "best": { - "theta_c": 0.97, - "theta_e": 0.5, - "catch": 0.6667, - "fp": 0.0 - }, - "best_kind": "fp=0" - }, - { - "k": 2, - "agg": "mean", - "best": { - "theta_c": 0.85, - "theta_e": 0.5, - "catch": 0.6667, - "fp": 0.0 - }, - "best_kind": "fp=0" - }, - { - "k": 3, - "agg": "mean", - "best": { - "theta_c": 0.8, - "theta_e": 0.5, - "catch": 0.6667, - "fp": 0.0 - }, - "best_kind": "fp=0" - }, - { - "k": 4, - "agg": "mean", - "best": { - "theta_c": 0.7, - "theta_e": 0.5, - "catch": 0.6667, - "fp": 0.0 - }, - "best_kind": "fp=0" - }, - { - "k": 6, - "agg": "mean", - "best": { - "theta_c": 0.7, - "theta_e": 0.5, - "catch": 0.6667, - "fp": 0.0 - }, - "best_kind": "fp=0" - }, - { - "k": 12, - "agg": "mean", - "best": { - "theta_c": 0.7, - "theta_e": 0.5, - "catch": 0.6667, - "fp": 0.0 - }, - "best_kind": "fp=0" - }, - { - "k": 3, - "agg": "max", - "best": { - "theta_c": 0.99, - "theta_e": 0.5, - "catch": 0.4167, - "fp": 0.0 - }, - "best_kind": "fp=0" - }, - { - "k": 4, - "agg": "max", - "best": { - "theta_c": 0.99, - "theta_e": 0.5, - "catch": 0.4167, - "fp": 0.0 - }, - "best_kind": "fp=0" - }, - { - "k": 6, - "agg": "max", - "best": { - "theta_c": 0.99, - "theta_e": 0.5, - "catch": 0.4167, - "fp": 0.0 - }, - "best_kind": "fp=0" - }, - { - "k": 12, - "agg": "max", - "best": { - "theta_c": 0.99, - "theta_e": 0.5, - "catch": 0.4167, - "fp": 0.0 - }, - "best_kind": "fp=0" - }, - { - "k": 2, - "agg": "top2", - "best": { - "theta_c": 0.8, - "theta_e": 0.5, - "catch": 0.25, - "fp": 0.0 - }, - "best_kind": "fp=0" - }, - { - "k": 3, - "agg": "top2", - "best": { - "theta_c": 0.8, - "theta_e": 0.5, - "catch": 0.25, - "fp": 0.0 - }, - "best_kind": "fp=0" - }, - { - "k": 4, - "agg": "top2", - "best": { - "theta_c": 0.8, - "theta_e": 0.5, - "catch": 0.25, - "fp": 0.0 - }, - "best_kind": "fp=0" - }, - { - "k": 6, - "agg": "top2", - "best": { - "theta_c": 0.97, - "theta_e": 0.5, - "catch": 0.0833, - "fp": 0.0 - }, - "best_kind": "fp=0" - }, - { - "k": 12, - "agg": "top2", - "best": { - "theta_c": 0.97, - "theta_e": 0.5, - "catch": 0.0833, - "fp": 0.0 - }, - "best_kind": "fp=0" - }, - { - "k": 1, - "agg": "top2", - "best": { - "theta_c": 0.3, - "theta_e": 0.5, - "catch": 0.0, - "fp": 0.0 - }, - "best_kind": "fp=0" - } - ] - } - } -} \ No newline at end of file