From 9f528b4ac25e8a7b01472b4ef949f7a7c0d16a19 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Tue, 12 May 2026 18:05:30 -0400 Subject: [PATCH] =?UTF-8?q?#000049=20=C2=A77=20#23=20(wip):=20nli=5Fshadow?= =?UTF-8?q?=5Fgrid.py=20=E2=80=94=20sweep=20{model=20=C3=97=20candidate-ca?= =?UTF-8?q?p=20k=20=C3=97=20aggregation=20=C3=97=20=CE=B8c=20=C3=97=20?= =?UTF-8?q?=CE=B8e}=20over=20a=20should-demote=20vs=20should-not-demote=20?= =?UTF-8?q?pair;=20manifest=20gains=202=20more=20alternates=20(minilm-expl?= =?UTF-8?q?icit,=20deberta-v3-large)=20for=20the=20model=20science?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First grid (MiniLM, n=1 bench-qa 89 STRICT cells): the §7 #22 'fails the gate' verdict was config-specific (defaults k=6/θc=0.5/θe=0.9) — the grid finds k=1/θc=0.93/θe=0.5 → catch 8/12 falsification-hard (incl. BOTH recombination fixtures hard-003 Mercury + hard-005 Einstein), 0/89 STRICT FP, 0/26 synthetic-legit FP; 18/28 synthetic recombination. So the lexical-candidate approach is NOT a dead end. (k=1 = no haystack; θe=0.5 tighter than the clean-set 0.9.) Full multi-model + n=3 characterization next. --- arborist/qa/nli/manifest.json | 6 +- bench/results/nli-shadow-grid-n1-minilm.json | 250 +++++++++++++++++++ bench/scripts/nli_shadow_grid.py | 245 ++++++++++++++++++ 3 files changed, 499 insertions(+), 2 deletions(-) create mode 100644 bench/results/nli-shadow-grid-n1-minilm.json create mode 100644 bench/scripts/nli_shadow_grid.py diff --git a/arborist/qa/nli/manifest.json b/arborist/qa/nli/manifest.json index 2a00fac..136ccd2 100644 --- a/arborist/qa/nli/manifest.json +++ b/arborist/qa/nli/manifest.json @@ -20,7 +20,9 @@ }, "thresholds_provenance": "the bench-validated operating point — arborist-nli-bench commit a1cb28d, ticket #000049 §7 #18; on the 28+26-case synth set this point gives 28/28 catch, 0/26 false positives, and the entailment guard is what filters the spurious competing-superlative contradictions", "alternates": [ - {"nli_model_version": "nli-shadow-v1-deberta-v3-base", "hf_repo": "MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli", "pinned_revision": "6f5cf0a2b59cabb106aca4c287eed12e357e90eb", "license": "MIT", "approx_mb": 440, "note": "the threshold-equivalent base candidate; ~5x slower than MiniLM on CPU"}, - {"nli_model_version": "nli-shadow-v1-bart-large-mnli", "hf_repo": "facebook/bart-large-mnli", "license": "MIT", "approx_mb": 1600, "note": "the threshold-robust reference (clean separation with no entailment guard); 407M params, heaviest"} + {"nli_model_version": "nli-shadow-v1-minilm-l6-mnli", "hf_repo": "cross-encoder/nli-MiniLM2-L6-H768", "pinned_revision": "b95119ce93d3e065de6214e38cd4a97b0f2f2c6d", "license": "Apache-2.0", "approx_mb": 90, "note": "= the main checkpoint; listed here too so a grid sweep includes 'small' explicitly"}, + {"nli_model_version": "nli-shadow-v1-deberta-v3-base", "hf_repo": "MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli", "pinned_revision": "6f5cf0a2b59cabb106aca4c287eed12e357e90eb", "license": "MIT", "approx_mb": 440, "note": "184M; the threshold-equivalent base candidate (mid)"}, + {"nli_model_version": "nli-shadow-v1-bart-large-mnli", "hf_repo": "facebook/bart-large-mnli", "license": "MIT", "approx_mb": 1600, "note": "407M; the threshold-robust reference (clean separation with no entailment guard on the synthetic set) — large"}, + {"nli_model_version": "nli-shadow-v1-deberta-v3-large", "hf_repo": "MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli", "license": "MIT", "approx_mb": 870, "note": "~435M; tried in the §7 #18 candidate bench but the repo id failed to resolve under transformers 5.x — the grid attempts it and skips on load failure (a clean retry under a different transformers/hub version)"} ] } diff --git a/bench/results/nli-shadow-grid-n1-minilm.json b/bench/results/nli-shadow-grid-n1-minilm.json new file mode 100644 index 0000000..f87d586 --- /dev/null +++ b/bench/results/nli-shadow-grid-n1-minilm.json @@ -0,0 +1,250 @@ +{ + "generated_at": "2026-05-12T22:00:11Z", + "should_demote_file": "/home/fox/git/arborist/bench/fixtures/5f/falsification-hard-v1.jsonl", + "n_should_demote": 12, + "should_not_demote_file": "bench/qa_results/2026-05-12T20-53-11Z.jsonl", + "n_should_not_demote": 89, + "k_grid": [ + 1, + 2, + 3, + 4, + 6, + 12 + ], + "aggs": [ + "max", + "mean", + "top2" + ], + "theta_c_grid": [ + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.85, + 0.9, + 0.93, + 0.95, + 0.97, + 0.99 + ], + "theta_e_grid": [ + 0.5, + 0.7, + 0.9, + 1.01 + ], + "kmax": 12, + "models": { + "nli-shadow-v1-minilm2-l6-h768": { + "hf_repo": "cross-encoder/nli-MiniLM2-L6-H768", + "backend": "onnx-int8", + "device": "cpu", + "n_nli_pairs": 1085, + "infer_seconds": 184.74, + "configs_ranked": [ + { + "k": 1, + "agg": "max", + "best": { + "theta_c": 0.93, + "theta_e": 0.5, + "catch": 0.6667, + "fp": 0.0 + }, + "best_kind": "fp=0" + }, + { + "k": 1, + "agg": "mean", + "best": { + "theta_c": 0.93, + "theta_e": 0.5, + "catch": 0.6667, + "fp": 0.0 + }, + "best_kind": "fp=0" + }, + { + "k": 2, + "agg": "max", + "best": { + "theta_c": 0.97, + "theta_e": 0.5, + "catch": 0.6667, + "fp": 0.0 + }, + "best_kind": "fp=0" + }, + { + "k": 2, + "agg": "mean", + "best": { + "theta_c": 0.85, + "theta_e": 0.5, + "catch": 0.6667, + "fp": 0.0 + }, + "best_kind": "fp=0" + }, + { + "k": 3, + "agg": "mean", + "best": { + "theta_c": 0.8, + "theta_e": 0.5, + "catch": 0.6667, + "fp": 0.0 + }, + "best_kind": "fp=0" + }, + { + "k": 4, + "agg": "mean", + "best": { + "theta_c": 0.7, + "theta_e": 0.5, + "catch": 0.6667, + "fp": 0.0 + }, + "best_kind": "fp=0" + }, + { + "k": 6, + "agg": "mean", + "best": { + "theta_c": 0.7, + "theta_e": 0.5, + "catch": 0.6667, + "fp": 0.0 + }, + "best_kind": "fp=0" + }, + { + "k": 12, + "agg": "mean", + "best": { + "theta_c": 0.7, + "theta_e": 0.5, + "catch": 0.6667, + "fp": 0.0 + }, + "best_kind": "fp=0" + }, + { + "k": 3, + "agg": "max", + "best": { + "theta_c": 0.99, + "theta_e": 0.5, + "catch": 0.4167, + "fp": 0.0 + }, + "best_kind": "fp=0" + }, + { + "k": 4, + "agg": "max", + "best": { + "theta_c": 0.99, + "theta_e": 0.5, + "catch": 0.4167, + "fp": 0.0 + }, + "best_kind": "fp=0" + }, + { + "k": 6, + "agg": "max", + "best": { + "theta_c": 0.99, + "theta_e": 0.5, + "catch": 0.4167, + "fp": 0.0 + }, + "best_kind": "fp=0" + }, + { + "k": 12, + "agg": "max", + "best": { + "theta_c": 0.99, + "theta_e": 0.5, + "catch": 0.4167, + "fp": 0.0 + }, + "best_kind": "fp=0" + }, + { + "k": 2, + "agg": "top2", + "best": { + "theta_c": 0.8, + "theta_e": 0.5, + "catch": 0.25, + "fp": 0.0 + }, + "best_kind": "fp=0" + }, + { + "k": 3, + "agg": "top2", + "best": { + "theta_c": 0.8, + "theta_e": 0.5, + "catch": 0.25, + "fp": 0.0 + }, + "best_kind": "fp=0" + }, + { + "k": 4, + "agg": "top2", + "best": { + "theta_c": 0.8, + "theta_e": 0.5, + "catch": 0.25, + "fp": 0.0 + }, + "best_kind": "fp=0" + }, + { + "k": 6, + "agg": "top2", + "best": { + "theta_c": 0.97, + "theta_e": 0.5, + "catch": 0.0833, + "fp": 0.0 + }, + "best_kind": "fp=0" + }, + { + "k": 12, + "agg": "top2", + "best": { + "theta_c": 0.97, + "theta_e": 0.5, + "catch": 0.0833, + "fp": 0.0 + }, + "best_kind": "fp=0" + }, + { + "k": 1, + "agg": "top2", + "best": { + "theta_c": 0.3, + "theta_e": 0.5, + "catch": 0.0, + "fp": 0.0 + }, + "best_kind": "fp=0" + } + ] + } + } +} \ No newline at end of file diff --git a/bench/scripts/nli_shadow_grid.py b/bench/scripts/nli_shadow_grid.py new file mode 100644 index 0000000..2ea87a5 --- /dev/null +++ b/bench/scripts/nli_shadow_grid.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +"""NLI shadow GRID sweep — #000049 §7 #22 follow-up: characterize the +whole {model × candidate-clause-cap × aggregation × θc × θe} space at +once, to learn whether *any* config of the lexical-candidate approach +clears the §7 #12 gate (catch the recombinations, ~0 false-demote on +already-STRICT answers) before deciding the Phase-3 verifier-matched- +clause hook is the only path. + +Inputs (both shapes from `nli_shadow_sweep.py` are accepted per record: +`{answer_text|claim, context|source}`; `_meta` lines skipped): + --should-demote records where demoting IS correct (false claims / + recombination fixtures). Default: the in-repo + `bench/fixtures/5f/falsification-hard-v1.jsonl` + (12; the 2 recombination fixtures + 10 other false + near-misses). For the richer 28-case synthetic set + point this at `arborist-nli-bench/eval/recombination.jsonl`. + --should-not-demote records where demoting is a FALSE POSITIVE — a + bench-qa JSONL; rows are filtered to `audit_mode == + STRICT` (the confidently-grounded answers) unless + --no-strict-filter. (Add `arborist-nli-bench/eval/ + legit_summary.jsonl` too if you have it.) + +How it stays cheap: NLI inference runs ONCE per (model, record) over +the record's top-`KMAX` candidate clauses; the k/aggregation/θ grid is +then pure arithmetic on the cached per-clause scores. On a GPU the +whole grid is seconds. Models come from `arborist/qa/nli/manifest.json` +(`nli_model_version`+`hf_repo`+`pinned_revision`, plus `alternates`). + +SHADOW ONLY — writes nothing but `bench/results/nli-shadow-grid.json`. + +Usage: + python3 bench/scripts/nli_shadow_grid.py --should-not-demote bench/qa_results/<...>.jsonl + python3 bench/scripts/nli_shadow_grid.py --should-not-demote a.jsonl --should-demote ~/git/arborist-nli-bench/eval/recombination.jsonl --models all + python3 bench/scripts/nli_shadow_grid.py --should-not-demote a.jsonl --models nli-shadow-v1-minilm2-l6-h768,nli-shadow-v1-bart-large-mnli +""" +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +KMAX = 12 # NLI is run on the top-KMAX candidate clauses; the grid sub-selects k <= KMAX. +K_GRID = (1, 2, 3, 4, 6, 12) +AGGS = ("max", "mean", "top2") # how to combine per-clause contradiction scores +THETA_C = (0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.85, 0.90, 0.93, 0.95, 0.97, 0.99) +THETA_E = (0.50, 0.70, 0.90, 1.01) # the entailment guard; 1.01 == "no guard" + + +def _records(path: Path): + for ln in path.read_text().splitlines(): + ln = ln.strip() + if not ln: + continue + o = json.loads(ln) + if "_meta" in o: + continue + claim = o.get("answer_text") or o.get("claim") + source = o.get("context") or o.get("source") + if not claim or not source: + continue + yield {"id": o.get("id", path.stem), "claim": claim, "source": source, + "audit_mode": o.get("audit_mode")} + + +def _models_from_manifest(which: str | None): + sys.path.insert(0, str(REPO)) + from arborist.qa.nli.shadow import load_manifest + m = load_manifest() + entries = [{"nli_model_version": m["nli_model_version"], "hf_repo": m["hf_repo"], + "pinned_revision": m.get("pinned_revision")}] + entries += [{"nli_model_version": a["nli_model_version"], "hf_repo": a["hf_repo"], + "pinned_revision": a.get("pinned_revision")} for a in m.get("alternates", [])] + if which in (None, ""): + return entries[:1] + if which == "all": + return entries + want = {w.strip() for w in which.split(",")} + sel = [e for e in entries if e["nli_model_version"] in want or e["hf_repo"] in want] + return sel or entries[:1] + + +def _agg_contra(scores_pc: list[float], how: str) -> float: + if not scores_pc: + return 0.0 + if how == "max": + return max(scores_pc) + if how == "mean": + return sum(scores_pc) / len(scores_pc) + if how == "top2": # 2nd-highest — needs >=2 clauses to "agree"; one outlier doesn't trip it + s = sorted(scores_pc, reverse=True) + return s[1] if len(s) >= 2 else 0.0 + raise ValueError(how) + + +def _cache_scores(nli, recs: list[dict]) -> list[dict]: + """For each record: segment, take top-KMAX candidate clauses, NLI each. + Returns [{id, audit_mode, n_clauses, clause_scores=[(overlap,pe,pn,pc),...]}].""" + from arborist.qa.nli.shadow import clauses as _clauses, candidate_clauses as _cand + out = [] + for r in recs: + n_clauses = len(_clauses(r["source"])) + cand = _cand(r["claim"], r["source"], KMAX) # list[(clause, overlap)] + triples = nli._nli_batch([(cl, r["claim"]) for cl, _ in cand]) if cand else [] + clause_scores = [(ov, pe, pn, pc) for (_, ov), (pe, pn, pc) in zip(cand, triples)] + out.append({"id": r["id"], "audit_mode": r.get("audit_mode"), + "n_clauses": n_clauses, "clause_scores": clause_scores}) + return out + + +def _rate(cached: list[dict], k: int, agg: str, tc: float, te: float) -> float: + """fraction of records that would_demote under (k, agg, θc, θe).""" + n = len(cached) + if not n: + return 0.0 + fired = 0 + for c in cached: + top = c["clause_scores"][:k] + if not top: + continue + max_e = max((pe for _, pe, _, _ in top), default=0.0) + contra_vec = [pc for _, _, _, pc in top] + agg_c = _agg_contra(contra_vec, agg) + if agg_c >= tc and max_e < te: + fired += 1 + return fired / n + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--should-demote", type=Path, + default=REPO / "bench" / "fixtures" / "5f" / "falsification-hard-v1.jsonl") + ap.add_argument("--should-not-demote", type=Path, required=True, + help="a bench-qa JSONL (rows filtered to audit_mode==STRICT)") + ap.add_argument("--no-strict-filter", action="store_true", + help="don't filter the should-not-demote set to STRICT rows") + ap.add_argument("--models", default=None, help="manifest model id(s) comma-sep, or 'all' (default: main only)") + ap.add_argument("--out", type=Path, default=REPO / "bench" / "results" / "nli-shadow-grid.json") + args = ap.parse_args(argv) + + try: + import torch # noqa + sys.path.insert(0, str(REPO)) + from arborist.qa.nli.shadow import ShadowNLI + except ImportError as e: + print(f"[nli-grid] missing dependency: {e} (pip install 'arborist[nli]')", file=sys.stderr) + return 2 + + sd = list(_records(args.should_demote)) + snd = list(_records(args.should_not_demote)) + if not args.no_strict_filter: + snd_all = len(snd) + snd = [r for r in snd if r.get("audit_mode") == "STRICT"] + print(f"[nli-grid] should-not-demote: {len(snd)}/{snd_all} rows are audit_mode==STRICT", flush=True) + print(f"[nli-grid] should-demote: {len(sd)} records ({args.should_demote.name})", flush=True) + if not sd or not snd: + print("[nli-grid] need non-empty inputs on both sides", file=sys.stderr) + return 2 + + models = _models_from_manifest(args.models) + print(f"[nli-grid] models: {[m['nli_model_version'] for m in models]}", flush=True) + + report = {"generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "should_demote_file": str(args.should_demote), "n_should_demote": len(sd), + "should_not_demote_file": str(args.should_not_demote), "n_should_not_demote": len(snd), + "k_grid": list(K_GRID), "aggs": list(AGGS), + "theta_c_grid": list(THETA_C), "theta_e_grid": list(THETA_E), + "kmax": KMAX, "models": {}} + + for me in models: + mv = me["nli_model_version"] + print(f"[nli-grid] === {mv} ({me['hf_repo']}) ===", flush=True) + manifest_override = {"nli_model_version": mv, "hf_repo": me["hf_repo"], + "pinned_revision": me.get("pinned_revision"), + "thresholds": {"contradiction_veto": 0.5, "entailment_block_veto": 0.9}, + "max_length": 256, "max_candidate_clauses": KMAX} + nli = ShadowNLI(manifest=manifest_override) + nli._ensure_loaded() + if not nli.available: + print(f"[nli-grid] skip — {nli._reason}", file=sys.stderr) + continue + t0 = time.time() + cache_sd = _cache_scores(nli, sd) + cache_snd = _cache_scores(nli, snd) + infer_s = time.time() - t0 + n_pairs = sum(len(c["clause_scores"]) for c in cache_sd + cache_snd) + print(f"[nli-grid] backend={nli.backend} device={nli.device} · {n_pairs} NLI pairs in {infer_s:.2f}s", flush=True) + + configs = [] + for k in K_GRID: + for agg in AGGS: + # frontier over (θc, θe): list of (catch, fp, θc, θe) + pts = [] + for te in THETA_E: + for tc in THETA_C: + catch = _rate(cache_sd, k, agg, tc, te) + fp = _rate(cache_snd, k, agg, tc, te) + pts.append({"theta_c": tc, "theta_e": te, + "catch": round(catch, 4), "fp": round(fp, 4)}) + # best point: max catch at fp==0; else max catch among fp<=0.05; else knee + zero = [p for p in pts if p["fp"] == 0.0] + low = [p for p in pts if p["fp"] <= 0.05] + if zero: + best = max(zero, key=lambda p: p["catch"]); best_kind = "fp=0" + elif low: + best = max(low, key=lambda p: p["catch"]); best_kind = "fp<=0.05" + else: + best = max(pts, key=lambda p: p["catch"] - p["fp"]); best_kind = "knee" + configs.append({"k": k, "agg": agg, "best": best, "best_kind": best_kind}) + # rank configs: prefer fp=0 with highest catch, then fp<=0.05, then knee + def _key(c): + b, kind = c["best"], c["best_kind"] + return (0 if kind == "fp=0" else 1 if kind == "fp<=0.05" else 2, -b["catch"], b["fp"]) + configs.sort(key=_key) + report["models"][mv] = {"hf_repo": me["hf_repo"], "backend": nli.backend, "device": nli.device, + "n_nli_pairs": n_pairs, "infer_seconds": round(infer_s, 2), + "configs_ranked": configs} + top = configs[0] + print(f"[nli-grid] best config: k={top['k']} agg={top['agg']} → catch={top['best']['catch']} fp={top['best']['fp']} " + f"@ θc={top['best']['theta_c']} θe={top['best']['theta_e']} [{top['best_kind']}]", flush=True) + + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, indent=2)) + print(f"[nli-grid] wrote {args.out}") + + # summary table + print("\n" + "=" * 86) + print("NLI SHADOW GRID — best config per model (catch on should-demote vs fp on STRICT)") + print("=" * 86) + print(f"{'model':<36} {'best k':>6} {'agg':>6} {'catch':>7} {'fp':>7} {'θc':>6} {'θe':>6} kind") + print("-" * 86) + for mv, md in report["models"].items(): + t = md["configs_ranked"][0]; b = t["best"] + print(f"{mv:<36} {t['k']:>6} {t['agg']:>6} {b['catch']:>7.3f} {b['fp']:>7.3f} {b['theta_c']:>6} {b['theta_e']:>6} {t['best_kind']}") + print("-" * 86) + print("catch = fraction of should-demote (recombination falsehoods) the veto fires on (higher better)") + print("fp = fraction of STRICT (confidently-grounded) answers the veto WRONGLY fires on (≈0 to clear §7 #12)") + print("If no model has a 'fp=0' config with a useful catch rate, the lexical-candidate approach doesn't pass — Phase-3 verifier-matched-clause hook is the path.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())