russell@unturf.com
9f528b4ac2
First grid (MiniLM, n=1 bench-qa 89 STRICT cells): the §7 #22 'fails the gate' verdict was config-specific (defaults k=6/θc=0.5/θe=0.9) — the grid finds k=1/θc=0.93/θe=0.5 → catch 8/12 falsification-hard (incl. BOTH recombination fixtures hard-003 Mercury + hard-005 Einstein), 0/89 STRICT FP, 0/26 synthetic-legit FP; 18/28 synthetic recombination. So the lexical-candidate approach is NOT a dead end. (k=1 = no haystack; θe=0.5 tighter than the clean-set 0.9.) Full multi-model + n=3 characterization next.
245 lines
12 KiB
Python
245 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""NLI shadow GRID sweep — #000049 §7 #22 follow-up: characterize the
|
||
whole {model × candidate-clause-cap × aggregation × θc × θe} space at
|
||
once, to learn whether *any* config of the lexical-candidate approach
|
||
clears the §7 #12 gate (catch the recombinations, ~0 false-demote on
|
||
already-STRICT answers) before deciding the Phase-3 verifier-matched-
|
||
clause hook is the only path.
|
||
|
||
Inputs (both shapes from `nli_shadow_sweep.py` are accepted per record:
|
||
`{answer_text|claim, context|source}`; `_meta` lines skipped):
|
||
--should-demote records where demoting IS correct (false claims /
|
||
recombination fixtures). Default: the in-repo
|
||
`bench/fixtures/5f/falsification-hard-v1.jsonl`
|
||
(12; the 2 recombination fixtures + 10 other false
|
||
near-misses). For the richer 28-case synthetic set
|
||
point this at `arborist-nli-bench/eval/recombination.jsonl`.
|
||
--should-not-demote records where demoting is a FALSE POSITIVE — a
|
||
bench-qa JSONL; rows are filtered to `audit_mode ==
|
||
STRICT` (the confidently-grounded answers) unless
|
||
--no-strict-filter. (Add `arborist-nli-bench/eval/
|
||
legit_summary.jsonl` too if you have it.)
|
||
|
||
How it stays cheap: NLI inference runs ONCE per (model, record) over
|
||
the record's top-`KMAX` candidate clauses; the k/aggregation/θ grid is
|
||
then pure arithmetic on the cached per-clause scores. On a GPU the
|
||
whole grid is seconds. Models come from `arborist/qa/nli/manifest.json`
|
||
(`nli_model_version`+`hf_repo`+`pinned_revision`, plus `alternates`).
|
||
|
||
SHADOW ONLY — writes nothing but `bench/results/nli-shadow-grid.json`.
|
||
|
||
Usage:
|
||
python3 bench/scripts/nli_shadow_grid.py --should-not-demote bench/qa_results/<...>.jsonl
|
||
python3 bench/scripts/nli_shadow_grid.py --should-not-demote a.jsonl --should-demote ~/git/arborist-nli-bench/eval/recombination.jsonl --models all
|
||
python3 bench/scripts/nli_shadow_grid.py --should-not-demote a.jsonl --models nli-shadow-v1-minilm2-l6-h768,nli-shadow-v1-bart-large-mnli
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
REPO = Path(__file__).resolve().parents[2]
|
||
KMAX = 12 # NLI is run on the top-KMAX candidate clauses; the grid sub-selects k <= KMAX.
|
||
K_GRID = (1, 2, 3, 4, 6, 12)
|
||
AGGS = ("max", "mean", "top2") # how to combine per-clause contradiction scores
|
||
THETA_C = (0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.85, 0.90, 0.93, 0.95, 0.97, 0.99)
|
||
THETA_E = (0.50, 0.70, 0.90, 1.01) # the entailment guard; 1.01 == "no guard"
|
||
|
||
|
||
def _records(path: Path):
|
||
for ln in path.read_text().splitlines():
|
||
ln = ln.strip()
|
||
if not ln:
|
||
continue
|
||
o = json.loads(ln)
|
||
if "_meta" in o:
|
||
continue
|
||
claim = o.get("answer_text") or o.get("claim")
|
||
source = o.get("context") or o.get("source")
|
||
if not claim or not source:
|
||
continue
|
||
yield {"id": o.get("id", path.stem), "claim": claim, "source": source,
|
||
"audit_mode": o.get("audit_mode")}
|
||
|
||
|
||
def _models_from_manifest(which: str | None):
|
||
sys.path.insert(0, str(REPO))
|
||
from arborist.qa.nli.shadow import load_manifest
|
||
m = load_manifest()
|
||
entries = [{"nli_model_version": m["nli_model_version"], "hf_repo": m["hf_repo"],
|
||
"pinned_revision": m.get("pinned_revision")}]
|
||
entries += [{"nli_model_version": a["nli_model_version"], "hf_repo": a["hf_repo"],
|
||
"pinned_revision": a.get("pinned_revision")} for a in m.get("alternates", [])]
|
||
if which in (None, ""):
|
||
return entries[:1]
|
||
if which == "all":
|
||
return entries
|
||
want = {w.strip() for w in which.split(",")}
|
||
sel = [e for e in entries if e["nli_model_version"] in want or e["hf_repo"] in want]
|
||
return sel or entries[:1]
|
||
|
||
|
||
def _agg_contra(scores_pc: list[float], how: str) -> float:
|
||
if not scores_pc:
|
||
return 0.0
|
||
if how == "max":
|
||
return max(scores_pc)
|
||
if how == "mean":
|
||
return sum(scores_pc) / len(scores_pc)
|
||
if how == "top2": # 2nd-highest — needs >=2 clauses to "agree"; one outlier doesn't trip it
|
||
s = sorted(scores_pc, reverse=True)
|
||
return s[1] if len(s) >= 2 else 0.0
|
||
raise ValueError(how)
|
||
|
||
|
||
def _cache_scores(nli, recs: list[dict]) -> list[dict]:
|
||
"""For each record: segment, take top-KMAX candidate clauses, NLI each.
|
||
Returns [{id, audit_mode, n_clauses, clause_scores=[(overlap,pe,pn,pc),...]}]."""
|
||
from arborist.qa.nli.shadow import clauses as _clauses, candidate_clauses as _cand
|
||
out = []
|
||
for r in recs:
|
||
n_clauses = len(_clauses(r["source"]))
|
||
cand = _cand(r["claim"], r["source"], KMAX) # list[(clause, overlap)]
|
||
triples = nli._nli_batch([(cl, r["claim"]) for cl, _ in cand]) if cand else []
|
||
clause_scores = [(ov, pe, pn, pc) for (_, ov), (pe, pn, pc) in zip(cand, triples)]
|
||
out.append({"id": r["id"], "audit_mode": r.get("audit_mode"),
|
||
"n_clauses": n_clauses, "clause_scores": clause_scores})
|
||
return out
|
||
|
||
|
||
def _rate(cached: list[dict], k: int, agg: str, tc: float, te: float) -> float:
|
||
"""fraction of records that would_demote under (k, agg, θc, θe)."""
|
||
n = len(cached)
|
||
if not n:
|
||
return 0.0
|
||
fired = 0
|
||
for c in cached:
|
||
top = c["clause_scores"][:k]
|
||
if not top:
|
||
continue
|
||
max_e = max((pe for _, pe, _, _ in top), default=0.0)
|
||
contra_vec = [pc for _, _, _, pc in top]
|
||
agg_c = _agg_contra(contra_vec, agg)
|
||
if agg_c >= tc and max_e < te:
|
||
fired += 1
|
||
return fired / n
|
||
|
||
|
||
def main(argv=None) -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--should-demote", type=Path,
|
||
default=REPO / "bench" / "fixtures" / "5f" / "falsification-hard-v1.jsonl")
|
||
ap.add_argument("--should-not-demote", type=Path, required=True,
|
||
help="a bench-qa JSONL (rows filtered to audit_mode==STRICT)")
|
||
ap.add_argument("--no-strict-filter", action="store_true",
|
||
help="don't filter the should-not-demote set to STRICT rows")
|
||
ap.add_argument("--models", default=None, help="manifest model id(s) comma-sep, or 'all' (default: main only)")
|
||
ap.add_argument("--out", type=Path, default=REPO / "bench" / "results" / "nli-shadow-grid.json")
|
||
args = ap.parse_args(argv)
|
||
|
||
try:
|
||
import torch # noqa
|
||
sys.path.insert(0, str(REPO))
|
||
from arborist.qa.nli.shadow import ShadowNLI
|
||
except ImportError as e:
|
||
print(f"[nli-grid] missing dependency: {e} (pip install 'arborist[nli]')", file=sys.stderr)
|
||
return 2
|
||
|
||
sd = list(_records(args.should_demote))
|
||
snd = list(_records(args.should_not_demote))
|
||
if not args.no_strict_filter:
|
||
snd_all = len(snd)
|
||
snd = [r for r in snd if r.get("audit_mode") == "STRICT"]
|
||
print(f"[nli-grid] should-not-demote: {len(snd)}/{snd_all} rows are audit_mode==STRICT", flush=True)
|
||
print(f"[nli-grid] should-demote: {len(sd)} records ({args.should_demote.name})", flush=True)
|
||
if not sd or not snd:
|
||
print("[nli-grid] need non-empty inputs on both sides", file=sys.stderr)
|
||
return 2
|
||
|
||
models = _models_from_manifest(args.models)
|
||
print(f"[nli-grid] models: {[m['nli_model_version'] for m in models]}", flush=True)
|
||
|
||
report = {"generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||
"should_demote_file": str(args.should_demote), "n_should_demote": len(sd),
|
||
"should_not_demote_file": str(args.should_not_demote), "n_should_not_demote": len(snd),
|
||
"k_grid": list(K_GRID), "aggs": list(AGGS),
|
||
"theta_c_grid": list(THETA_C), "theta_e_grid": list(THETA_E),
|
||
"kmax": KMAX, "models": {}}
|
||
|
||
for me in models:
|
||
mv = me["nli_model_version"]
|
||
print(f"[nli-grid] === {mv} ({me['hf_repo']}) ===", flush=True)
|
||
manifest_override = {"nli_model_version": mv, "hf_repo": me["hf_repo"],
|
||
"pinned_revision": me.get("pinned_revision"),
|
||
"thresholds": {"contradiction_veto": 0.5, "entailment_block_veto": 0.9},
|
||
"max_length": 256, "max_candidate_clauses": KMAX}
|
||
nli = ShadowNLI(manifest=manifest_override)
|
||
nli._ensure_loaded()
|
||
if not nli.available:
|
||
print(f"[nli-grid] skip — {nli._reason}", file=sys.stderr)
|
||
continue
|
||
t0 = time.time()
|
||
cache_sd = _cache_scores(nli, sd)
|
||
cache_snd = _cache_scores(nli, snd)
|
||
infer_s = time.time() - t0
|
||
n_pairs = sum(len(c["clause_scores"]) for c in cache_sd + cache_snd)
|
||
print(f"[nli-grid] backend={nli.backend} device={nli.device} · {n_pairs} NLI pairs in {infer_s:.2f}s", flush=True)
|
||
|
||
configs = []
|
||
for k in K_GRID:
|
||
for agg in AGGS:
|
||
# frontier over (θc, θe): list of (catch, fp, θc, θe)
|
||
pts = []
|
||
for te in THETA_E:
|
||
for tc in THETA_C:
|
||
catch = _rate(cache_sd, k, agg, tc, te)
|
||
fp = _rate(cache_snd, k, agg, tc, te)
|
||
pts.append({"theta_c": tc, "theta_e": te,
|
||
"catch": round(catch, 4), "fp": round(fp, 4)})
|
||
# best point: max catch at fp==0; else max catch among fp<=0.05; else knee
|
||
zero = [p for p in pts if p["fp"] == 0.0]
|
||
low = [p for p in pts if p["fp"] <= 0.05]
|
||
if zero:
|
||
best = max(zero, key=lambda p: p["catch"]); best_kind = "fp=0"
|
||
elif low:
|
||
best = max(low, key=lambda p: p["catch"]); best_kind = "fp<=0.05"
|
||
else:
|
||
best = max(pts, key=lambda p: p["catch"] - p["fp"]); best_kind = "knee"
|
||
configs.append({"k": k, "agg": agg, "best": best, "best_kind": best_kind})
|
||
# rank configs: prefer fp=0 with highest catch, then fp<=0.05, then knee
|
||
def _key(c):
|
||
b, kind = c["best"], c["best_kind"]
|
||
return (0 if kind == "fp=0" else 1 if kind == "fp<=0.05" else 2, -b["catch"], b["fp"])
|
||
configs.sort(key=_key)
|
||
report["models"][mv] = {"hf_repo": me["hf_repo"], "backend": nli.backend, "device": nli.device,
|
||
"n_nli_pairs": n_pairs, "infer_seconds": round(infer_s, 2),
|
||
"configs_ranked": configs}
|
||
top = configs[0]
|
||
print(f"[nli-grid] best config: k={top['k']} agg={top['agg']} → catch={top['best']['catch']} fp={top['best']['fp']} "
|
||
f"@ θc={top['best']['theta_c']} θe={top['best']['theta_e']} [{top['best_kind']}]", flush=True)
|
||
|
||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||
args.out.write_text(json.dumps(report, indent=2))
|
||
print(f"[nli-grid] wrote {args.out}")
|
||
|
||
# summary table
|
||
print("\n" + "=" * 86)
|
||
print("NLI SHADOW GRID — best config per model (catch on should-demote vs fp on STRICT)")
|
||
print("=" * 86)
|
||
print(f"{'model':<36} {'best k':>6} {'agg':>6} {'catch':>7} {'fp':>7} {'θc':>6} {'θe':>6} kind")
|
||
print("-" * 86)
|
||
for mv, md in report["models"].items():
|
||
t = md["configs_ranked"][0]; b = t["best"]
|
||
print(f"{mv:<36} {t['k']:>6} {t['agg']:>6} {b['catch']:>7.3f} {b['fp']:>7.3f} {b['theta_c']:>6} {b['theta_e']:>6} {t['best_kind']}")
|
||
print("-" * 86)
|
||
print("catch = fraction of should-demote (recombination falsehoods) the veto fires on (higher better)")
|
||
print("fp = fraction of STRICT (confidently-grounded) answers the veto WRONGLY fires on (≈0 to clear §7 #12)")
|
||
print("If no model has a 'fp=0' config with a useful catch rate, the lexical-candidate approach doesn't pass — Phase-3 verifier-matched-clause hook is the path.")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|