diff --git a/bench/scripts/nli_shadow_grid.py b/bench/scripts/nli_shadow_grid.py index 2ea87a5..f7898c5 100644 --- a/bench/scripts/nli_shadow_grid.py +++ b/bench/scripts/nli_shadow_grid.py @@ -43,10 +43,19 @@ from pathlib import Path REPO = Path(__file__).resolve().parents[2] KMAX = 12 # NLI is run on the top-KMAX candidate clauses; the grid sub-selects k <= KMAX. -K_GRID = (1, 2, 3, 4, 6, 12) -AGGS = ("max", "mean", "top2") # how to combine per-clause contradiction scores -THETA_C = (0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.85, 0.90, 0.93, 0.95, 0.97, 0.99) -THETA_E = (0.50, 0.70, 0.90, 1.01) # the entailment guard; 1.01 == "no guard" +K_GRID = (1, 2, 3, 4, 6, 8, 12) +# how to combine the per-clause contradiction scores over the top-k clauses: +# max / mean — over all k; top2 / top3 — the 2nd / 3rd highest (needs agreement); +# meanTop2 / meanTop3 — mean of the 2 / 3 highest; margin — max over clauses of +# (p_contra - p_entail) of THAT clause (folds the entailment guard into the score). +AGGS = ("max", "mean", "top2", "top3", "meanTop2", "meanTop3", "margin") +# the entailment guard variant: max_entail = max p_entail over the top-k clauses; +# paired_entail = p_entail of the SAME clause that produced the aggregated contra +# (the principled version — "the contradicting clause itself doesn't entail"). +GUARDS = ("max_entail", "paired_entail") +THETA_C = (0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.85, 0.90, 0.92, 0.94, 0.95, 0.96, + 0.97, 0.975, 0.98, 0.985, 0.99, 0.995, 0.998, 0.999) +THETA_E = (0.30, 0.50, 0.70, 0.90, 0.95, 1.01) # 1.01 == "no guard" def _records(path: Path): @@ -82,16 +91,31 @@ def _models_from_manifest(which: str | None): return sel or entries[:1] -def _agg_contra(scores_pc: list[float], how: str) -> float: - if not scores_pc: - return 0.0 +def _aggregate(top: list[tuple], how: str) -> tuple[float, int]: + """Return (aggregated_contradiction_score, idx_of_the_clause_it_came_from) + over `top` = [(overlap, pe, pn, pc), ...] (top-k clauses, overlap-ranked). + For pooled aggs (mean/topN/meanTopN) the 'idx' is the highest-pc clause in + the pool — used for the paired-entail guard.""" + if not top: + return 0.0, -1 + pcs = [t[3] for t in top] + order = sorted(range(len(top)), key=lambda i: pcs[i], reverse=True) if how == "max": - return max(scores_pc) + return pcs[order[0]], order[0] if how == "mean": - return sum(scores_pc) / len(scores_pc) - if how == "top2": # 2nd-highest — needs >=2 clauses to "agree"; one outlier doesn't trip it - s = sorted(scores_pc, reverse=True) - return s[1] if len(s) >= 2 else 0.0 + return sum(pcs) / len(pcs), order[0] + if how == "top2": + return (pcs[order[1]], order[1]) if len(order) >= 2 else (0.0, order[0]) + if how == "top3": + return (pcs[order[2]], order[2]) if len(order) >= 3 else (0.0, order[-1]) + if how == "meanTop2": + sel = order[:2]; return sum(pcs[i] for i in sel) / len(sel), order[0] + if how == "meanTop3": + sel = order[:3]; return sum(pcs[i] for i in sel) / len(sel), order[0] + if how == "margin": # max over clauses of (pc - pe); guard folded in + margins = [t[3] - t[1] for t in top] + bi = max(range(len(top)), key=lambda i: margins[i]) + return margins[bi], bi raise ValueError(how) @@ -110,24 +134,44 @@ def _cache_scores(nli, recs: list[dict]) -> list[dict]: return out -def _rate(cached: list[dict], k: int, agg: str, tc: float, te: float) -> float: - """fraction of records that would_demote under (k, agg, θc, θe).""" +def _rate(cached: list[dict], k: int, agg: str, guard: str, tc: float, te: float) -> float: + """fraction of records that would_demote under (k, agg, guard, θc, θe). + For agg='margin' the guard is folded into the score (margin = pc-pe), so + only θc applies; the explicit guard loop is skipped for it (te is ignored).""" n = len(cached) if not n: return 0.0 fired = 0 + is_margin = (agg == "margin") for c in cached: top = c["clause_scores"][:k] if not top: continue - max_e = max((pe for _, pe, _, _ in top), default=0.0) - contra_vec = [pc for _, _, _, pc in top] - agg_c = _agg_contra(contra_vec, agg) - if agg_c >= tc and max_e < te: + agg_c, src_i = _aggregate(top, agg) + if agg_c < tc: + continue + if is_margin: + fired += 1 + continue + if guard == "max_entail": + guard_e = max(t[1] for t in top) + else: # paired_entail — the entail of the clause that produced agg_c + guard_e = top[src_i][1] if 0 <= src_i < len(top) else max(t[1] for t in top) + if guard_e < te: fired += 1 return fired / n +def _pareto(pts: list[dict]) -> list[dict]: + """Pareto frontier: sorted by fp asc, keep a point only if its catch beats + the best catch seen at a lower-or-equal fp.""" + out, best = [], -1.0 + for p in sorted(pts, key=lambda p: (p["fp"], -p["catch"])): + if p["catch"] > best: + out.append(p); best = p["catch"] + return out + + def main(argv=None) -> int: ap = argparse.ArgumentParser() ap.add_argument("--should-demote", type=Path, @@ -137,6 +181,7 @@ def main(argv=None) -> int: ap.add_argument("--no-strict-filter", action="store_true", help="don't filter the should-not-demote set to STRICT rows") ap.add_argument("--models", default=None, help="manifest model id(s) comma-sep, or 'all' (default: main only)") + ap.add_argument("--extra-models", default=None, help="comma-sep HF repo ids to also test (no manifest pin needed) — for 'no stone unturned' breadth") ap.add_argument("--out", type=Path, default=REPO / "bench" / "results" / "nli-shadow-grid.json") args = ap.parse_args(argv) @@ -160,12 +205,17 @@ def main(argv=None) -> int: return 2 models = _models_from_manifest(args.models) - print(f"[nli-grid] models: {[m['nli_model_version'] for m in models]}", flush=True) + if args.extra_models: + for repo in [r.strip() for r in args.extra_models.split(",") if r.strip()]: + mv = "extra:" + repo.split("/")[-1] + if not any(m["hf_repo"] == repo for m in models): + models.append({"nli_model_version": mv, "hf_repo": repo, "pinned_revision": None}) + print(f"[nli-grid] models ({len(models)}): {[m['nli_model_version'] for m in models]}", flush=True) report = {"generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "should_demote_file": str(args.should_demote), "n_should_demote": len(sd), "should_not_demote_file": str(args.should_not_demote), "n_should_not_demote": len(snd), - "k_grid": list(K_GRID), "aggs": list(AGGS), + "k_grid": list(K_GRID), "aggs": list(AGGS), "guards": list(GUARDS), "theta_c_grid": list(THETA_C), "theta_e_grid": list(THETA_E), "kmax": KMAX, "models": {}} @@ -188,56 +238,65 @@ def main(argv=None) -> int: n_pairs = sum(len(c["clause_scores"]) for c in cache_sd + cache_snd) print(f"[nli-grid] backend={nli.backend} device={nli.device} · {n_pairs} NLI pairs in {infer_s:.2f}s", flush=True) - configs = [] + configs = [] # best fp=0 (else fp<=.05 / knee) point per (k, agg, guard) + all_pts = [] # every (model-internal) point, for the global pareto + global best for k in K_GRID: for agg in AGGS: - # frontier over (θc, θe): list of (catch, fp, θc, θe) - pts = [] - for te in THETA_E: - for tc in THETA_C: - catch = _rate(cache_sd, k, agg, tc, te) - fp = _rate(cache_snd, k, agg, tc, te) - pts.append({"theta_c": tc, "theta_e": te, - "catch": round(catch, 4), "fp": round(fp, 4)}) - # best point: max catch at fp==0; else max catch among fp<=0.05; else knee - zero = [p for p in pts if p["fp"] == 0.0] - low = [p for p in pts if p["fp"] <= 0.05] - if zero: - best = max(zero, key=lambda p: p["catch"]); best_kind = "fp=0" - elif low: - best = max(low, key=lambda p: p["catch"]); best_kind = "fp<=0.05" - else: - best = max(pts, key=lambda p: p["catch"] - p["fp"]); best_kind = "knee" - configs.append({"k": k, "agg": agg, "best": best, "best_kind": best_kind}) - # rank configs: prefer fp=0 with highest catch, then fp<=0.05, then knee - def _key(c): + guard_list = ("max_entail",) if agg == "margin" else GUARDS + te_list = (1.01,) if agg == "margin" else THETA_E + for guard in guard_list: + pts = [] + for te in te_list: + for tc in THETA_C: + catch = _rate(cache_sd, k, agg, guard, tc, te) + fp = _rate(cache_snd, k, agg, guard, tc, te) + p = {"k": k, "agg": agg, "guard": guard, "theta_c": tc, + "theta_e": te, "catch": round(catch, 4), "fp": round(fp, 4)} + pts.append(p); all_pts.append(p) + zero = [p for p in pts if p["fp"] == 0.0] + low = [p for p in pts if p["fp"] <= 0.05] + if zero: + best = max(zero, key=lambda p: p["catch"]); best_kind = "fp=0" + elif low: + best = max(low, key=lambda p: p["catch"]); best_kind = "fp<=0.05" + else: + best = max(pts, key=lambda p: p["catch"] - p["fp"]); best_kind = "knee" + configs.append({"k": k, "agg": agg, "guard": guard, "best": best, "best_kind": best_kind}) + def _ck(c): b, kind = c["best"], c["best_kind"] return (0 if kind == "fp=0" else 1 if kind == "fp<=0.05" else 2, -b["catch"], b["fp"]) - configs.sort(key=_key) + configs.sort(key=_ck) + # global best fp=0 point across ALL (k,agg,guard,θc,θe), then the pareto frontier + zero_all = [p for p in all_pts if p["fp"] == 0.0] + global_best = max(zero_all, key=lambda p: p["catch"]) if zero_all else max(all_pts, key=lambda p: p["catch"] - p["fp"]) + pareto = _pareto(all_pts) report["models"][mv] = {"hf_repo": me["hf_repo"], "backend": nli.backend, "device": nli.device, "n_nli_pairs": n_pairs, "infer_seconds": round(infer_s, 2), - "configs_ranked": configs} - top = configs[0] - print(f"[nli-grid] best config: k={top['k']} agg={top['agg']} → catch={top['best']['catch']} fp={top['best']['fp']} " - f"@ θc={top['best']['theta_c']} θe={top['best']['theta_e']} [{top['best_kind']}]", flush=True) + "global_best": global_best, "pareto": pareto, "configs_ranked": configs} + gb = global_best + print(f"[nli-grid] GLOBAL best fp=0: k={gb['k']} agg={gb['agg']} guard={gb['guard']} θc={gb['theta_c']} θe={gb['theta_e']} " + f"→ catch={gb['catch']} fp={gb['fp']}", flush=True) args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text(json.dumps(report, indent=2)) print(f"[nli-grid] wrote {args.out}") # summary table - print("\n" + "=" * 86) - print("NLI SHADOW GRID — best config per model (catch on should-demote vs fp on STRICT)") - print("=" * 86) - print(f"{'model':<36} {'best k':>6} {'agg':>6} {'catch':>7} {'fp':>7} {'θc':>6} {'θe':>6} kind") - print("-" * 86) - for mv, md in report["models"].items(): - t = md["configs_ranked"][0]; b = t["best"] - print(f"{mv:<36} {t['k']:>6} {t['agg']:>6} {b['catch']:>7.3f} {b['fp']:>7.3f} {b['theta_c']:>6} {b['theta_e']:>6} {t['best_kind']}") - print("-" * 86) + print("\n" + "=" * 100) + print(f"NLI SHADOW GRID — global-best fp=0 config per model ({len(sd)} should-demote vs {len(snd)} should-not-demote)") + print("=" * 100) + print(f"{'model':<32} {'hf_repo':<44} {'k':>3} {'agg':>9} {'guard':>14} {'θc':>6} {'θe':>5} {'catch':>7} {'fp':>6}") + print("-" * 100) + ranked = sorted(report["models"].items(), key=lambda kv: (-(kv[1]["global_best"]["catch"] if kv[1]["global_best"]["fp"] == 0 else -1), kv[1]["global_best"]["fp"])) + for mv, md in ranked: + g = md["global_best"] + print(f"{mv:<32} {md['hf_repo']:<44} {g['k']:>3} {g['agg']:>9} {g['guard']:>14} {g['theta_c']:>6} {g['theta_e']:>5} {g['catch']:>7.3f} {g['fp']:>6.3f}") + print("-" * 100) print("catch = fraction of should-demote (recombination falsehoods) the veto fires on (higher better)") - print("fp = fraction of STRICT (confidently-grounded) answers the veto WRONGLY fires on (≈0 to clear §7 #12)") - print("If no model has a 'fp=0' config with a useful catch rate, the lexical-candidate approach doesn't pass — Phase-3 verifier-matched-clause hook is the path.") + print("fp = fraction of should-not-demote (confidently-grounded STRICT answers) the veto WRONGLY fires on (≈0 clears §7 #12)") + print("'global_best' = the highest-catch point with fp==0 across the FULL {k × agg × guard × θc × θe} grid for that model.") + print("Each model's 'pareto' (in the JSON) is the catch-vs-fp frontier — read it to pick an operating point above fp=0 if a small false-demote rate is acceptable.") + print("If no model reaches a useful catch at fp=0, the lexical-candidate approach is precision-limited — Phase-3 verifier-matched-clause hook is the path.") return 0