nli_shadow_grid: expand to no-stone-unturned sweep — 7 aggregations (incl. margin + paired-entail guard), finer θc grid (20 pts up to 0.999), --extra-models flag, global-best-fp=0 + Pareto-frontier output per model
This commit is contained in:
parent
421b2c5fd5
commit
c452f0d58a
1 changed files with 116 additions and 57 deletions
|
|
@ -43,10 +43,19 @@ from pathlib import Path
|
|||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
KMAX = 12 # NLI is run on the top-KMAX candidate clauses; the grid sub-selects k <= KMAX.
|
||||
K_GRID = (1, 2, 3, 4, 6, 12)
|
||||
AGGS = ("max", "mean", "top2") # how to combine per-clause contradiction scores
|
||||
THETA_C = (0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.85, 0.90, 0.93, 0.95, 0.97, 0.99)
|
||||
THETA_E = (0.50, 0.70, 0.90, 1.01) # the entailment guard; 1.01 == "no guard"
|
||||
K_GRID = (1, 2, 3, 4, 6, 8, 12)
|
||||
# how to combine the per-clause contradiction scores over the top-k clauses:
|
||||
# max / mean — over all k; top2 / top3 — the 2nd / 3rd highest (needs agreement);
|
||||
# meanTop2 / meanTop3 — mean of the 2 / 3 highest; margin — max over clauses of
|
||||
# (p_contra - p_entail) of THAT clause (folds the entailment guard into the score).
|
||||
AGGS = ("max", "mean", "top2", "top3", "meanTop2", "meanTop3", "margin")
|
||||
# the entailment guard variant: max_entail = max p_entail over the top-k clauses;
|
||||
# paired_entail = p_entail of the SAME clause that produced the aggregated contra
|
||||
# (the principled version — "the contradicting clause itself doesn't entail").
|
||||
GUARDS = ("max_entail", "paired_entail")
|
||||
THETA_C = (0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.85, 0.90, 0.92, 0.94, 0.95, 0.96,
|
||||
0.97, 0.975, 0.98, 0.985, 0.99, 0.995, 0.998, 0.999)
|
||||
THETA_E = (0.30, 0.50, 0.70, 0.90, 0.95, 1.01) # 1.01 == "no guard"
|
||||
|
||||
|
||||
def _records(path: Path):
|
||||
|
|
@ -82,16 +91,31 @@ def _models_from_manifest(which: str | None):
|
|||
return sel or entries[:1]
|
||||
|
||||
|
||||
def _agg_contra(scores_pc: list[float], how: str) -> float:
|
||||
if not scores_pc:
|
||||
return 0.0
|
||||
def _aggregate(top: list[tuple], how: str) -> tuple[float, int]:
|
||||
"""Return (aggregated_contradiction_score, idx_of_the_clause_it_came_from)
|
||||
over `top` = [(overlap, pe, pn, pc), ...] (top-k clauses, overlap-ranked).
|
||||
For pooled aggs (mean/topN/meanTopN) the 'idx' is the highest-pc clause in
|
||||
the pool — used for the paired-entail guard."""
|
||||
if not top:
|
||||
return 0.0, -1
|
||||
pcs = [t[3] for t in top]
|
||||
order = sorted(range(len(top)), key=lambda i: pcs[i], reverse=True)
|
||||
if how == "max":
|
||||
return max(scores_pc)
|
||||
return pcs[order[0]], order[0]
|
||||
if how == "mean":
|
||||
return sum(scores_pc) / len(scores_pc)
|
||||
if how == "top2": # 2nd-highest — needs >=2 clauses to "agree"; one outlier doesn't trip it
|
||||
s = sorted(scores_pc, reverse=True)
|
||||
return s[1] if len(s) >= 2 else 0.0
|
||||
return sum(pcs) / len(pcs), order[0]
|
||||
if how == "top2":
|
||||
return (pcs[order[1]], order[1]) if len(order) >= 2 else (0.0, order[0])
|
||||
if how == "top3":
|
||||
return (pcs[order[2]], order[2]) if len(order) >= 3 else (0.0, order[-1])
|
||||
if how == "meanTop2":
|
||||
sel = order[:2]; return sum(pcs[i] for i in sel) / len(sel), order[0]
|
||||
if how == "meanTop3":
|
||||
sel = order[:3]; return sum(pcs[i] for i in sel) / len(sel), order[0]
|
||||
if how == "margin": # max over clauses of (pc - pe); guard folded in
|
||||
margins = [t[3] - t[1] for t in top]
|
||||
bi = max(range(len(top)), key=lambda i: margins[i])
|
||||
return margins[bi], bi
|
||||
raise ValueError(how)
|
||||
|
||||
|
||||
|
|
@ -110,24 +134,44 @@ def _cache_scores(nli, recs: list[dict]) -> list[dict]:
|
|||
return out
|
||||
|
||||
|
||||
def _rate(cached: list[dict], k: int, agg: str, tc: float, te: float) -> float:
|
||||
"""fraction of records that would_demote under (k, agg, θc, θe)."""
|
||||
def _rate(cached: list[dict], k: int, agg: str, guard: str, tc: float, te: float) -> float:
|
||||
"""fraction of records that would_demote under (k, agg, guard, θc, θe).
|
||||
For agg='margin' the guard is folded into the score (margin = pc-pe), so
|
||||
only θc applies; the explicit guard loop is skipped for it (te is ignored)."""
|
||||
n = len(cached)
|
||||
if not n:
|
||||
return 0.0
|
||||
fired = 0
|
||||
is_margin = (agg == "margin")
|
||||
for c in cached:
|
||||
top = c["clause_scores"][:k]
|
||||
if not top:
|
||||
continue
|
||||
max_e = max((pe for _, pe, _, _ in top), default=0.0)
|
||||
contra_vec = [pc for _, _, _, pc in top]
|
||||
agg_c = _agg_contra(contra_vec, agg)
|
||||
if agg_c >= tc and max_e < te:
|
||||
agg_c, src_i = _aggregate(top, agg)
|
||||
if agg_c < tc:
|
||||
continue
|
||||
if is_margin:
|
||||
fired += 1
|
||||
continue
|
||||
if guard == "max_entail":
|
||||
guard_e = max(t[1] for t in top)
|
||||
else: # paired_entail — the entail of the clause that produced agg_c
|
||||
guard_e = top[src_i][1] if 0 <= src_i < len(top) else max(t[1] for t in top)
|
||||
if guard_e < te:
|
||||
fired += 1
|
||||
return fired / n
|
||||
|
||||
|
||||
def _pareto(pts: list[dict]) -> list[dict]:
|
||||
"""Pareto frontier: sorted by fp asc, keep a point only if its catch beats
|
||||
the best catch seen at a lower-or-equal fp."""
|
||||
out, best = [], -1.0
|
||||
for p in sorted(pts, key=lambda p: (p["fp"], -p["catch"])):
|
||||
if p["catch"] > best:
|
||||
out.append(p); best = p["catch"]
|
||||
return out
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--should-demote", type=Path,
|
||||
|
|
@ -137,6 +181,7 @@ def main(argv=None) -> int:
|
|||
ap.add_argument("--no-strict-filter", action="store_true",
|
||||
help="don't filter the should-not-demote set to STRICT rows")
|
||||
ap.add_argument("--models", default=None, help="manifest model id(s) comma-sep, or 'all' (default: main only)")
|
||||
ap.add_argument("--extra-models", default=None, help="comma-sep HF repo ids to also test (no manifest pin needed) — for 'no stone unturned' breadth")
|
||||
ap.add_argument("--out", type=Path, default=REPO / "bench" / "results" / "nli-shadow-grid.json")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
|
|
@ -160,12 +205,17 @@ def main(argv=None) -> int:
|
|||
return 2
|
||||
|
||||
models = _models_from_manifest(args.models)
|
||||
print(f"[nli-grid] models: {[m['nli_model_version'] for m in models]}", flush=True)
|
||||
if args.extra_models:
|
||||
for repo in [r.strip() for r in args.extra_models.split(",") if r.strip()]:
|
||||
mv = "extra:" + repo.split("/")[-1]
|
||||
if not any(m["hf_repo"] == repo for m in models):
|
||||
models.append({"nli_model_version": mv, "hf_repo": repo, "pinned_revision": None})
|
||||
print(f"[nli-grid] models ({len(models)}): {[m['nli_model_version'] for m in models]}", flush=True)
|
||||
|
||||
report = {"generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"should_demote_file": str(args.should_demote), "n_should_demote": len(sd),
|
||||
"should_not_demote_file": str(args.should_not_demote), "n_should_not_demote": len(snd),
|
||||
"k_grid": list(K_GRID), "aggs": list(AGGS),
|
||||
"k_grid": list(K_GRID), "aggs": list(AGGS), "guards": list(GUARDS),
|
||||
"theta_c_grid": list(THETA_C), "theta_e_grid": list(THETA_E),
|
||||
"kmax": KMAX, "models": {}}
|
||||
|
||||
|
|
@ -188,56 +238,65 @@ def main(argv=None) -> int:
|
|||
n_pairs = sum(len(c["clause_scores"]) for c in cache_sd + cache_snd)
|
||||
print(f"[nli-grid] backend={nli.backend} device={nli.device} · {n_pairs} NLI pairs in {infer_s:.2f}s", flush=True)
|
||||
|
||||
configs = []
|
||||
configs = [] # best fp=0 (else fp<=.05 / knee) point per (k, agg, guard)
|
||||
all_pts = [] # every (model-internal) point, for the global pareto + global best
|
||||
for k in K_GRID:
|
||||
for agg in AGGS:
|
||||
# frontier over (θc, θe): list of (catch, fp, θc, θe)
|
||||
pts = []
|
||||
for te in THETA_E:
|
||||
for tc in THETA_C:
|
||||
catch = _rate(cache_sd, k, agg, tc, te)
|
||||
fp = _rate(cache_snd, k, agg, tc, te)
|
||||
pts.append({"theta_c": tc, "theta_e": te,
|
||||
"catch": round(catch, 4), "fp": round(fp, 4)})
|
||||
# best point: max catch at fp==0; else max catch among fp<=0.05; else knee
|
||||
zero = [p for p in pts if p["fp"] == 0.0]
|
||||
low = [p for p in pts if p["fp"] <= 0.05]
|
||||
if zero:
|
||||
best = max(zero, key=lambda p: p["catch"]); best_kind = "fp=0"
|
||||
elif low:
|
||||
best = max(low, key=lambda p: p["catch"]); best_kind = "fp<=0.05"
|
||||
else:
|
||||
best = max(pts, key=lambda p: p["catch"] - p["fp"]); best_kind = "knee"
|
||||
configs.append({"k": k, "agg": agg, "best": best, "best_kind": best_kind})
|
||||
# rank configs: prefer fp=0 with highest catch, then fp<=0.05, then knee
|
||||
def _key(c):
|
||||
guard_list = ("max_entail",) if agg == "margin" else GUARDS
|
||||
te_list = (1.01,) if agg == "margin" else THETA_E
|
||||
for guard in guard_list:
|
||||
pts = []
|
||||
for te in te_list:
|
||||
for tc in THETA_C:
|
||||
catch = _rate(cache_sd, k, agg, guard, tc, te)
|
||||
fp = _rate(cache_snd, k, agg, guard, tc, te)
|
||||
p = {"k": k, "agg": agg, "guard": guard, "theta_c": tc,
|
||||
"theta_e": te, "catch": round(catch, 4), "fp": round(fp, 4)}
|
||||
pts.append(p); all_pts.append(p)
|
||||
zero = [p for p in pts if p["fp"] == 0.0]
|
||||
low = [p for p in pts if p["fp"] <= 0.05]
|
||||
if zero:
|
||||
best = max(zero, key=lambda p: p["catch"]); best_kind = "fp=0"
|
||||
elif low:
|
||||
best = max(low, key=lambda p: p["catch"]); best_kind = "fp<=0.05"
|
||||
else:
|
||||
best = max(pts, key=lambda p: p["catch"] - p["fp"]); best_kind = "knee"
|
||||
configs.append({"k": k, "agg": agg, "guard": guard, "best": best, "best_kind": best_kind})
|
||||
def _ck(c):
|
||||
b, kind = c["best"], c["best_kind"]
|
||||
return (0 if kind == "fp=0" else 1 if kind == "fp<=0.05" else 2, -b["catch"], b["fp"])
|
||||
configs.sort(key=_key)
|
||||
configs.sort(key=_ck)
|
||||
# global best fp=0 point across ALL (k,agg,guard,θc,θe), then the pareto frontier
|
||||
zero_all = [p for p in all_pts if p["fp"] == 0.0]
|
||||
global_best = max(zero_all, key=lambda p: p["catch"]) if zero_all else max(all_pts, key=lambda p: p["catch"] - p["fp"])
|
||||
pareto = _pareto(all_pts)
|
||||
report["models"][mv] = {"hf_repo": me["hf_repo"], "backend": nli.backend, "device": nli.device,
|
||||
"n_nli_pairs": n_pairs, "infer_seconds": round(infer_s, 2),
|
||||
"configs_ranked": configs}
|
||||
top = configs[0]
|
||||
print(f"[nli-grid] best config: k={top['k']} agg={top['agg']} → catch={top['best']['catch']} fp={top['best']['fp']} "
|
||||
f"@ θc={top['best']['theta_c']} θe={top['best']['theta_e']} [{top['best_kind']}]", flush=True)
|
||||
"global_best": global_best, "pareto": pareto, "configs_ranked": configs}
|
||||
gb = global_best
|
||||
print(f"[nli-grid] GLOBAL best fp=0: k={gb['k']} agg={gb['agg']} guard={gb['guard']} θc={gb['theta_c']} θe={gb['theta_e']} "
|
||||
f"→ catch={gb['catch']} fp={gb['fp']}", flush=True)
|
||||
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(json.dumps(report, indent=2))
|
||||
print(f"[nli-grid] wrote {args.out}")
|
||||
|
||||
# summary table
|
||||
print("\n" + "=" * 86)
|
||||
print("NLI SHADOW GRID — best config per model (catch on should-demote vs fp on STRICT)")
|
||||
print("=" * 86)
|
||||
print(f"{'model':<36} {'best k':>6} {'agg':>6} {'catch':>7} {'fp':>7} {'θc':>6} {'θe':>6} kind")
|
||||
print("-" * 86)
|
||||
for mv, md in report["models"].items():
|
||||
t = md["configs_ranked"][0]; b = t["best"]
|
||||
print(f"{mv:<36} {t['k']:>6} {t['agg']:>6} {b['catch']:>7.3f} {b['fp']:>7.3f} {b['theta_c']:>6} {b['theta_e']:>6} {t['best_kind']}")
|
||||
print("-" * 86)
|
||||
print("\n" + "=" * 100)
|
||||
print(f"NLI SHADOW GRID — global-best fp=0 config per model ({len(sd)} should-demote vs {len(snd)} should-not-demote)")
|
||||
print("=" * 100)
|
||||
print(f"{'model':<32} {'hf_repo':<44} {'k':>3} {'agg':>9} {'guard':>14} {'θc':>6} {'θe':>5} {'catch':>7} {'fp':>6}")
|
||||
print("-" * 100)
|
||||
ranked = sorted(report["models"].items(), key=lambda kv: (-(kv[1]["global_best"]["catch"] if kv[1]["global_best"]["fp"] == 0 else -1), kv[1]["global_best"]["fp"]))
|
||||
for mv, md in ranked:
|
||||
g = md["global_best"]
|
||||
print(f"{mv:<32} {md['hf_repo']:<44} {g['k']:>3} {g['agg']:>9} {g['guard']:>14} {g['theta_c']:>6} {g['theta_e']:>5} {g['catch']:>7.3f} {g['fp']:>6.3f}")
|
||||
print("-" * 100)
|
||||
print("catch = fraction of should-demote (recombination falsehoods) the veto fires on (higher better)")
|
||||
print("fp = fraction of STRICT (confidently-grounded) answers the veto WRONGLY fires on (≈0 to clear §7 #12)")
|
||||
print("If no model has a 'fp=0' config with a useful catch rate, the lexical-candidate approach doesn't pass — Phase-3 verifier-matched-clause hook is the path.")
|
||||
print("fp = fraction of should-not-demote (confidently-grounded STRICT answers) the veto WRONGLY fires on (≈0 clears §7 #12)")
|
||||
print("'global_best' = the highest-catch point with fp==0 across the FULL {k × agg × guard × θc × θe} grid for that model.")
|
||||
print("Each model's 'pareto' (in the JSON) is the catch-vs-fp frontier — read it to pick an operating point above fp=0 if a small false-demote rate is acceptable.")
|
||||
print("If no model reaches a useful catch at fp=0, the lexical-candidate approach is precision-limited — Phase-3 verifier-matched-clause hook is the path.")
|
||||
return 0
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue