#!/usr/bin/env python3 """Join the live es+sandwich sweep against the deterministic round-trip buckets → does MT round-trip drift PREDICT grounding loss? sweep JSONL (question=es, audit_mode, status) ⋈ es_roundtrip.json (es → bucket CLEAN/DRIFT/COLLAPSE) → crosstab bucket × audit_mode, plus the MT-engine error rate (the concurrency defect the fan-out surfaced) reported separately so it doesn't get conflated with the MT-quality signal. """ from __future__ import annotations import json import sys from collections import Counter, defaultdict from pathlib import Path ROOT = Path(__file__).resolve().parents[1] RES = ROOT / "bench" / "qa_results" def main() -> int: sweep_path = Path(sys.argv[1]) if len(sys.argv) > 1 else max( RES.glob("2026-*.jsonl"), key=lambda p: p.stat().st_mtime) rt = {r["es"]: r for r in json.loads((RES / "es_roundtrip.json").read_text())} rows = [json.loads(ln) for ln in sweep_path.read_text().splitlines() if ln.strip()] err = [r for r in rows if (r.get("status") or "").startswith(("err", "error")) or r.get("audit_mode") is None] ok = [r for r in rows if r not in err] print(f"sweep: {sweep_path.name} rows={len(rows)} " f"engine/other ERROR={len(err)} ({len(err)/max(1,len(rows)):.0%}) " f"scored={len(ok)}") ct = defaultdict(Counter) unmatched = 0 for r in ok: b = rt.get(r["question"], {}).get("bucket") if b is None: unmatched += 1 continue ct[b][r.get("audit_mode") or "?"] += 1 print("\nbucket × audit_mode (scored rows only):") modes = ["STRICT", "HYBRID", "UNGROUNDED", "?"] print(f" {'bucket':<9} " + " ".join(f"{m:>10}" for m in modes) + " ground%") for b in ("CLEAN", "DRIFT", "COLLAPSE"): c = ct.get(b, Counter()) tot = sum(c.values()) g = c["STRICT"] + c["HYBRID"] print(f" {b:<9} " + " ".join(f"{c[m]:>10}" for m in modes) + f" {g}/{tot}" + (f" ({g/tot:.0%})" if tot else "")) if unmatched: print(f" ({unmatched} scored rows had no round-trip match)") print("\nreading: if CLEAN grounds >> COLLAPSE, MT round-trip drift " "predicts grounding loss — the lever is entity-preserving MT, " "not the sandwich architecture.") return 0 if __name__ == "__main__": raise SystemExit(main())