From a2e9b49c0ac230b233ff660685a58656d3bd8dea Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Tue, 19 May 2026 17:42:49 -0400 Subject: [PATCH] feat(#000057): --judge {code,opus} switch in control sweep/AB, default=code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire bench/judge_code.py into the sweep harnesses as the default judge. Both control_sweep.py and control_ab.py grow a --judge {code,opus} CLI arg; both share the same Verdict shape so the dispatch is a pointer assignment + threading the judge_fn through _process_item. Behaviour: - DEFAULT = code: zero LLM, zero quota, deterministic. Self-test gate is the code judge's 4-fixture contract. - --judge opus: original gated Opus path; needs ARBORIST_JUDGE_ENABLE=1 set per 1cabfe6's fail-closed gate, otherwise every record returns JUDGE_ERROR with rationale 'disabled — set ARBORIST_JUDGE_ENABLE=1' and the sweep records that label honestly. Reporting: - Header line now records which judge ran ('Judge = code (...)' or 'Judge = opus (...)') so partial-reports & resumes don't lie about provenance. - Spend banner shows '0 LLM calls' for the code path so the no-burn property is visible in the operator output. Test surface: pytest sweep across tests/ still 136/136 (no regressions); new --judge flag visible in --help on both harnesses. Next: bench/score_with_code_judge.py to re-grade existing sweep JSONLs (written under the gated-Opus run) with the code judge; agreement matrix surfaces residue size for the eventual LLM-batch needle-haystack. --- bench/control_ab.py | 18 +++++++++++++- bench/control_sweep.py | 54 +++++++++++++++++++++++++++++++++--------- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/bench/control_ab.py b/bench/control_ab.py index 2c8a16d..d775a60 100644 --- a/bench/control_ab.py +++ b/bench/control_ab.py @@ -53,7 +53,16 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -from bench.judge import judge # noqa: E402 +# Judge selection at run time (fox 2026-05-19, see CLAUDE.md +# 'Budget discipline'). Both modules share the Verdict shape so the +# downstream record-emit path is judge-agnostic. +import bench.judge as _judge_opus # noqa: E402 +import bench.judge_code as _judge_code # noqa: E402 + +_JUDGES = { + "code": (_judge_code.judge, _judge_code.JUDGE_MODEL), + "opus": (_judge_opus.judge, _judge_opus.JUDGE_MODEL), +} _EVID = re.compile(r"\[E\d+[^\]]*\]", re.S) # [E1 | Title | hash:"…"] _PTR = re.compile(r"\s*\[E\d+(?:\s*,\s*E\d+)*\]") # bare [E1] / [E1,E2] @@ -106,7 +115,14 @@ def main() -> int: "ARBORIST_LLM_MODEL", "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic")) ap.add_argument("--out-dir", default="bench/qa_results") + ap.add_argument("--judge", choices=sorted(_JUDGES.keys()), + default="code", + help="which judge to use. 'code' (default, no LLM) is " + "deterministic / no quota; 'opus' is the gated " + "headless Opus judge (requires " + "ARBORIST_JUDGE_ENABLE=1).") a = ap.parse_args() + judge, _judge_model_id = _JUDGES[a.judge] from arborist.qa.client import OpenAICompatibleClient from arborist.qa.query import DEFAULT_QUERY_POLICY, query diff --git a/bench/control_sweep.py b/bench/control_sweep.py index 4466fd6..1e1dd97 100644 --- a/bench/control_sweep.py +++ b/bench/control_sweep.py @@ -74,7 +74,19 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from bench.control_ab import _SOLO_SYS, _descaffold, _gold # noqa: E402 -from bench.judge import judge, self_test # noqa: E402 +# Judge selection is now run-time via --judge {code,opus} so a sweep can +# collect data WITHOUT burning Opus quota (fox 2026-05-19). Both judges +# share the Verdict shape & verdict vocabulary so swapping is a pointer +# reassignment; nothing downstream changes. Default=code. +import bench.judge as _judge_opus # noqa: E402 +import bench.judge_code as _judge_code # noqa: E402 + +_JUDGES = { + "code": (_judge_code.judge, _judge_code.self_test, + _judge_code.JUDGE_MODEL), + "opus": (_judge_opus.judge, _judge_opus.self_test, + _judge_opus.JUDGE_MODEL), +} MODELS: dict[str, dict] = { "hermes": dict( @@ -122,9 +134,15 @@ VARIANTS = { def _process_item(idx: int, it: dict, variants: list[str], models: list[str], shards_dir: Path, arborist_on: bool, arborist_ref: str, - ts: str) -> list[dict]: + ts: str, judge_fn=None) -> list[dict]: """All variants × models for ONE fixture item. Self-contained: - its own qa_db, its own clients — safe to run concurrently.""" + its own qa_db, its own clients — safe to run concurrently. + + ``judge_fn`` defaults to the code judge for back-compat with any + direct callers that pre-date the --judge switch; main() passes + the user's choice explicitly.""" + if judge_fn is None: + judge_fn = _judge_code.judge from arborist.qa.client import OpenAICompatibleClient from arborist.qa.query import DEFAULT_QUERY_POLICY, query @@ -159,7 +177,7 @@ def _process_item(idx: int, it: dict, variants: list[str], extra_body=cfg["extra"]) except Exception as e: # noqa: BLE001 ans = f"[solo-error: {type(e).__name__}: {e}]" - v = judge(q_asked, _descaffold(ans), gold) + v = judge_fn(q_asked, _descaffold(ans), gold) out.append({"arm": "solo", "model": mkey, "variant": variant, "i": idx, "question_orig": q0, @@ -183,7 +201,7 @@ def _process_item(idx: int, it: dict, variants: list[str], except Exception as e: # noqa: BLE001 arb_raw = f"[arborist-error: {type(e).__name__}: {e}]" arb_mode = "ERROR" - va = judge(q_asked, _descaffold(arb_raw), gold) + va = judge_fn(q_asked, _descaffold(arb_raw), gold) eff = ("ABSTAINED" if arb_mode == "UNGROUNDED" and va.label in ("WRONG", "FABRICATED") else va.label) @@ -266,7 +284,10 @@ def _report(tally, n_items, n_recs, args, jsonl_path, ts, "Codes: CG=CORRECT_GROUNDED W=WRONG F=FABRICATED " "A=ABSTAINED JE=JUDGE_ERROR. Gold = fixed corpus-vintage " "article (~2010-2011, verified from the artefact, NOT the " - "2003 dump CLAUDE.md names). Judge = hermetic blinded Opus.", + "2003 dump CLAUDE.md names). " + f"Judge = {getattr(args, 'judge', 'opus')} " + "(see bench/judge_code.py for the code judge / " + "bench/judge.py for the gated Opus judge).", "", "## Raw verdicts (framing-neutral)", "", @@ -343,7 +364,15 @@ def main() -> int: "restarting at higher --max-workers without " "losing finished units") ap.add_argument("--skip-self-test", action="store_true") + ap.add_argument("--judge", choices=sorted(_JUDGES.keys()), + default="code", + help="which judge to use. 'code' (default, 2026-05-19) " + "is deterministic / no LLM / no quota; 'opus' is " + "the original headless Opus judge (requires " + "ARBORIST_JUDGE_ENABLE=1 — gated to prevent " + "accidental quota burn).") a = ap.parse_args() + judge_fn, judge_self_test, judge_model_id = _JUDGES[a.judge] if a.report_only: jp = Path(a.report_only) @@ -364,15 +393,18 @@ def main() -> int: print(f" solo N={len(items)} arborist-ref N={arb_units} " f"models={models} variants={variants} " f"workers={a.max_workers}") + print(f" judge={a.judge} ({judge_model_id})") + judge_cost_note = ("0 LLM calls" if a.judge == "code" + else f"~{n_solo + n_arb} claude -p judge") print(f" bounded spend: {n_solo} solo + {n_arb} Arborist + " - f"~{n_solo + n_arb} claude -p judge + 4 self-test") + f"{judge_cost_note} + 4 self-test") if a.skip_self_test: print(" !! self-test SKIPPED — verdicts UNTRUSTED") else: - print(" judge self-test (instrument gate) …") - if self_test() != 0: - print("ABORT: judge unreliable") + print(f" judge self-test ({a.judge}; instrument gate) …") + if judge_self_test() != 0: + print(f"ABORT: judge ({a.judge}) unreliable") return 1 skip_items: set[int] = set() @@ -411,7 +443,7 @@ def main() -> int: futs = { ex.submit(_process_item, i, it, variants, models, shards_dir, i <= arb_units, a.arborist_ref, - ts): i + ts, judge_fn): i for i, it in enumerate(items, 1) if i not in skip_items }