diff --git a/bench/judge.py b/bench/judge.py index 9c066be..03c0ad1 100644 --- a/bench/judge.py +++ b/bench/judge.py @@ -1,6 +1,16 @@ #!/usr/bin/env python3 """Hermetic external judge for the #000057 control experiment. +**Disabled by default as of 2026-05-19.** The huge-N control sweep +(`f63b00d` → `9dc02e4`) burned our Opus quota; per CLAUDE.md +"Budget discipline" we no longer call Opus on autopilot. `judge()` +now fails-closed with a `JUDGE_ERROR` verdict ("disabled — set +ARBORIST_JUDGE_ENABLE=1") unless that env flag is set, so a stray +re-run cannot re-burn the budget. Tool-based pre-filters +(deterministic verifier, NLI, lexical / recall@k scoring) front-run +the judge: only the residue actually worth Opus tokens gets passed +through later, with fox's explicit go. + The judge is a SOTA model (Opus via `claude -p`, headless) used as EXTERNAL SCIENCE — it sits outside both arms (Hermes-solo, Arborist), scores outputs post-hoc, and touches neither system's internals. This @@ -39,6 +49,19 @@ from dataclasses import dataclass JUDGE_PROMPT_ID = "ctrl-judge-v1" JUDGE_MODEL = "opus (claude -p)" +# Fail-closed gate (2026-05-19): the huge-N #000057 sweep burned our Opus +# quota. `judge()` short-circuits with a JUDGE_ERROR verdict unless +# ARBORIST_JUDGE_ENABLE=1 is exported. Existing callers (control_sweep, +# control_ab) already handle JUDGE_ERROR non-fatally, so disabled runs +# degrade cleanly — every record returns "disabled" in rationale, the +# tally goes 100% JE, and nothing fires `claude -p`. Re-enable per run: +# ARBORIST_JUDGE_ENABLE=1 python -m bench.control_sweep ... +_JUDGE_ENABLE_ENV = "ARBORIST_JUDGE_ENABLE" +_DISABLED_RATIONALE = ( + f"disabled — set {_JUDGE_ENABLE_ENV}=1 to enable Opus judge calls " + "(per CLAUDE.md 'Budget discipline')" +) + # Pinned, hermetic. The judge sees ONLY these three fields. It is NOT # told either system exists. Verdict vocabulary is closed so scoring # is deterministic. @@ -95,7 +118,14 @@ class Verdict: def judge(question: str, answer: str, gold_source: str, *, timeout: int = 180, gold_cap: int = 6000) -> Verdict: """One hermetic blinded reference-grounded verdict. `answer` MUST - already be arm-blinded by the caller.""" + already be arm-blinded by the caller. + + Fail-closed: returns `JUDGE_ERROR` immediately unless + `ARBORIST_JUDGE_ENABLE=1` is exported (see module docstring & + `_JUDGE_ENABLE_ENV`). No `claude -p` subprocess is spawned in the + disabled path — zero Opus tokens cost when the gate is closed.""" + if os.environ.get(_JUDGE_ENABLE_ENV) != "1": + return Verdict("JUDGE_ERROR", _DISABLED_RATIONALE, "") prompt = _PROMPT.format( q=question.strip(), gold=(gold_source or "").strip()[:gold_cap],