From 1cabfe685046dc358adc3abc8c62f72339af2973 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Tue, 19 May 2026 17:26:06 -0400 Subject: [PATCH] =?UTF-8?q?feat(#000057):=20fail-closed=20Opus=20judge=20g?= =?UTF-8?q?ate=20=E2=80=94=20ARBORIST=5FJUDGE=5FENABLE=3D1=20to=20run?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2026-05-19: huge-N #000057 control sweep (f63b00d → 9dc02e4) burned our Opus quota. Disable judge.py by default so a stray re-run can't re-burn — every call short-circuits to JUDGE_ERROR with rationale 'disabled — set ARBORIST_JUDGE_ENABLE=1 ...' and zero subprocess spawn (0ms in the disabled path, smoke-tested). Why a gate, not a model swap: - judge.py uses Opus deliberately as EXTERNAL SOTA outside both arms; swapping the judge to Hermes/Qwen would corrupt the experiment (Hermes is itself an arm under test). The hygiene comment at judge.py:26-29 already names same-family-judging as the live threat to validity at Opus level; downgrading further changes the science. - Gating instead preserves the science when fox re-enables, and gives us the data-first workflow he asked for: deterministic tool pre-filters (verifier / NLI / recall@k) up front, judge only on residue worth Opus tokens, with explicit go. Behaviour: - control_sweep.py + control_ab.py already treat JUDGE_ERROR non-fatally (counted as JE in _bucket); disabled runs degrade to 100% JE in the tally and surface the disable reason in rationale — the loudest possible 'judge did not run here' signal. - Re-enable per-run: ARBORIST_JUDGE_ENABLE=1 python -m bench.control_sweep ... - self_test() will report 4× JUDGE_ERROR when gated — intentional; if the instrument is off, the self-test must NOT silently pass. Smoke-test (without flag): label='JUDGE_ERROR' rationale='disabled — ...' dt=0.0ms · no claude subprocess spawned. Cross-referenced from CLAUDE.md '## Live endpoints' / 'Budget discipline' subsection added in 2365bd1. --- bench/judge.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/bench/judge.py b/bench/judge.py index 9c066be..03c0ad1 100644 --- a/bench/judge.py +++ b/bench/judge.py @@ -1,6 +1,16 @@ #!/usr/bin/env python3 """Hermetic external judge for the #000057 control experiment. +**Disabled by default as of 2026-05-19.** The huge-N control sweep +(`f63b00d` → `9dc02e4`) burned our Opus quota; per CLAUDE.md +"Budget discipline" we no longer call Opus on autopilot. `judge()` +now fails-closed with a `JUDGE_ERROR` verdict ("disabled — set +ARBORIST_JUDGE_ENABLE=1") unless that env flag is set, so a stray +re-run cannot re-burn the budget. Tool-based pre-filters +(deterministic verifier, NLI, lexical / recall@k scoring) front-run +the judge: only the residue actually worth Opus tokens gets passed +through later, with fox's explicit go. + The judge is a SOTA model (Opus via `claude -p`, headless) used as EXTERNAL SCIENCE — it sits outside both arms (Hermes-solo, Arborist), scores outputs post-hoc, and touches neither system's internals. This @@ -39,6 +49,19 @@ from dataclasses import dataclass JUDGE_PROMPT_ID = "ctrl-judge-v1" JUDGE_MODEL = "opus (claude -p)" +# Fail-closed gate (2026-05-19): the huge-N #000057 sweep burned our Opus +# quota. `judge()` short-circuits with a JUDGE_ERROR verdict unless +# ARBORIST_JUDGE_ENABLE=1 is exported. Existing callers (control_sweep, +# control_ab) already handle JUDGE_ERROR non-fatally, so disabled runs +# degrade cleanly — every record returns "disabled" in rationale, the +# tally goes 100% JE, and nothing fires `claude -p`. Re-enable per run: +# ARBORIST_JUDGE_ENABLE=1 python -m bench.control_sweep ... +_JUDGE_ENABLE_ENV = "ARBORIST_JUDGE_ENABLE" +_DISABLED_RATIONALE = ( + f"disabled — set {_JUDGE_ENABLE_ENV}=1 to enable Opus judge calls " + "(per CLAUDE.md 'Budget discipline')" +) + # Pinned, hermetic. The judge sees ONLY these three fields. It is NOT # told either system exists. Verdict vocabulary is closed so scoring # is deterministic. @@ -95,7 +118,14 @@ class Verdict: def judge(question: str, answer: str, gold_source: str, *, timeout: int = 180, gold_cap: int = 6000) -> Verdict: """One hermetic blinded reference-grounded verdict. `answer` MUST - already be arm-blinded by the caller.""" + already be arm-blinded by the caller. + + Fail-closed: returns `JUDGE_ERROR` immediately unless + `ARBORIST_JUDGE_ENABLE=1` is exported (see module docstring & + `_JUDGE_ENABLE_ENV`). No `claude -p` subprocess is spawned in the + disabled path — zero Opus tokens cost when the gate is closed.""" + if os.environ.get(_JUDGE_ENABLE_ENV) != "1": + return Verdict("JUDGE_ERROR", _DISABLED_RATIONALE, "") prompt = _PROMPT.format( q=question.strip(), gold=(gold_source or "").strip()[:gold_cap],