engineering/arborist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(#000057): --judge {code,opus} switch in control sweep/AB, default=code

Browse source 
Wire bench/judge_code.py into the sweep harnesses as the default judge.
Both control_sweep.py and control_ab.py grow a --judge {code,opus} CLI
arg; both share the same Verdict shape so the dispatch is a pointer
assignment + threading the judge_fn through _process_item.

Behaviour:
- DEFAULT = code: zero LLM, zero quota, deterministic. Self-test gate
  is the code judge's 4-fixture contract.
- --judge opus: original gated Opus path; needs ARBORIST_JUDGE_ENABLE=1
  set per 1cabfe6's fail-closed gate, otherwise every record returns
  JUDGE_ERROR with rationale 'disabled — set ARBORIST_JUDGE_ENABLE=1'
  and the sweep records that label honestly.

Reporting:
- Header line now records which judge ran ('Judge = code (...)' or
  'Judge = opus (...)') so partial-reports & resumes don't lie about
  provenance.
- Spend banner shows '0 LLM calls' for the code path so the no-burn
  property is visible in the operator output.

Test surface: pytest sweep across tests/ still 136/136 (no regressions);
new --judge flag visible in --help on both harnesses.

Next: bench/score_with_code_judge.py to re-grade existing sweep JSONLs
(written under the gated-Opus run) with the code judge; agreement
matrix surfaces residue size for the eventual LLM-batch needle-haystack.
This commit is contained in:
russell@unturf.com 2026-05-19 17:42:49 -04:00
parent f6a822ed8a
commit a2e9b49c0a
No known key found for this signature in database
2 changed files with 60 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

18
bench/control_ab.py
View file

@ -53,7 +53,16 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from bench.judge import judge # noqa: E402
# Judge selection at run time (fox 2026-05-19, see CLAUDE.md
# 'Budget discipline'). Both modules share the Verdict shape so the
# downstream record-emit path is judge-agnostic.
import bench.judge as _judge_opus # noqa: E402
import bench.judge_code as _judge_code # noqa: E402
_JUDGES = {
"code": (_judge_code.judge, _judge_code.JUDGE_MODEL),
"opus": (_judge_opus.judge, _judge_opus.JUDGE_MODEL),
}
_EVID = re.compile(r"\[E\d+[^\]]*\]", re.S) # [E1 | Title | hash:"…"]
_PTR = re.compile(r"\s*\[E\d+(?:\s*,\s*E\d+)*\]") # bare [E1] / [E1,E2]
@ -106,7 +115,14 @@ def main() -> int:
"ARBORIST_LLM_MODEL",
"adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic"))
ap.add_argument("--out-dir", default="bench/qa_results")
ap.add_argument("--judge", choices=sorted(_JUDGES.keys()),
default="code",
help="which judge to use. 'code' (default, no LLM) is "
"deterministic / no quota; 'opus' is the gated "
"headless Opus judge (requires "
"ARBORIST_JUDGE_ENABLE=1).")
a = ap.parse_args()
judge, _judge_model_id = _JUDGES[a.judge]
from arborist.qa.client import OpenAICompatibleClient
from arborist.qa.query import DEFAULT_QUERY_POLICY, query

54
bench/control_sweep.py
View file

@ -74,7 +74,19 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from bench.control_ab import _SOLO_SYS, _descaffold, _gold # noqa: E402
from bench.judge import judge, self_test # noqa: E402
# Judge selection is now run-time via --judge {code,opus} so a sweep can
# collect data WITHOUT burning Opus quota (fox 2026-05-19). Both judges
# share the Verdict shape & verdict vocabulary so swapping is a pointer
# reassignment; nothing downstream changes. Default=code.
import bench.judge as _judge_opus # noqa: E402
import bench.judge_code as _judge_code # noqa: E402
_JUDGES = {
"code": (_judge_code.judge, _judge_code.self_test,
_judge_code.JUDGE_MODEL),
"opus": (_judge_opus.judge, _judge_opus.self_test,
_judge_opus.JUDGE_MODEL),
}
MODELS: dict[str, dict] = {
"hermes": dict(
@ -122,9 +134,15 @@ VARIANTS = {
def _process_item(idx: int, it: dict, variants: list[str],
models: list[str], shards_dir: Path,
arborist_on: bool, arborist_ref: str,
ts: str) -> list[dict]:
ts: str, judge_fn=None) -> list[dict]:
"""All variants × models for ONE fixture item. Self-contained:
its own qa_db, its own clients — safe to run concurrently."""
its own qa_db, its own clients — safe to run concurrently.
``judge_fn`` defaults to the code judge for back-compat with any
direct callers that pre-date the --judge switch; main() passes
the user's choice explicitly."""
if judge_fn is None:
judge_fn = _judge_code.judge
from arborist.qa.client import OpenAICompatibleClient
from arborist.qa.query import DEFAULT_QUERY_POLICY, query
@ -159,7 +177,7 @@ def _process_item(idx: int, it: dict, variants: list[str],
extra_body=cfg["extra"])
except Exception as e: # noqa: BLE001
ans = f"[solo-error: {type(e).__name__}: {e}]"
v = judge(q_asked, _descaffold(ans), gold)
v = judge_fn(q_asked, _descaffold(ans), gold)
out.append({"arm": "solo", "model": mkey,
"variant": variant, "i": idx,
"question_orig": q0,
@ -183,7 +201,7 @@ def _process_item(idx: int, it: dict, variants: list[str],
except Exception as e: # noqa: BLE001
arb_raw = f"[arborist-error: {type(e).__name__}: {e}]"
arb_mode = "ERROR"
va = judge(q_asked, _descaffold(arb_raw), gold)
va = judge_fn(q_asked, _descaffold(arb_raw), gold)
eff = ("ABSTAINED" if arb_mode == "UNGROUNDED"
and va.label in ("WRONG", "FABRICATED")
else va.label)
@ -266,7 +284,10 @@ def _report(tally, n_items, n_recs, args, jsonl_path, ts,
"Codes: CG=CORRECT_GROUNDED W=WRONG F=FABRICATED "
"A=ABSTAINED JE=JUDGE_ERROR. Gold = fixed corpus-vintage "
"article (~2010-2011, verified from the artefact, NOT the "
"2003 dump CLAUDE.md names). Judge = hermetic blinded Opus.",
"2003 dump CLAUDE.md names). "
f"Judge = {getattr(args, 'judge', 'opus')} "
"(see bench/judge_code.py for the code judge / "
"bench/judge.py for the gated Opus judge).",
"",
"## Raw verdicts (framing-neutral)",
"",
@ -343,7 +364,15 @@ def main() -> int:
"restarting at higher --max-workers without "
"losing finished units")
ap.add_argument("--skip-self-test", action="store_true")
ap.add_argument("--judge", choices=sorted(_JUDGES.keys()),
default="code",
help="which judge to use. 'code' (default, 2026-05-19) "
"is deterministic / no LLM / no quota; 'opus' is "
"the original headless Opus judge (requires "
"ARBORIST_JUDGE_ENABLE=1 — gated to prevent "
"accidental quota burn).")
a = ap.parse_args()
judge_fn, judge_self_test, judge_model_id = _JUDGES[a.judge]
if a.report_only:
jp = Path(a.report_only)
@ -364,15 +393,18 @@ def main() -> int:
print(f" solo N={len(items)} arborist-ref N={arb_units} "
f"models={models} variants={variants} "
f"workers={a.max_workers}")
print(f" judge={a.judge} ({judge_model_id})")
judge_cost_note = ("0 LLM calls" if a.judge == "code"
else f"~{n_solo + n_arb} claude -p judge")
print(f" bounded spend: {n_solo} solo + {n_arb} Arborist + "
f"~{n_solo + n_arb} claude -p judge + 4 self-test")
f"{judge_cost_note} + 4 self-test")
if a.skip_self_test:
print(" !! self-test SKIPPED — verdicts UNTRUSTED")
else:
print(" judge self-test (instrument gate) …")
if self_test() != 0:
print("ABORT: judge unreliable")
print(f" judge self-test ({a.judge}; instrument gate) …")
if judge_self_test() != 0:
print(f"ABORT: judge ({a.judge}) unreliable")
return 1
skip_items: set[int] = set()
@ -411,7 +443,7 @@ def main() -> int:
futs = {
ex.submit(_process_item, i, it, variants, models,
shards_dir, i <= arb_units, a.arborist_ref,
ts): i
ts, judge_fn): i
for i, it in enumerate(items, 1)
if i not in skip_items
}