diff --git a/bench/control_sweep.py b/bench/control_sweep.py index 25d237a..27c24b6 100644 --- a/bench/control_sweep.py +++ b/bench/control_sweep.py @@ -134,13 +134,18 @@ VARIANTS = { def _process_item(idx: int, it: dict, variants: list[str], models: list[str], shards_dir: Path, arborist_on: bool, arborist_ref: str, - ts: str, judge_fn=None) -> list[dict]: + ts: str, judge_fn=None, + skip_solo: bool = False) -> list[dict]: """All variants × models for ONE fixture item. Self-contained: its own qa_db, its own clients — safe to run concurrently. ``judge_fn`` defaults to the code judge for back-compat with any direct callers that pre-date the --judge switch; main() passes - the user's choice explicitly.""" + the user's choice explicitly. ``skip_solo`` lets a caller run + ONLY the arborist arm — useful when the solo data for the same + model + fixture already exists in a prior sweep and the new run + is just adding the arborist arm (fox 2026-05-19: 'we don't need + to redo anything').""" if judge_fn is None: judge_fn = _judge_code.judge from arborist.qa.client import OpenAICompatibleClient @@ -166,7 +171,10 @@ def _process_item(idx: int, it: dict, variants: list[str], try: for variant in variants: q_asked = VARIANTS[variant](q0) - for mkey in models: + # Solo arm is skipped when --skip-solo is set OR when the + # caller passed no models. Either way the arborist arm + # below still runs if arborist_on. + for mkey in [] if skip_solo else models: cfg = MODELS[mkey] try: ans = cl(mkey).chat_completion( @@ -382,6 +390,11 @@ def main() -> int: "the original headless Opus judge (requires " "ARBORIST_JUDGE_ENABLE=1 — gated to prevent " "accidental quota burn).") + ap.add_argument("--skip-solo", action="store_true", + help="skip the solo arm; run ONLY the arborist arm. " + "Use when the model's solo data already exists " + "from a prior sweep on the same fixture — avoids " + "redundant LLM spend.") a = ap.parse_args() judge_fn, judge_self_test, judge_model_id = _JUDGES[a.judge] @@ -397,7 +410,7 @@ def main() -> int: variants = [v for v in a.variants.split(",") if v in VARIANTS] items = json.loads(Path(a.fixture).read_text())[:a.n] - n_solo = len(models) * len(variants) * len(items) + n_solo = 0 if a.skip_solo else len(models) * len(variants) * len(items) arb_units = min(a.arborist_n, len(items)) n_arb = len(variants) * arb_units print(f"#000057 control sweep — fixture={a.fixture}") @@ -454,7 +467,7 @@ def main() -> int: futs = { ex.submit(_process_item, i, it, variants, models, shards_dir, i <= arb_units, a.arborist_ref, - ts, judge_fn): i + ts, judge_fn, a.skip_solo): i for i, it in enumerate(items, 1) if i not in skip_items }