From 5863559445a5288d36a2179d4d92fa064cbacacc Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Tue, 19 May 2026 20:07:24 -0400 Subject: [PATCH] =?UTF-8?q?feat(#000057):=20--skip-solo=20flag=20=E2=80=94?= =?UTF-8?q?=20arborist-only=20sweeps=20without=20redundant=20solo=20data?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fox 2026-05-19: 'we don't need to redo anything'. The arborist+qwen- nothink sweep would re-run qwen-nothink solo on 1158 records that phase 1 already produced (control_sweep_2026-05-19T21-52-56Z.jsonl, calibrated rescore on disk). Wasteful — qwen-nothink solo behaviour doesn't depend on retrieval being on, so phase 1's numbers stand. --skip-solo runs ONLY the arborist arm. Implementation: skip the per-variant model loop in _process_item when the flag is set; the arborist arm below it still runs if arborist_on. The spend banner zeroes the solo-call count so the operator sees the actual LLM budget for the arborist-only run. Example: a full arborist+qwen-nothink sweep against the 386-item fixture goes from 2316 LLM calls (1158 solo + 1158 arborist) to 1158 calls (arborist only). Halves wall-clock on single-worker llama.cpp. pytest 27/27, imports clean, --help shows the flag. --- bench/control_sweep.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/bench/control_sweep.py b/bench/control_sweep.py index 25d237a..27c24b6 100644 --- a/bench/control_sweep.py +++ b/bench/control_sweep.py @@ -134,13 +134,18 @@ VARIANTS = { def _process_item(idx: int, it: dict, variants: list[str], models: list[str], shards_dir: Path, arborist_on: bool, arborist_ref: str, - ts: str, judge_fn=None) -> list[dict]: + ts: str, judge_fn=None, + skip_solo: bool = False) -> list[dict]: """All variants × models for ONE fixture item. Self-contained: its own qa_db, its own clients — safe to run concurrently. ``judge_fn`` defaults to the code judge for back-compat with any direct callers that pre-date the --judge switch; main() passes - the user's choice explicitly.""" + the user's choice explicitly. ``skip_solo`` lets a caller run + ONLY the arborist arm — useful when the solo data for the same + model + fixture already exists in a prior sweep and the new run + is just adding the arborist arm (fox 2026-05-19: 'we don't need + to redo anything').""" if judge_fn is None: judge_fn = _judge_code.judge from arborist.qa.client import OpenAICompatibleClient @@ -166,7 +171,10 @@ def _process_item(idx: int, it: dict, variants: list[str], try: for variant in variants: q_asked = VARIANTS[variant](q0) - for mkey in models: + # Solo arm is skipped when --skip-solo is set OR when the + # caller passed no models. Either way the arborist arm + # below still runs if arborist_on. + for mkey in [] if skip_solo else models: cfg = MODELS[mkey] try: ans = cl(mkey).chat_completion( @@ -382,6 +390,11 @@ def main() -> int: "the original headless Opus judge (requires " "ARBORIST_JUDGE_ENABLE=1 — gated to prevent " "accidental quota burn).") + ap.add_argument("--skip-solo", action="store_true", + help="skip the solo arm; run ONLY the arborist arm. " + "Use when the model's solo data already exists " + "from a prior sweep on the same fixture — avoids " + "redundant LLM spend.") a = ap.parse_args() judge_fn, judge_self_test, judge_model_id = _JUDGES[a.judge] @@ -397,7 +410,7 @@ def main() -> int: variants = [v for v in a.variants.split(",") if v in VARIANTS] items = json.loads(Path(a.fixture).read_text())[:a.n] - n_solo = len(models) * len(variants) * len(items) + n_solo = 0 if a.skip_solo else len(models) * len(variants) * len(items) arb_units = min(a.arborist_n, len(items)) n_arb = len(variants) * arb_units print(f"#000057 control sweep — fixture={a.fixture}") @@ -454,7 +467,7 @@ def main() -> int: futs = { ex.submit(_process_item, i, it, variants, models, shards_dir, i <= arb_units, a.arborist_ref, - ts, judge_fn): i + ts, judge_fn, a.skip_solo): i for i, it in enumerate(items, 1) if i not in skip_items }