feat(#000057): --skip-solo flag — arborist-only sweeps without redundant solo data

fox 2026-05-19: 'we don't need to redo anything'. The arborist+qwen-
nothink sweep would re-run qwen-nothink solo on 1158 records that
phase 1 already produced (control_sweep_2026-05-19T21-52-56Z.jsonl,
calibrated rescore on disk). Wasteful — qwen-nothink solo behaviour
doesn't depend on retrieval being on, so phase 1's numbers stand.

--skip-solo runs ONLY the arborist arm. Implementation: skip the
per-variant model loop in _process_item when the flag is set; the
arborist arm below it still runs if arborist_on. The spend banner
zeroes the solo-call count so the operator sees the actual LLM
budget for the arborist-only run.

Example: a full arborist+qwen-nothink sweep against the 386-item
fixture goes from 2316 LLM calls (1158 solo + 1158 arborist) to
1158 calls (arborist only). Halves wall-clock on single-worker
llama.cpp.

pytest 27/27, imports clean, --help shows the flag.
This commit is contained in:
russell@unturf.com 2026-05-19 20:07:24 -04:00
parent 866e67f43d
commit 5863559445
No known key found for this signature in database

View file

@ -134,13 +134,18 @@ VARIANTS = {
def _process_item(idx: int, it: dict, variants: list[str],
models: list[str], shards_dir: Path,
arborist_on: bool, arborist_ref: str,
ts: str, judge_fn=None) -> list[dict]:
ts: str, judge_fn=None,
skip_solo: bool = False) -> list[dict]:
"""All variants × models for ONE fixture item. Self-contained:
its own qa_db, its own clients — safe to run concurrently.
``judge_fn`` defaults to the code judge for back-compat with any
direct callers that pre-date the --judge switch; main() passes
the user's choice explicitly."""
the user's choice explicitly. ``skip_solo`` lets a caller run
ONLY the arborist arm — useful when the solo data for the same
model + fixture already exists in a prior sweep and the new run
is just adding the arborist arm (fox 2026-05-19: 'we don't need
to redo anything')."""
if judge_fn is None:
judge_fn = _judge_code.judge
from arborist.qa.client import OpenAICompatibleClient
@ -166,7 +171,10 @@ def _process_item(idx: int, it: dict, variants: list[str],
try:
for variant in variants:
q_asked = VARIANTS[variant](q0)
for mkey in models:
# Solo arm is skipped when --skip-solo is set OR when the
# caller passed no models. Either way the arborist arm
# below still runs if arborist_on.
for mkey in [] if skip_solo else models:
cfg = MODELS[mkey]
try:
ans = cl(mkey).chat_completion(
@ -382,6 +390,11 @@ def main() -> int:
"the original headless Opus judge (requires "
"ARBORIST_JUDGE_ENABLE=1 — gated to prevent "
"accidental quota burn).")
ap.add_argument("--skip-solo", action="store_true",
help="skip the solo arm; run ONLY the arborist arm. "
"Use when the model's solo data already exists "
"from a prior sweep on the same fixture — avoids "
"redundant LLM spend.")
a = ap.parse_args()
judge_fn, judge_self_test, judge_model_id = _JUDGES[a.judge]
@ -397,7 +410,7 @@ def main() -> int:
variants = [v for v in a.variants.split(",") if v in VARIANTS]
items = json.loads(Path(a.fixture).read_text())[:a.n]
n_solo = len(models) * len(variants) * len(items)
n_solo = 0 if a.skip_solo else len(models) * len(variants) * len(items)
arb_units = min(a.arborist_n, len(items))
n_arb = len(variants) * arb_units
print(f"#000057 control sweep — fixture={a.fixture}")
@ -454,7 +467,7 @@ def main() -> int:
futs = {
ex.submit(_process_item, i, it, variants, models,
shards_dir, i <= arb_units, a.arborist_ref,
ts, judge_fn): i
ts, judge_fn, a.skip_solo): i
for i, it in enumerate(items, 1)
if i not in skip_items
}