feat(#000057): --skip-solo flag — arborist-only sweeps without redundant solo data
fox 2026-05-19: 'we don't need to redo anything'. The arborist+qwen- nothink sweep would re-run qwen-nothink solo on 1158 records that phase 1 already produced (control_sweep_2026-05-19T21-52-56Z.jsonl, calibrated rescore on disk). Wasteful — qwen-nothink solo behaviour doesn't depend on retrieval being on, so phase 1's numbers stand. --skip-solo runs ONLY the arborist arm. Implementation: skip the per-variant model loop in _process_item when the flag is set; the arborist arm below it still runs if arborist_on. The spend banner zeroes the solo-call count so the operator sees the actual LLM budget for the arborist-only run. Example: a full arborist+qwen-nothink sweep against the 386-item fixture goes from 2316 LLM calls (1158 solo + 1158 arborist) to 1158 calls (arborist only). Halves wall-clock on single-worker llama.cpp. pytest 27/27, imports clean, --help shows the flag.
This commit is contained in:
parent
866e67f43d
commit
5863559445
1 changed files with 18 additions and 5 deletions
|
|
@ -134,13 +134,18 @@ VARIANTS = {
|
|||
def _process_item(idx: int, it: dict, variants: list[str],
|
||||
models: list[str], shards_dir: Path,
|
||||
arborist_on: bool, arborist_ref: str,
|
||||
ts: str, judge_fn=None) -> list[dict]:
|
||||
ts: str, judge_fn=None,
|
||||
skip_solo: bool = False) -> list[dict]:
|
||||
"""All variants × models for ONE fixture item. Self-contained:
|
||||
its own qa_db, its own clients — safe to run concurrently.
|
||||
|
||||
``judge_fn`` defaults to the code judge for back-compat with any
|
||||
direct callers that pre-date the --judge switch; main() passes
|
||||
the user's choice explicitly."""
|
||||
the user's choice explicitly. ``skip_solo`` lets a caller run
|
||||
ONLY the arborist arm — useful when the solo data for the same
|
||||
model + fixture already exists in a prior sweep and the new run
|
||||
is just adding the arborist arm (fox 2026-05-19: 'we don't need
|
||||
to redo anything')."""
|
||||
if judge_fn is None:
|
||||
judge_fn = _judge_code.judge
|
||||
from arborist.qa.client import OpenAICompatibleClient
|
||||
|
|
@ -166,7 +171,10 @@ def _process_item(idx: int, it: dict, variants: list[str],
|
|||
try:
|
||||
for variant in variants:
|
||||
q_asked = VARIANTS[variant](q0)
|
||||
for mkey in models:
|
||||
# Solo arm is skipped when --skip-solo is set OR when the
|
||||
# caller passed no models. Either way the arborist arm
|
||||
# below still runs if arborist_on.
|
||||
for mkey in [] if skip_solo else models:
|
||||
cfg = MODELS[mkey]
|
||||
try:
|
||||
ans = cl(mkey).chat_completion(
|
||||
|
|
@ -382,6 +390,11 @@ def main() -> int:
|
|||
"the original headless Opus judge (requires "
|
||||
"ARBORIST_JUDGE_ENABLE=1 — gated to prevent "
|
||||
"accidental quota burn).")
|
||||
ap.add_argument("--skip-solo", action="store_true",
|
||||
help="skip the solo arm; run ONLY the arborist arm. "
|
||||
"Use when the model's solo data already exists "
|
||||
"from a prior sweep on the same fixture — avoids "
|
||||
"redundant LLM spend.")
|
||||
a = ap.parse_args()
|
||||
judge_fn, judge_self_test, judge_model_id = _JUDGES[a.judge]
|
||||
|
||||
|
|
@ -397,7 +410,7 @@ def main() -> int:
|
|||
variants = [v for v in a.variants.split(",") if v in VARIANTS]
|
||||
items = json.loads(Path(a.fixture).read_text())[:a.n]
|
||||
|
||||
n_solo = len(models) * len(variants) * len(items)
|
||||
n_solo = 0 if a.skip_solo else len(models) * len(variants) * len(items)
|
||||
arb_units = min(a.arborist_n, len(items))
|
||||
n_arb = len(variants) * arb_units
|
||||
print(f"#000057 control sweep — fixture={a.fixture}")
|
||||
|
|
@ -454,7 +467,7 @@ def main() -> int:
|
|||
futs = {
|
||||
ex.submit(_process_item, i, it, variants, models,
|
||||
shards_dir, i <= arb_units, a.arborist_ref,
|
||||
ts, judge_fn): i
|
||||
ts, judge_fn, a.skip_solo): i
|
||||
for i, it in enumerate(items, 1)
|
||||
if i not in skip_items
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue