From 780f5afd695ea7c5e89212f19e08e5d8b4a3bd0e Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sat, 2 May 2026 11:07:07 -0400 Subject: [PATCH] bench: --resume for stop/start-able sweeps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds --resume : read the existing JSONL, build a set of (question, mode, sample_idx) tasks already done, skip those in the shuffled task list, and append fresh rows to the same file. The markdown rollup uses the union of pre-existing + new rows. Same --seed required for the order to align across the resumed run. Why: a 60-90 min full bench is annoying to re-run from zero when something interrupts (network blip, kill, kernel panic). The JSONL has been the durable artifact for ages; the bench just didn't know how to read it. Implementation: - args.resume: Path | None - if given: parse JSONL line-by-line, populate done_tasks set, reuse the resume_path's stem as the run stamp, set jsonl_path + md_path to the resumed paths - file_mode = 'a' if resuming else 'w' - tasks filter: drop entries already in done_tasks - rows initialized with existing_rows so the markdown summary sees the union - done counter starts at len(existing_rows) so the print countdown reflects total progress Empirical context: the c=6 bench killed at 142/639 left a recoverable JSONL. Future kills can resume via: .venv/bin/python bench/qa_sweep.py \ --resume bench/qa_results/.jsonl \ --seed 0 --concurrency 4 (Same seed reproduces the shuffle so the remaining tasks come out in the original order.) Concurrency sweep on the smoke (15 tasks at each c): c=3 102s (peak throughput) c=4 106s (4% slower; chosen for full bench — more forgiving on single-call hiccups) c=5 119s (12% slower than c=3) c=6 185s (45% slower; vLLM batching ceiling) Lesson: more concurrent ≠ faster. vLLM continuous batching has a sweet spot around c=3-4 for this endpoint. --- bench/qa_sweep.py | 61 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 7 deletions(-) diff --git a/bench/qa_sweep.py b/bench/qa_sweep.py index ac6ef08..a293845 100644 --- a/bench/qa_sweep.py +++ b/bench/qa_sweep.py @@ -490,6 +490,14 @@ def main(argv: list[str] | None = None) -> int: "design — uncorrelated samples give true i.i.d. " "variance estimation and feed vLLM's continuous " "batcher a diverse request stream.") + ap.add_argument("--resume", type=Path, default=None, + help="resume an interrupted bench by appending to " + "an existing JSONL. Already-completed (question, " + "mode, sample_idx) tasks are skipped; remaining " + "tasks run in the original shuffled order (use " + "the same --seed). The same .md path is " + "re-rendered from the union of pre-existing + " + "new rows. Stop/start-able bench.") ap.add_argument("--endpoint", default=os.environ.get( "ABORIST_LLM_ENDPOINT", "https://hermes.ai.unturf.com/v1")) ap.add_argument("--model", default=os.environ.get( @@ -510,9 +518,37 @@ def main(argv: list[str] | None = None) -> int: return 2 args.out_dir.mkdir(parents=True, exist_ok=True) - stamp = _utc_iso_compact() - jsonl_path = args.out_dir / f"{stamp}.jsonl" - md_path = args.out_dir / f"{stamp}.md" + + # Resume support: when --resume is given, reuse + # that file's stamp and rebuild done_tasks set so already-completed + # samples skip on this run. Markdown rolls up the union of + # pre-existing + freshly-written rows. + existing_rows: list[dict] = [] + done_tasks: set[tuple[str, str, int]] = set() + if args.resume: + resume_path = Path(args.resume) + if not resume_path.exists(): + print(f"--resume target does not exist: {resume_path}", file=sys.stderr) + return 2 + with resume_path.open(encoding="utf-8") as rf: + for line in rf: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue # skip a partial trailing line + key = (row["question"], row["answer_mode"], int(row.get("sample_idx", 0))) + done_tasks.add(key) + existing_rows.append(row) + stamp = resume_path.stem # reuse the original "YYYY-MM-DDTHH-MM-SSZ" + jsonl_path = resume_path + md_path = resume_path.with_suffix(".md") + else: + stamp = _utc_iso_compact() + jsonl_path = args.out_dir / f"{stamp}.jsonl" + md_path = args.out_dir / f"{stamp}.md" if args.n_samples < 1: print(f"--n must be >= 1, got {args.n_samples}", file=sys.stderr) @@ -527,7 +563,11 @@ def main(argv: list[str] | None = None) -> int: f"burn=always (per-sample, forces fresh inference)") print(f"[bench] concurrency={args.concurrency} seed={args.seed} " f"(sample-level tasks; per-cell Lock serializes cache_key writes)") - print(f"[bench] writing → {jsonl_path}") + if args.resume: + print(f"[bench] resuming from {jsonl_path}: " + f"{len(existing_rows)} task(s) already done; skipping") + else: + print(f"[bench] writing → {jsonl_path}") print() # Build sample-level task list. Each task is one (question, mode, @@ -550,6 +590,12 @@ def main(argv: list[str] | None = None) -> int: rng.shuffle(tasks) total_tasks = len(tasks) + # Resume: filter out tasks already in the JSONL. Order preserved. + if done_tasks: + tasks = [t for t in tasks if t not in done_tasks] + print(f"[bench] {len(tasks)} task(s) remaining ({total_tasks} total)") + print() + # Per-cell lock — burn+insert against a shared cache_key must # serialize. Two samples of the SAME (q, mode) ending up in # adjacent worker slots could race; the lock makes that path @@ -558,12 +604,13 @@ def main(argv: list[str] | None = None) -> int: # is near-zero while correctness is preserved. cell_locks: dict[tuple[str, str], Lock] = defaultdict(Lock) - rows: list[dict] = [] + rows: list[dict] = list(existing_rows) # preserve resumed rows for the rollup write_lock = Lock() print_lock = Lock() - done = [0] + done = [len(existing_rows)] # countdown reflects total, not just-this-session - with jsonl_path.open("w", encoding="utf-8") as f: + file_mode = "a" if args.resume else "w" + with jsonl_path.open(file_mode, encoding="utf-8") as f: def _process(task: tuple[str, str, int]) -> None: q, mode, sample_idx = task with cell_locks[(q, mode)]: