diff --git a/bench/qa_sweep.py b/bench/qa_sweep.py index 3254a62..0c7cd57 100644 --- a/bench/qa_sweep.py +++ b/bench/qa_sweep.py @@ -346,19 +346,36 @@ def _render_markdown( "Buckets capacity in `messages_total` chars (system + reminder + " "evidence/context + question). ~4 chars/token English prose; " "~2.5 chars/token JSON-evidence-heavy. Tells you whether the " - "model strict-rate degrades with input size." + "model strict-rate degrades with input size. Buckets log-scale " + "from 8 KB to >= 1 MB so the same harness covers Hermes-3-8B " + "(82K context) through 1M-context models without code change." ) lines.append("") + # Log-scale buckets — covers the entire model spectrum from + # 8B-class models (Hermes 82K context, max useful prompt ~32-64KB) + # through giant-context models (Gemini 1.5 / Claude 1M, useful + # prompt potentially 100KB-800KB). Models simply don't populate + # buckets beyond their context window; the substrate stays + # universal. Adding new boundaries doesn't require a code change + # for existing data — empty buckets get skipped at render. buckets = [ ("<8KB", 0, 8000), ("8-16KB", 8000, 16000), ("16-32KB", 16000, 32000), ("32-64KB", 32000, 64000), - (">=64KB", 64000, 10**9), + ("64-128KB", 64000, 128000), + ("128-256KB", 128000, 256000), + ("256-512KB", 256000, 512000), + ("512K-1M", 512000, 1048576), + (">=1M", 1048576, 10**12), ] lines.append("| mode | bucket | runs | STRICT | strict-rate | mean evidence chars | mean answer chars |") lines.append("|------|--------|------|--------|-------------|---------------------|-------------------|") + # Track per-mode (label, strict-rate, n) tuples for the + # "recommended context budget" recommendation below. + per_mode_buckets: dict[str, list[tuple[str, float, int]]] = {} for mode in modes: + per_mode_buckets[mode] = [] for label, lo, hi in buckets: sub = [ r for r in rows @@ -372,11 +389,47 @@ def _render_markdown( n_strict = sum(1 for r in sub if r["audit_mode"] == "STRICT") mean_ev = sum((r.get("prompt_chars_evidence") or 0) for r in sub) / n_b mean_ans = sum((r.get("answer_chars") or 0) for r in sub) / n_b + strict_rate = n_strict / n_b + per_mode_buckets[mode].append((label, strict_rate, n_b)) lines.append( f"| {mode} | {label} | {n_b} | {n_strict} | " - f"{n_strict / n_b:.2f} | {int(mean_ev)} | {int(mean_ans)} |" + f"{strict_rate:.2f} | {int(mean_ev)} | {int(mean_ans)} |" ) lines.append("") + lines.append("## recommended context budget (learned from this bench)") + lines.append("") + lines.append( + "Per-mode peak strict-rate bucket from the table above. Bench " + "observations feed back into a recommended `max_context_chars` " + "default. Operator-driven landing per the five-step algorithm " + "step 5 — surfaced here, not auto-applied. Minimum bucket size " + "of 5 runs to be considered (smaller samples are noise). The " + "log-scale buckets cover everything from 8B-class models " + "through 1M-context models; a model whose context window stops " + "at 82K simply never populates the giant buckets." + ) + lines.append("") + lines.append("| mode | peak bucket | strict-rate | n |") + lines.append("|------|-------------|-------------|---|") + for mode in modes: + candidates = [t for t in per_mode_buckets.get(mode, []) if t[2] >= 5] + if not candidates: + lines.append( + f"| {mode} | (insufficient samples; ≥5 runs/bucket " + f"required) | n/a | n/a |" + ) + continue + # Peak strict-rate; tie-break on smaller bucket (less context + # is cheaper at equivalent strict-rate). + peak_label, peak_rate, peak_n = max( + candidates, key=lambda t: (t[1], -buckets.index( + next(b for b in buckets if b[0] == t[0]) + )) + ) + lines.append( + f"| {mode} | {peak_label} | {peak_rate:.2f} | {peak_n} |" + ) + lines.append("") lines.append("## per question") lines.append("") lines.append( diff --git a/tests/test_bench_qa_sweep.py b/tests/test_bench_qa_sweep.py index 9ecb56d..c87374e 100644 --- a/tests/test_bench_qa_sweep.py +++ b/tests/test_bench_qa_sweep.py @@ -151,6 +151,89 @@ def test_render_markdown_buckets_strict_rate_by_size(qa_sweep): assert " 0.00 " in lines_16kb[0] +def test_render_markdown_recommended_context_budget_section(qa_sweep): + """Bench surfaces a per-mode peak-strict-rate bucket as the + 'recommended context budget' — operator-readable, not auto- + applied. Minimum sample size of 5 runs/bucket; smaller samples + fall through as 'insufficient samples'.""" + # 6 STRICT in <8KB bucket + 6 UNGROUNDED in 16-32KB bucket + # → <8KB has strict-rate 1.0, 16-32KB has 0.0; <8KB recommended. + rows = [] + for _ in range(6): + rows.append(_row( + answer_mode="claim_lattice", + audit_mode="STRICT", + prompt_chars_total=4000, + )) + for _ in range(6): + rows.append(_row( + answer_mode="claim_lattice", + audit_mode="UNGROUNDED", + prompt_chars_total=20000, + )) + summary = qa_sweep._summarize(rows) + md = qa_sweep._render_markdown( + rows, summary, "2026-05-02T00-00-00Z", + ["claim_lattice"], ["q"], n_samples=12, + ) + assert "recommended context budget (learned from this bench)" in md + rec_section = md.split("recommended context budget")[1] + # Peak bucket is <8KB at 100%. + assert "<8KB" in rec_section + assert "1.00" in rec_section + # 16-32KB bucket (0% strict) does NOT show as the peak. + rec_table_lines = [ + L for L in rec_section.splitlines() + if "claim_lattice" in L and "<8KB" in L + ] + assert rec_table_lines, f"missing recommended-bucket row:\n{rec_section[:500]}" + + +def test_render_markdown_recommended_budget_skips_undersampled(qa_sweep): + """Buckets with fewer than 5 runs don't qualify as + 'recommended' — the recommendation needs statistical weight.""" + # 3 STRICT in <8KB (under sample-size floor) + 3 STRICT in + # 16-32KB (also under floor). All buckets have <5 runs → + # 'insufficient samples' fallback. + rows = [ + _row(answer_mode="claim_lattice", audit_mode="STRICT", prompt_chars_total=4000), + _row(answer_mode="claim_lattice", audit_mode="STRICT", prompt_chars_total=4000), + _row(answer_mode="claim_lattice", audit_mode="STRICT", prompt_chars_total=4000), + _row(answer_mode="claim_lattice", audit_mode="UNGROUNDED", prompt_chars_total=20000), + ] + summary = qa_sweep._summarize(rows) + md = qa_sweep._render_markdown( + rows, summary, "2026-05-02T00-00-00Z", + ["claim_lattice"], ["q"], n_samples=4, + ) + rec_section = md.split("recommended context budget")[1] + assert "insufficient samples" in rec_section + + +def test_render_markdown_log_scale_buckets_cover_giant_context(qa_sweep): + """Log-scale buckets reach 1M+ chars so the same bench harness + covers 8B-class models (Hermes 82K context) through 1M-context + models (Gemini / Claude). A 500K-char prompt lands in the + 512K-1M bucket without code change.""" + rows = [] + for _ in range(6): + rows.append(_row( + answer_mode="claim_lattice", + audit_mode="STRICT", + prompt_chars_total=600_000, # 600KB → 512K-1M bucket + )) + summary = qa_sweep._summarize(rows) + md = qa_sweep._render_markdown( + rows, summary, "2026-05-02T00-00-00Z", + ["claim_lattice"], ["q"], n_samples=6, + ) + assert "512K-1M" in md + rec_section = md.split("recommended context budget")[1] + # 600K-char prompts land in the giant bucket; recommendation + # should pick it. + assert "512K-1M" in rec_section + + def test_render_markdown_skips_empty_buckets(qa_sweep): """Bucket with zero runs in a mode shouldn't render an empty row — the table should only include cells with data."""