diff --git a/bench/cross_model_selfplay.py b/bench/cross_model_selfplay.py index c1b5623..8d44744 100644 --- a/bench/cross_model_selfplay.py +++ b/bench/cross_model_selfplay.py @@ -9,24 +9,28 @@ arborist once per configured model, then tabulate: POINTER-LINKED → UNGROUNDED; HYBRID variants flatten to base) * agreement on primary source URI * grounding rate per model - * estimated $/grounded-answer (Hermes $0.09, Qwen $0.16 per call — - override via --price model=cents,model=cents) + * implied spend in fox's natural unit: $/1000 grounded answers + (Hermes $0.09, Qwen $0.16 by default — override via --baseline) * latency per call +The COGS unit is **dollars per 1000 grounded answers**, an outcome +metric — failed/ungrounded calls are already amortized into the +baseline. A model's baseline × grounded_count / 1000 = implied spend +for those grounded answers. + Outputs: bench/cross_model_results/.jsonl one row per (question, model) bench/cross_model_results/.md markdown summary -The cost-per-grounded-answer column is the score that matters: cheap -calls that ground are wins; expensive calls that ground are wins worth -defending; ungrounded calls are wasted spend. - Usage: make bench-cross-model # smoke fixture, Hermes + Qwen make bench-cross-model BENCH_CM_QUESTIONS=bench/qa_questions.txt + # custom baseline (dollars per 1000 grounded answers): + python3 bench/cross_model_selfplay.py --baseline 'hermes=0.09,qwen=0.16' + # custom model list (name=endpoint:model_id, comma-separated): python3 bench/cross_model_selfplay.py \\ --models 'hermes=https://hermes.ai.unturf.com/v1:adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic,qwen=https://qwen.ai.unturf.com/v1:Qwen3.6-27B-UD-Q4_K_XL.gguf' @@ -52,9 +56,11 @@ DEFAULT_MODELS = [ ("qwen", "https://qwen.ai.unturf.com/v1", "Qwen3.6-27B-UD-Q4_K_XL.gguf"), ] -# Per fox 2026-05-31 — average cost per grounded answer at our infra -# scale. Hermes (8B) on 3090, Qwen (30B) on 4090. Override via --price. -DEFAULT_PRICE_CENTS = {"hermes": 9.0, "qwen": 16.0} +# Per fox 2026-05-31 — baseline COGS in dollars per 1000 grounded +# answers. Hermes (8B) on 3090, Qwen (30B) on 4090. The baseline +# already amortizes ungrounded calls — it's an outcome metric, not a +# per-call price. Override via --baseline. +DEFAULT_DOLLARS_PER_1K_GROUNDED = {"hermes": 0.09, "qwen": 0.16} # audit_mode → numeric rank for "stronger" comparison. Strips # "-PARTIAL" suffix that HYBRID modes carry. @@ -115,16 +121,17 @@ def _parse_models(spec: str) -> list[tuple[str, str, str]]: return out -def _parse_price(spec: str | None) -> dict[str, float]: +def _parse_baseline(spec: str | None) -> dict[str, float]: + """Parse name=dollars_per_1k_grounded,... into a dict.""" if not spec: - return dict(DEFAULT_PRICE_CENTS) - out = dict(DEFAULT_PRICE_CENTS) + return dict(DEFAULT_DOLLARS_PER_1K_GROUNDED) + out = dict(DEFAULT_DOLLARS_PER_1K_GROUNDED) for chunk in spec.split(","): if "=" not in chunk: continue - name, cents = chunk.split("=", 1) + name, dollars = chunk.split("=", 1) try: - out[name.strip()] = float(cents.strip()) + out[name.strip()] = float(dollars.strip()) except ValueError: pass return out @@ -195,9 +202,9 @@ def _stronger_model(per_model: dict[str, dict]) -> str: def _summarize( rows: list[dict], models: list[tuple[str, str, str]], - price_cents: dict[str, float], + baseline_per_1k: dict[str, float], ) -> str: - """Build the markdown summary.""" + """Build the markdown summary. COGS unit: $ per 1k grounded answers.""" by_q: dict[str, dict[str, dict]] = defaultdict(dict) for r in rows: by_q[r["question"]][r["model_name"]] = r["result"] @@ -209,12 +216,15 @@ def _summarize( out.append("") out.append(f"Questions: {n_q}. Models: {', '.join(model_names)}.") out.append("") + out.append("COGS unit: **dollars per 1000 grounded answers** (fox's natural unit). " + "Implied spend uses each model's baseline × grounded_count / 1000.") + out.append("") # Per-model summary out.append("## Per-model summary") out.append("") - out.append("| model | grounded | rate | avg latency | est $/grounded |") - out.append("|---|---:|---:|---:|---:|") + out.append("| model | grounded | rate | avg latency | baseline $/1k grounded | implied spend |") + out.append("|---|---:|---:|---:|---:|---:|") for name in model_names: grounded = sum( 1 for q in by_q if _is_grounded(by_q[q].get(name, {}).get("audit_mode")) @@ -224,12 +234,11 @@ def _summarize( avg_lat = ( sum(by_q[q].get(name, {}).get("_elapsed_s", 0) for q in by_q) / max(total, 1) ) - per_call_cents = price_cents.get(name, 0.0) - cost_per_grounded = (per_call_cents * total / grounded) if grounded else float("nan") - cost_str = "n/a" if cost_per_grounded != cost_per_grounded else f"${cost_per_grounded / 100:.2f}" + base = baseline_per_1k.get(name, 0.0) + spend = grounded * base / 1000 out.append( f"| {name} | {grounded}/{total} | {rate:.0%} " - f"| {avg_lat:.1f}s | {cost_str} |" + f"| {avg_lat:.1f}s | ${base:.2f} | ${spend:.6f} |" ) out.append("") @@ -257,48 +266,78 @@ def _summarize( out.append("") # Cost analysis - out.append("## Cost analysis (north star: $/grounded)") + out.append("## Cost analysis (north star: $/1k grounded)") out.append("") - total_cost = 0.0 + total_spend = 0.0 total_grounded = 0 for name in model_names: - per_call_cents = price_cents.get(name, 0.0) - n_calls = sum(1 for q in by_q if name in by_q[q] and "_error" not in by_q[q][name]) + base = baseline_per_1k.get(name, 0.0) n_g = sum(1 for q in by_q if _is_grounded(by_q[q].get(name, {}).get("audit_mode"))) - spent = per_call_cents * n_calls / 100 - total_cost += spent + spent = n_g * base / 1000 + total_spend += spent total_grounded += n_g - out.append(f"* **{name}** — {n_calls} calls × ${per_call_cents/100:.2f} = ${spent:.2f}, {n_g} grounded") + out.append(f"* **{name}** — {n_g} grounded × ${base:.2f}/1k = ${spent:.6f}") out.append("") - out.append(f"**Total spend**: ${total_cost:.2f}. **Total grounded answers**: {total_grounded}.") + out.append(f"**Total implied spend**: ${total_spend:.6f}. **Total grounded answers**: {total_grounded}.") if total_grounded: - out.append(f"**Effective $/grounded** (sum of all model spend / total grounded answers): ${total_cost / total_grounded:.3f}") + # Effective $/1k = total_spend × (1000 / total_grounded) + out.append(f"**Effective COGS**: ${total_spend * 1000 / total_grounded:.4f} per 1k grounded " + f"(weighted across models in this run).") out.append("") - # Routing hint — what cheap-first cascade would have cost + # Routing hint — cheap-first cascade out.append("## Cheap-first cascade (try cheapest, escalate on UNGROUNDED)") out.append("") - ordered = sorted(model_names, key=lambda n: price_cents.get(n, 0)) - cascade_cost = 0.0 - cascade_grounded = 0 + ordered = sorted(model_names, key=lambda n: baseline_per_1k.get(n, 0)) + # Per-q: which model ultimately grounded? Count by source. + cascade_grounded_by_model: dict[str, int] = defaultdict(int) + cascade_total_grounded = 0 for q in by_q: - for i, name in enumerate(ordered): + for name in ordered: res = by_q[q].get(name, {}) if "_error" in res: continue - cascade_cost += price_cents.get(name, 0) / 100 if _is_grounded(res.get("audit_mode")): - cascade_grounded += 1 + cascade_grounded_by_model[name] += 1 + cascade_total_grounded += 1 break - # else fall through to next model - if cascade_grounded: - out.append( - f"Cascade ({' → '.join(ordered)}): ${cascade_cost:.2f} for " - f"{cascade_grounded}/{n_q} grounded — " - f"${cascade_cost / cascade_grounded:.3f}/grounded." + # else fall through to next model in cascade + cascade_spend = sum( + cnt * baseline_per_1k.get(name, 0) / 1000 + for name, cnt in cascade_grounded_by_model.items() + ) + if cascade_total_grounded: + breakdown = ", ".join( + f"{cnt} from {name}" for name, cnt in cascade_grounded_by_model.items() ) + cascade_cogs = cascade_spend * 1000 / cascade_total_grounded + out.append( + f"Cascade ({' → '.join(ordered)}): {cascade_total_grounded}/{n_q} grounded " + f"({breakdown}). Implied spend ${cascade_spend:.6f} → " + f"**${cascade_cogs:.4f}/1k grounded**." + ) + # Compare to always-most-expensive (defensive baseline) + most_expensive = ordered[-1] + max_base = baseline_per_1k.get(most_expensive, 0) + always_grounded = sum( + 1 for q in by_q + if _is_grounded(by_q[q].get(most_expensive, {}).get("audit_mode")) + ) + always_spend = always_grounded * max_base / 1000 + if always_grounded: + always_cogs = always_spend * 1000 / always_grounded + delta_pct = (cascade_cogs / always_cogs - 1) * 100 if always_cogs else 0 + sign = "+" if delta_pct >= 0 else "" + out.append("") + out.append( + f"vs always-{most_expensive}: {always_grounded}/{n_q} grounded, " + f"${always_spend:.6f} spend, ${always_cogs:.4f}/1k. " + f"Cascade delta: **{sign}{delta_pct:.1f}%**." + ) out.append("") - out.append("Cascade beats always-Qwen when the cheaper model grounds often enough that escalation is rare. Compare to per-model rows above.") + out.append("Caveat: cascade math credits each grounded answer to its source model's baseline. " + "Real-world failed-call overhead (an UNGROUNDED hermes call that escalates to qwen) " + "is not separately accounted — it's amortized into hermes's baseline by definition.") out.append("") return "\n".join(out) @@ -315,11 +354,16 @@ def main() -> int: help="comma-separated name=endpoint:model_id; defaults to hermes + qwen", ) ap.add_argument( - "--price", default=None, - help="comma-separated name=cents per call; defaults to hermes=9,qwen=16", + "--baseline", default=None, + help="comma-separated name=dollars per 1k grounded answers; " + "defaults to hermes=0.09,qwen=0.16 (fox's COGS unit)", ) ap.add_argument("--top-k", type=int, default=8) - ap.add_argument("--timeout", type=int, default=180) + # 600s default — queue depth on shared endpoints (e.g. hermes + # under contention from another agent) can push a single call + # past 3 minutes even when the model is healthy. 180s was too + # aggressive against real-world infrastructure load. + ap.add_argument("--timeout", type=int, default=600) ap.add_argument( "--burn", action="store_true", help="force fresh inference (skip cache) on every call", @@ -341,7 +385,7 @@ def main() -> int: models = _parse_models(args.models) else: models = list(DEFAULT_MODELS) - price = _parse_price(args.price) + baseline = _parse_baseline(args.baseline) if not ARBORIST.exists(): print(f"arborist binary not found: {ARBORIST}", file=sys.stderr) @@ -378,7 +422,7 @@ def main() -> int: elapsed = res.get("_elapsed_s", "?") print(f" → {audit} ({elapsed}s)", file=sys.stderr) - md_path.write_text(_summarize(rows, models, price)) + md_path.write_text(_summarize(rows, models, baseline)) print(f"\nJSONL: {jsonl_path}", file=sys.stderr) print(f"Summary: {md_path}", file=sys.stderr) print()