bench: cross-model self-play — fix COGS units to $/1k grounded
Renamed DEFAULT_PRICE_CENTS (per-call) → DEFAULT_DOLLARS_PER_1K_GROUNDED. Per fox 2026-05-31: at owned-hardware scale, COGS is naturally expressed per-1000-grounded-answers (hermes $0.09, qwen $0.16), not per-call cents. The baseline already amortizes ungrounded calls. Report now shows: per-model implied spend, effective COGS in $/1k grounded, and cascade delta as % vs always-most-expensive. 20-question live bench (real run): cascade beats always-qwen by -39% COGS AND grounds more answers (19/20 vs 17/20). Hermes and qwen are equally reliable (17/20 each) but catch *different* questions — additive coverage.
This commit is contained in:
parent
43c97a03e7
commit
d8ed8fbd5b
1 changed files with 94 additions and 50 deletions
|
|
@ -9,24 +9,28 @@ arborist once per configured model, then tabulate:
|
|||
POINTER-LINKED → UNGROUNDED; HYBRID variants flatten to base)
|
||||
* agreement on primary source URI
|
||||
* grounding rate per model
|
||||
* estimated $/grounded-answer (Hermes $0.09, Qwen $0.16 per call —
|
||||
override via --price model=cents,model=cents)
|
||||
* implied spend in fox's natural unit: $/1000 grounded answers
|
||||
(Hermes $0.09, Qwen $0.16 by default — override via --baseline)
|
||||
* latency per call
|
||||
|
||||
The COGS unit is **dollars per 1000 grounded answers**, an outcome
|
||||
metric — failed/ungrounded calls are already amortized into the
|
||||
baseline. A model's baseline × grounded_count / 1000 = implied spend
|
||||
for those grounded answers.
|
||||
|
||||
Outputs:
|
||||
|
||||
bench/cross_model_results/<utc-iso>.jsonl one row per (question, model)
|
||||
bench/cross_model_results/<utc-iso>.md markdown summary
|
||||
|
||||
The cost-per-grounded-answer column is the score that matters: cheap
|
||||
calls that ground are wins; expensive calls that ground are wins worth
|
||||
defending; ungrounded calls are wasted spend.
|
||||
|
||||
Usage:
|
||||
|
||||
make bench-cross-model # smoke fixture, Hermes + Qwen
|
||||
make bench-cross-model BENCH_CM_QUESTIONS=bench/qa_questions.txt
|
||||
|
||||
# custom baseline (dollars per 1000 grounded answers):
|
||||
python3 bench/cross_model_selfplay.py --baseline 'hermes=0.09,qwen=0.16'
|
||||
|
||||
# custom model list (name=endpoint:model_id, comma-separated):
|
||||
python3 bench/cross_model_selfplay.py \\
|
||||
--models 'hermes=https://hermes.ai.unturf.com/v1:adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic,qwen=https://qwen.ai.unturf.com/v1:Qwen3.6-27B-UD-Q4_K_XL.gguf'
|
||||
|
|
@ -52,9 +56,11 @@ DEFAULT_MODELS = [
|
|||
("qwen", "https://qwen.ai.unturf.com/v1",
|
||||
"Qwen3.6-27B-UD-Q4_K_XL.gguf"),
|
||||
]
|
||||
# Per fox 2026-05-31 — average cost per grounded answer at our infra
|
||||
# scale. Hermes (8B) on 3090, Qwen (30B) on 4090. Override via --price.
|
||||
DEFAULT_PRICE_CENTS = {"hermes": 9.0, "qwen": 16.0}
|
||||
# Per fox 2026-05-31 — baseline COGS in dollars per 1000 grounded
|
||||
# answers. Hermes (8B) on 3090, Qwen (30B) on 4090. The baseline
|
||||
# already amortizes ungrounded calls — it's an outcome metric, not a
|
||||
# per-call price. Override via --baseline.
|
||||
DEFAULT_DOLLARS_PER_1K_GROUNDED = {"hermes": 0.09, "qwen": 0.16}
|
||||
|
||||
# audit_mode → numeric rank for "stronger" comparison. Strips
|
||||
# "-PARTIAL" suffix that HYBRID modes carry.
|
||||
|
|
@ -115,16 +121,17 @@ def _parse_models(spec: str) -> list[tuple[str, str, str]]:
|
|||
return out
|
||||
|
||||
|
||||
def _parse_price(spec: str | None) -> dict[str, float]:
|
||||
def _parse_baseline(spec: str | None) -> dict[str, float]:
|
||||
"""Parse name=dollars_per_1k_grounded,... into a dict."""
|
||||
if not spec:
|
||||
return dict(DEFAULT_PRICE_CENTS)
|
||||
out = dict(DEFAULT_PRICE_CENTS)
|
||||
return dict(DEFAULT_DOLLARS_PER_1K_GROUNDED)
|
||||
out = dict(DEFAULT_DOLLARS_PER_1K_GROUNDED)
|
||||
for chunk in spec.split(","):
|
||||
if "=" not in chunk:
|
||||
continue
|
||||
name, cents = chunk.split("=", 1)
|
||||
name, dollars = chunk.split("=", 1)
|
||||
try:
|
||||
out[name.strip()] = float(cents.strip())
|
||||
out[name.strip()] = float(dollars.strip())
|
||||
except ValueError:
|
||||
pass
|
||||
return out
|
||||
|
|
@ -195,9 +202,9 @@ def _stronger_model(per_model: dict[str, dict]) -> str:
|
|||
|
||||
def _summarize(
|
||||
rows: list[dict], models: list[tuple[str, str, str]],
|
||||
price_cents: dict[str, float],
|
||||
baseline_per_1k: dict[str, float],
|
||||
) -> str:
|
||||
"""Build the markdown summary."""
|
||||
"""Build the markdown summary. COGS unit: $ per 1k grounded answers."""
|
||||
by_q: dict[str, dict[str, dict]] = defaultdict(dict)
|
||||
for r in rows:
|
||||
by_q[r["question"]][r["model_name"]] = r["result"]
|
||||
|
|
@ -209,12 +216,15 @@ def _summarize(
|
|||
out.append("")
|
||||
out.append(f"Questions: {n_q}. Models: {', '.join(model_names)}.")
|
||||
out.append("")
|
||||
out.append("COGS unit: **dollars per 1000 grounded answers** (fox's natural unit). "
|
||||
"Implied spend uses each model's baseline × grounded_count / 1000.")
|
||||
out.append("")
|
||||
|
||||
# Per-model summary
|
||||
out.append("## Per-model summary")
|
||||
out.append("")
|
||||
out.append("| model | grounded | rate | avg latency | est $/grounded |")
|
||||
out.append("|---|---:|---:|---:|---:|")
|
||||
out.append("| model | grounded | rate | avg latency | baseline $/1k grounded | implied spend |")
|
||||
out.append("|---|---:|---:|---:|---:|---:|")
|
||||
for name in model_names:
|
||||
grounded = sum(
|
||||
1 for q in by_q if _is_grounded(by_q[q].get(name, {}).get("audit_mode"))
|
||||
|
|
@ -224,12 +234,11 @@ def _summarize(
|
|||
avg_lat = (
|
||||
sum(by_q[q].get(name, {}).get("_elapsed_s", 0) for q in by_q) / max(total, 1)
|
||||
)
|
||||
per_call_cents = price_cents.get(name, 0.0)
|
||||
cost_per_grounded = (per_call_cents * total / grounded) if grounded else float("nan")
|
||||
cost_str = "n/a" if cost_per_grounded != cost_per_grounded else f"${cost_per_grounded / 100:.2f}"
|
||||
base = baseline_per_1k.get(name, 0.0)
|
||||
spend = grounded * base / 1000
|
||||
out.append(
|
||||
f"| {name} | {grounded}/{total} | {rate:.0%} "
|
||||
f"| {avg_lat:.1f}s | {cost_str} |"
|
||||
f"| {avg_lat:.1f}s | ${base:.2f} | ${spend:.6f} |"
|
||||
)
|
||||
out.append("")
|
||||
|
||||
|
|
@ -257,48 +266,78 @@ def _summarize(
|
|||
out.append("")
|
||||
|
||||
# Cost analysis
|
||||
out.append("## Cost analysis (north star: $/grounded)")
|
||||
out.append("## Cost analysis (north star: $/1k grounded)")
|
||||
out.append("")
|
||||
total_cost = 0.0
|
||||
total_spend = 0.0
|
||||
total_grounded = 0
|
||||
for name in model_names:
|
||||
per_call_cents = price_cents.get(name, 0.0)
|
||||
n_calls = sum(1 for q in by_q if name in by_q[q] and "_error" not in by_q[q][name])
|
||||
base = baseline_per_1k.get(name, 0.0)
|
||||
n_g = sum(1 for q in by_q if _is_grounded(by_q[q].get(name, {}).get("audit_mode")))
|
||||
spent = per_call_cents * n_calls / 100
|
||||
total_cost += spent
|
||||
spent = n_g * base / 1000
|
||||
total_spend += spent
|
||||
total_grounded += n_g
|
||||
out.append(f"* **{name}** — {n_calls} calls × ${per_call_cents/100:.2f} = ${spent:.2f}, {n_g} grounded")
|
||||
out.append(f"* **{name}** — {n_g} grounded × ${base:.2f}/1k = ${spent:.6f}")
|
||||
out.append("")
|
||||
out.append(f"**Total spend**: ${total_cost:.2f}. **Total grounded answers**: {total_grounded}.")
|
||||
out.append(f"**Total implied spend**: ${total_spend:.6f}. **Total grounded answers**: {total_grounded}.")
|
||||
if total_grounded:
|
||||
out.append(f"**Effective $/grounded** (sum of all model spend / total grounded answers): ${total_cost / total_grounded:.3f}")
|
||||
# Effective $/1k = total_spend × (1000 / total_grounded)
|
||||
out.append(f"**Effective COGS**: ${total_spend * 1000 / total_grounded:.4f} per 1k grounded "
|
||||
f"(weighted across models in this run).")
|
||||
out.append("")
|
||||
|
||||
# Routing hint — what cheap-first cascade would have cost
|
||||
# Routing hint — cheap-first cascade
|
||||
out.append("## Cheap-first cascade (try cheapest, escalate on UNGROUNDED)")
|
||||
out.append("")
|
||||
ordered = sorted(model_names, key=lambda n: price_cents.get(n, 0))
|
||||
cascade_cost = 0.0
|
||||
cascade_grounded = 0
|
||||
ordered = sorted(model_names, key=lambda n: baseline_per_1k.get(n, 0))
|
||||
# Per-q: which model ultimately grounded? Count by source.
|
||||
cascade_grounded_by_model: dict[str, int] = defaultdict(int)
|
||||
cascade_total_grounded = 0
|
||||
for q in by_q:
|
||||
for i, name in enumerate(ordered):
|
||||
for name in ordered:
|
||||
res = by_q[q].get(name, {})
|
||||
if "_error" in res:
|
||||
continue
|
||||
cascade_cost += price_cents.get(name, 0) / 100
|
||||
if _is_grounded(res.get("audit_mode")):
|
||||
cascade_grounded += 1
|
||||
cascade_grounded_by_model[name] += 1
|
||||
cascade_total_grounded += 1
|
||||
break
|
||||
# else fall through to next model
|
||||
if cascade_grounded:
|
||||
out.append(
|
||||
f"Cascade ({' → '.join(ordered)}): ${cascade_cost:.2f} for "
|
||||
f"{cascade_grounded}/{n_q} grounded — "
|
||||
f"${cascade_cost / cascade_grounded:.3f}/grounded."
|
||||
# else fall through to next model in cascade
|
||||
cascade_spend = sum(
|
||||
cnt * baseline_per_1k.get(name, 0) / 1000
|
||||
for name, cnt in cascade_grounded_by_model.items()
|
||||
)
|
||||
if cascade_total_grounded:
|
||||
breakdown = ", ".join(
|
||||
f"{cnt} from {name}" for name, cnt in cascade_grounded_by_model.items()
|
||||
)
|
||||
cascade_cogs = cascade_spend * 1000 / cascade_total_grounded
|
||||
out.append(
|
||||
f"Cascade ({' → '.join(ordered)}): {cascade_total_grounded}/{n_q} grounded "
|
||||
f"({breakdown}). Implied spend ${cascade_spend:.6f} → "
|
||||
f"**${cascade_cogs:.4f}/1k grounded**."
|
||||
)
|
||||
# Compare to always-most-expensive (defensive baseline)
|
||||
most_expensive = ordered[-1]
|
||||
max_base = baseline_per_1k.get(most_expensive, 0)
|
||||
always_grounded = sum(
|
||||
1 for q in by_q
|
||||
if _is_grounded(by_q[q].get(most_expensive, {}).get("audit_mode"))
|
||||
)
|
||||
always_spend = always_grounded * max_base / 1000
|
||||
if always_grounded:
|
||||
always_cogs = always_spend * 1000 / always_grounded
|
||||
delta_pct = (cascade_cogs / always_cogs - 1) * 100 if always_cogs else 0
|
||||
sign = "+" if delta_pct >= 0 else ""
|
||||
out.append("")
|
||||
out.append(
|
||||
f"vs always-{most_expensive}: {always_grounded}/{n_q} grounded, "
|
||||
f"${always_spend:.6f} spend, ${always_cogs:.4f}/1k. "
|
||||
f"Cascade delta: **{sign}{delta_pct:.1f}%**."
|
||||
)
|
||||
out.append("")
|
||||
out.append("Cascade beats always-Qwen when the cheaper model grounds often enough that escalation is rare. Compare to per-model rows above.")
|
||||
out.append("Caveat: cascade math credits each grounded answer to its source model's baseline. "
|
||||
"Real-world failed-call overhead (an UNGROUNDED hermes call that escalates to qwen) "
|
||||
"is not separately accounted — it's amortized into hermes's baseline by definition.")
|
||||
out.append("")
|
||||
return "\n".join(out)
|
||||
|
||||
|
|
@ -315,11 +354,16 @@ def main() -> int:
|
|||
help="comma-separated name=endpoint:model_id; defaults to hermes + qwen",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--price", default=None,
|
||||
help="comma-separated name=cents per call; defaults to hermes=9,qwen=16",
|
||||
"--baseline", default=None,
|
||||
help="comma-separated name=dollars per 1k grounded answers; "
|
||||
"defaults to hermes=0.09,qwen=0.16 (fox's COGS unit)",
|
||||
)
|
||||
ap.add_argument("--top-k", type=int, default=8)
|
||||
ap.add_argument("--timeout", type=int, default=180)
|
||||
# 600s default — queue depth on shared endpoints (e.g. hermes
|
||||
# under contention from another agent) can push a single call
|
||||
# past 3 minutes even when the model is healthy. 180s was too
|
||||
# aggressive against real-world infrastructure load.
|
||||
ap.add_argument("--timeout", type=int, default=600)
|
||||
ap.add_argument(
|
||||
"--burn", action="store_true",
|
||||
help="force fresh inference (skip cache) on every call",
|
||||
|
|
@ -341,7 +385,7 @@ def main() -> int:
|
|||
models = _parse_models(args.models)
|
||||
else:
|
||||
models = list(DEFAULT_MODELS)
|
||||
price = _parse_price(args.price)
|
||||
baseline = _parse_baseline(args.baseline)
|
||||
|
||||
if not ARBORIST.exists():
|
||||
print(f"arborist binary not found: {ARBORIST}", file=sys.stderr)
|
||||
|
|
@ -378,7 +422,7 @@ def main() -> int:
|
|||
elapsed = res.get("_elapsed_s", "?")
|
||||
print(f" → {audit} ({elapsed}s)", file=sys.stderr)
|
||||
|
||||
md_path.write_text(_summarize(rows, models, price))
|
||||
md_path.write_text(_summarize(rows, models, baseline))
|
||||
print(f"\nJSONL: {jsonl_path}", file=sys.stderr)
|
||||
print(f"Summary: {md_path}", file=sys.stderr)
|
||||
print()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue