diff --git a/bench/watt_bench.py b/bench/watt_bench.py index 0b7822d..40b7f04 100644 --- a/bench/watt_bench.py +++ b/bench/watt_bench.py @@ -387,12 +387,17 @@ def energy_cogs(gpu_joules: float | None, window_s: float | None, ready to answer. * gen_draw_w — the actual generation draw (high band), with ``gen_duty`` = fraction of the window generating. - * gross COGS — ALL measured joules / tokens: the all-in cost, - amortized across throughput (state-agnostic, so - this number was always correct). - * marginal COGS — joules ABOVE the serving floor / tokens: what - one more request's generation actually adds. - Falls back to idle floor if no band split. + * marginal COGS — THE GPU COST OF A QUERY: joules ABOVE the + serving floor / tokens — the generation burst. + Retrieval/verify are CPU/SQLite (GPU untouched), + so this generation-only number is what the query + actually causes on the GPU. Falls back to idle + floor if no band split. + * gross — wall-clock window joules / tokens. NOT this + query's GPU cost: it charges the always-on + model-resident floor for the seconds we spend in + CPU-side retrieval (the card is free for other + traffic then). Kept for reference, not as COGS. kWh = J / 3.6e6; ``$/1k-tok`` is the unit that compares to API pricing. Only ``price_per_kwh`` is an operator input. See docs/stock-v1-config.md. @@ -668,12 +673,17 @@ def main() -> int: + ("" if b.get("bimodal") else " [unimodal — no gen state]")) cg = cell["energy_cogs"] if cg.get("available"): - print(f" COGS @${cg['price_per_kwh']}/kWh: " - f"gross ${cg['gross_usd_per_1k_tok']}/1k-tok" - + (f" · marginal ${cg['marginal_usd_per_1k_tok']}/1k-tok " - f"(vs {cg['marginal_floor']}-floor {cg['marginal_floor_w']}W)" - if "marginal_usd_per_1k_tok" in cg else - " · marginal n/a (no floor)")) + # GPU COGS = generation energy ABOVE the always-on model- + # resident floor (the marginal). Retrieval/verify are CPU/ + # SQLite — the GPU is untouched then, so the gross window + # integral OVER-counts (it charges the shared floor during our + # retrieval gap) and is NOT this query's GPU cost (fox). + print(f" GPU COGS @${cg['price_per_kwh']}/kWh: " + + (f"${cg['marginal_usd_per_1k_tok']}/1k-tok generation " + f"(above {cg['marginal_floor']}-floor {cg['marginal_floor_w']}W)" + if "marginal_usd_per_1k_tok" in cg else "n/a") + + f" · [window-total ${cg['gross_usd_per_1k_tok']}/1k-tok " + f"incl. shared idle during retrieval — NOT query-attributable]") for mkey in models: run_cell("solo", mkey) diff --git a/docs/stock-v1-config.md b/docs/stock-v1-config.md index 99218f4..4977fa2 100644 --- a/docs/stock-v1-config.md +++ b/docs/stock-v1-config.md @@ -84,18 +84,26 @@ by `bench/jaggedness.py` (#000060). ## Cost axis — energy COGS (companion, #000057) -`bench/watt_bench.py` adds the cost side: GPU energy per question / per -completion-token, on each card. Energy COGS decomposes into the power -states the card actually occupies — idle, warm-idle (model resident, -waiting), and generation (request burst): +`bench/watt_bench.py` adds the cost side: GPU energy per completion-token, +on each card. The card occupies DISTINCT measured states — idle, +middle-idle (model resident, between requests), generation burst — split +data-derived by `watt_probe.classify_power_bands` (largest-gap, no +hardcoded watts). **Crucially: retrieval + verification are CPU/SQLite +work; the GPU is untouched during them** (fox 2026-05-21). So: -- **marginal** `(P_gen − P_warm_idle) · t_gen / tokens` — what one more - token actually costs; -- **gross / amortized** — all-in, including the warm-idle cost of - keeping the model hot, divided across throughput (the COGS-vs- - utilization curve); -- **dollar COGS** — `joules / 3.6e6 → kWh × $/kWh`, then `÷ tokens` - for `$/1k-tok` against API pricing. +- **marginal = THE GPU COGS of a query** — generation energy above the + serving floor, `÷ tokens`. This is what the query actually causes on + the GPU. Measured qwen-nothink/4090 (n=10, isolated): claim_lattice + **$0.21/M-tok**, quote **$0.83/M-tok**, bare solo ~$0.5–0.65/M-tok. +- **gross = wall-clock window total, NOT query-attributable.** It charges + the always-on model-resident floor for the seconds spent in CPU-side + retrieval — energy that exists regardless of the query (the card is + free for other traffic then). Reference only, never the COGS. +- **dollars** — `joules / 3.6e6 → kWh × $/kWh ÷ tokens` for `$/1k-tok` + against API pricing. +- **duty cycle** exposes the shape: a substrate query generates only + ~9–25 % of its wall-clock; the rest is CPU retrieval. The substrate's + real overhead is **latency, not GPU watts.** **Every power state is MEASURED per (card, model, inference-server) at runtime — never hardcoded.** `P_warm_idle` comes from the no-request idle