From 1aff09f021b999f83cbdc3bb655a2a9f03d36e3b Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Thu, 21 May 2026 10:19:41 -0400 Subject: [PATCH] =?UTF-8?q?feat(#000057):=20energy-COGS=20layer=20for=20wa?= =?UTF-8?q?tt=5Fbench=20=E2=80=94=20marginal=20vs=20gross=20$/1k-tok?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fox 2026-05-21: compute cost-of-goods-sold by kWh vs tokens, with the three power states (idle / warm-idle / generation) MEASURED per card×model×server — never hardcoded (his 40/127/380 W were illustrative of one 3090). The only operator input is --price-per-kwh (default 0.33 USD/kWh, a configurable site rate). energy_cogs() (pure, unit-tested) decomposes measured generation energy against the measured warm-idle baseline: * gross — all measured joules over the window (all-in, includes the warm-idle cost of keeping the model hot, amortized). * marginal — joules ABOVE warm-idle: what one more request's burst actually costs (clamped >=0). kWh = J/3.6e6; $/1k-tok is the unit that compares to API pricing. Both surface per cell + a COGS print line. watt_bench's arborist arm now loads the frozen bench.stock_v1 policy (--answer-mode, drift-guarded on non-reasoning) so cost is measured for the SAME substrate the campaign grades. Cells record window_start/end_unix so a post-hoc load_monitor queue-depth cross-ref can flag organic-traffic contamination on the non-isolated single-slot endpoints. 6 COGS tests; full suite 2534 passed. --- bench/watt_bench.py | 103 +++++++++++++++++++++++++++++++++++++--- docs/stock-v1-config.md | 11 +++-- tests/test_watt_cogs.py | 59 +++++++++++++++++++++++ 3 files changed, 163 insertions(+), 10 deletions(-) create mode 100644 tests/test_watt_cogs.py diff --git a/bench/watt_bench.py b/bench/watt_bench.py index 57ae9c7..3f36d87 100644 --- a/bench/watt_bench.py +++ b/bench/watt_bench.py @@ -355,6 +355,61 @@ def _solo_call(client, mkey: str, question: str) -> tuple[str, int]: return ans, max(1, len(ans) // 4) +def energy_cogs(gpu_joules: float | None, window_s: float | None, + tot_tok: int, idle_mean_w: float | None, + price_per_kwh: float) -> dict: + """Energy cost-of-goods-sold from MEASURED power — no hardcoded states. + + fox 2026-05-21: idle / warm-idle / generation watts differ for every + card × model × inference-server, so every power number here is + MEASURED at runtime — ``gpu_joules`` / ``window_s`` / ``mean_gen_w`` + from the driven window, ``idle_mean_w`` from the no-request idle + baseline. The ONLY operator input is ``price_per_kwh`` (a configurable + site rate, default 0.33 USD/kWh). Nothing is baked in. + + Decomposition (the three power states the card occupies): + + * warm-idle — model resident, waiting (measured ``idle_mean_w``). + * generation — measured mean power over the driven window. + * gross COGS — all measured joules over the window, including the + warm-idle cost of keeping the model hot; the all-in + cost amortized across throughput. + * marginal COGS — joules ABOVE the warm-idle baseline: what one more + request's generation burst actually costs. Clamped + ≥0 (a noisy idle window can exceed a quiet load one). + + kWh = J / 3.6e6; ``$/1k-tok`` = $ / tokens × 1000 — the unit that + compares to hosted-API pricing. See docs/stock-v1-config.md. + """ + if not gpu_joules or not tot_tok or not window_s: + return {"available": False, "price_per_kwh": price_per_kwh} + J_PER_KWH = 3.6e6 + mean_gen_w = gpu_joules / window_s + gross_usd = gpu_joules / J_PER_KWH * price_per_kwh + out = { + "available": True, + "price_per_kwh": price_per_kwh, + "warm_idle_w_measured": (round(idle_mean_w, 1) + if idle_mean_w is not None else None), + "mean_gen_w_measured": round(mean_gen_w, 1), + "window_s": round(window_s, 1), + "gross_joules": round(gpu_joules, 1), + "gross_usd": round(gross_usd, 6), + "gross_usd_per_1k_tok": round(gross_usd / tot_tok * 1000, 6), + } + if idle_mean_w is not None: + idle_joules = idle_mean_w * window_s + marginal_joules = max(0.0, gpu_joules - idle_joules) + marginal_usd = marginal_joules / J_PER_KWH * price_per_kwh + out.update({ + "warm_idle_joules": round(idle_joules, 1), + "marginal_joules": round(marginal_joules, 1), + "marginal_usd": round(marginal_usd, 6), + "marginal_usd_per_1k_tok": round(marginal_usd / tot_tok * 1000, 6), + }) + return out + + def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--fixture", default="bench/qa_questions_stale_map.json") @@ -386,7 +441,17 @@ def main() -> int: "entry's endpoint.") ap.add_argument("--sample-interval", type=float, default=0.5) ap.add_argument("--idle-baseline-s", type=float, default=5.0, - help="seconds to sample idle power before driving load") + help="seconds to sample idle (warm-idle) power before " + "driving load — MEASURED, never hardcoded") + ap.add_argument("--price-per-kwh", type=float, default=0.33, + help="electricity rate for energy COGS (USD/kWh; " + "configurable site rate, fox default 0.33). The " + "only non-measured input — power states are all " + "measured per card/model/server at runtime.") + ap.add_argument("--answer-mode", choices=["quote", "claim_lattice"], + default="claim_lattice", + help="STOCK V.1 substrate-ON answer shape for the " + "arborist arm (frozen bench.stock_v1 policy).") ap.add_argument("--cpu-energy-cmd", default="", help="optional privileged reader for RAPL energy_uj " "when it's root-only (PLATYPUS mitigation), e.g. " @@ -457,13 +522,21 @@ def main() -> int: if arm == "solo": ans, ctoks = _solo_call(client, mkey, q) else: # arborist - from arborist.qa.query import ( - DEFAULT_QUERY_POLICY, query) + from arborist.qa.query import query + from bench.stock_v1 import ( + assert_not_drifted as _assert_stock) + from bench.stock_v1 import policy_for as _stock_policy_for gold = _gold(shards_dir, it.get("shard", ""), it["target_root"]) - pol = dict(DEFAULT_QUERY_POLICY, - answer_mode="claim_lattice") - if MODELS[mkey].get("reasoning"): + # STOCK V.1 frozen substrate-ON policy (cost is + # measured for the SAME substrate the campaign + # grades). Reasoning refs (phase 3) layer the + # documented JSON overrides and skip the assert. + reasoning = bool(MODELS[mkey].get("reasoning")) + if not reasoning: + _assert_stock(a.answer_mode) + pol = _stock_policy_for(a.answer_mode) + if reasoning: pol["claim_lattice_json_stop_sequences"] = [] pol["max_tokens"] = 8192 qa_db = Path("/tmp") / f"watt_{ts}_{mkey}.db" @@ -510,6 +583,15 @@ def main() -> int: if gpu_j and tot_tok else None), "mean_latency_s": (round(sum(r["latency_s"] for r in rows) / nq, 2) if nq else None), + # Energy COGS — measured power vs measured warm-idle baseline, + # only price_per_kwh is an operator input. Window timestamps + # let a post-hoc load_monitor cross-ref flag organic-traffic + # contamination under non-isolation (single-slot endpoints). + "energy_cogs": energy_cogs( + gpu_j, st.get("window_s"), tot_tok, + (idle or {}).get("mean_w"), a.price_per_kwh), + "window_start_unix": round(t0, 3), + "window_end_unix": round(t0 + elapsed, 3), } cells.append(cell) with open(outp, "a") as f: @@ -522,6 +604,15 @@ def main() -> int: f"cpu={cell['cpu_joules_per_question']} " f"tot={cell['total_joules_per_question']} " f"lat={cell['mean_latency_s']}s") + cg = cell["energy_cogs"] + if cg.get("available"): + print(f" COGS @${cg['price_per_kwh']}/kWh: " + f"gross ${cg['gross_usd_per_1k_tok']}/1k-tok " + f"(gen {cg['mean_gen_w_measured']}W)" + + (f" · marginal ${cg['marginal_usd_per_1k_tok']}/1k-tok " + f"(vs warm-idle {cg['warm_idle_w_measured']}W)" + if "marginal_usd_per_1k_tok" in cg else + " · marginal n/a (no idle baseline)")) for mkey in models: run_cell("solo", mkey) diff --git a/docs/stock-v1-config.md b/docs/stock-v1-config.md index c35e61e..99218f4 100644 --- a/docs/stock-v1-config.md +++ b/docs/stock-v1-config.md @@ -98,10 +98,13 @@ waiting), and generation (request burst): for `$/1k-tok` against API pricing. **Every power state is MEASURED per (card, model, inference-server) at -runtime — never hardcoded.** `P_warm_idle` comes from a no-request -calibration window; `P_gen` from the burst during actual completions. -Each card × model × server has its own profile. The only operator input -is `$/kWh`, a configurable site flag. +runtime — never hardcoded.** `P_warm_idle` comes from the no-request idle +baseline (`--idle-baseline-s`, default 5 s); `P_gen` from the burst +during actual completions. Each card × model × server has its own +profile. The only operator input is `--price-per-kwh` (default 0.33 +USD/kWh, a configurable site rate). `bench/watt_bench.py:energy_cogs` +emits gross + marginal `$/1k-tok` per cell and records the measurement +window (`window_start_unix`/`window_end_unix`) for the cross-ref below. Because the single-slot endpoints (hermes 3090 / vLLM, qwen 4090 / llama.cpp) serve **real internet traffic and cannot be isolated**, diff --git a/tests/test_watt_cogs.py b/tests/test_watt_cogs.py new file mode 100644 index 0000000..3552419 --- /dev/null +++ b/tests/test_watt_cogs.py @@ -0,0 +1,59 @@ +"""Energy-COGS math for bench/watt_bench.py:energy_cogs (#000057). + +Pure function, no GPU / no nvidia-smi. fox 2026-05-21: power states are +MEASURED per card/model/server; the only operator input is price_per_kwh. +These tests pin the joules→kWh→$ arithmetic and the warm-idle marginal +decomposition. See docs/calculator-test-patterns.md. +""" +from bench.watt_bench import energy_cogs + + +def test_gross_cogs_one_kwh(): + # 3.6e6 J == exactly 1 kWh. + c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, tot_tok=1000, + idle_mean_w=100.0, price_per_kwh=0.33) + assert c["available"] is True + assert c["gross_usd"] == 0.33 # 1 kWh * $0.33 + assert c["gross_usd_per_1k_tok"] == 0.33 # 0.33 / 1000 * 1000 + assert c["mean_gen_w_measured"] == 36000.0 # 3.6e6 J / 100 s + + +def test_marginal_subtracts_measured_warm_idle(): + # warm-idle 100 W over 100 s = 10_000 J of the 3.6e6 J is "keep hot". + c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, tot_tok=1000, + idle_mean_w=100.0, price_per_kwh=0.33) + assert c["warm_idle_joules"] == 10000.0 + assert c["marginal_joules"] == 3.6e6 - 10000.0 + # marginal < gross because warm-idle energy is removed. + assert c["marginal_usd_per_1k_tok"] < c["gross_usd_per_1k_tok"] + + +def test_marginal_clamps_to_zero_when_idle_exceeds_load(): + # A noisy idle window can read hotter than a short quiet load window. + c = energy_cogs(gpu_joules=5000.0, window_s=100.0, tot_tok=10, + idle_mean_w=100.0, price_per_kwh=0.33) + assert c["marginal_joules"] == 0.0 + assert c["marginal_usd"] == 0.0 + + +def test_no_idle_baseline_gives_gross_only(): + c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, tot_tok=1000, + idle_mean_w=None, price_per_kwh=0.33) + assert c["available"] is True + assert c["warm_idle_w_measured"] is None + assert "marginal_usd_per_1k_tok" not in c # can't decompose without it + + +def test_unavailable_when_inputs_missing(): + assert energy_cogs(None, 100.0, 1000, 100.0, 0.33)["available"] is False + assert energy_cogs(3.6e6, None, 1000, 100.0, 0.33)["available"] is False + assert energy_cogs(3.6e6, 100.0, 0, 100.0, 0.33)["available"] is False + + +def test_price_is_the_only_lever(): + # Same measured energy, double the rate -> double the dollar COGS. + a = energy_cogs(3.6e6, 100.0, 1000, 100.0, 0.33) + b = energy_cogs(3.6e6, 100.0, 1000, 100.0, 0.66) + # Tolerate the function's 6-decimal display rounding on the ratio. + assert round(b["gross_usd"] / a["gross_usd"], 3) == 2.0 + assert round(b["marginal_usd"] / a["marginal_usd"], 3) == 2.0