feat(#000057): energy-COGS layer for watt_bench — marginal vs gross $/1k-tok

fox 2026-05-21: compute cost-of-goods-sold by kWh vs tokens, with the
three power states (idle / warm-idle / generation) MEASURED per
card×model×server — never hardcoded (his 40/127/380 W were illustrative
of one 3090). The only operator input is --price-per-kwh (default 0.33
USD/kWh, a configurable site rate).

energy_cogs() (pure, unit-tested) decomposes measured generation energy
against the measured warm-idle baseline:
  * gross    — all measured joules over the window (all-in, includes the
               warm-idle cost of keeping the model hot, amortized).
  * marginal — joules ABOVE warm-idle: what one more request's burst
               actually costs (clamped >=0).
kWh = J/3.6e6; $/1k-tok is the unit that compares to API pricing. Both
surface per cell + a COGS print line.

watt_bench's arborist arm now loads the frozen bench.stock_v1 policy
(--answer-mode, drift-guarded on non-reasoning) so cost is measured for
the SAME substrate the campaign grades. Cells record
window_start/end_unix so a post-hoc load_monitor queue-depth cross-ref
can flag organic-traffic contamination on the non-isolated single-slot
endpoints. 6 COGS tests; full suite 2534 passed.
This commit is contained in:
russell@unturf.com 2026-05-21 10:19:41 -04:00
parent 1a7f8eb4ea
commit 1aff09f021
No known key found for this signature in database
3 changed files with 163 additions and 10 deletions

View file

@ -355,6 +355,61 @@ def _solo_call(client, mkey: str, question: str) -> tuple[str, int]:
return ans, max(1, len(ans) // 4)
def energy_cogs(gpu_joules: float | None, window_s: float | None,
tot_tok: int, idle_mean_w: float | None,
price_per_kwh: float) -> dict:
"""Energy cost-of-goods-sold from MEASURED power — no hardcoded states.
fox 2026-05-21: idle / warm-idle / generation watts differ for every
card × model × inference-server, so every power number here is
MEASURED at runtime — ``gpu_joules`` / ``window_s`` / ``mean_gen_w``
from the driven window, ``idle_mean_w`` from the no-request idle
baseline. The ONLY operator input is ``price_per_kwh`` (a configurable
site rate, default 0.33 USD/kWh). Nothing is baked in.
Decomposition (the three power states the card occupies):
* warm-idle — model resident, waiting (measured ``idle_mean_w``).
* generation — measured mean power over the driven window.
* gross COGS — all measured joules over the window, including the
warm-idle cost of keeping the model hot; the all-in
cost amortized across throughput.
* marginal COGS — joules ABOVE the warm-idle baseline: what one more
request's generation burst actually costs. Clamped
≥0 (a noisy idle window can exceed a quiet load one).
kWh = J / 3.6e6; ``$/1k-tok`` = $ / tokens × 1000 — the unit that
compares to hosted-API pricing. See docs/stock-v1-config.md.
"""
if not gpu_joules or not tot_tok or not window_s:
return {"available": False, "price_per_kwh": price_per_kwh}
J_PER_KWH = 3.6e6
mean_gen_w = gpu_joules / window_s
gross_usd = gpu_joules / J_PER_KWH * price_per_kwh
out = {
"available": True,
"price_per_kwh": price_per_kwh,
"warm_idle_w_measured": (round(idle_mean_w, 1)
if idle_mean_w is not None else None),
"mean_gen_w_measured": round(mean_gen_w, 1),
"window_s": round(window_s, 1),
"gross_joules": round(gpu_joules, 1),
"gross_usd": round(gross_usd, 6),
"gross_usd_per_1k_tok": round(gross_usd / tot_tok * 1000, 6),
}
if idle_mean_w is not None:
idle_joules = idle_mean_w * window_s
marginal_joules = max(0.0, gpu_joules - idle_joules)
marginal_usd = marginal_joules / J_PER_KWH * price_per_kwh
out.update({
"warm_idle_joules": round(idle_joules, 1),
"marginal_joules": round(marginal_joules, 1),
"marginal_usd": round(marginal_usd, 6),
"marginal_usd_per_1k_tok": round(marginal_usd / tot_tok * 1000, 6),
})
return out
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--fixture", default="bench/qa_questions_stale_map.json")
@ -386,7 +441,17 @@ def main() -> int:
"entry's endpoint.")
ap.add_argument("--sample-interval", type=float, default=0.5)
ap.add_argument("--idle-baseline-s", type=float, default=5.0,
help="seconds to sample idle power before driving load")
help="seconds to sample idle (warm-idle) power before "
"driving load — MEASURED, never hardcoded")
ap.add_argument("--price-per-kwh", type=float, default=0.33,
help="electricity rate for energy COGS (USD/kWh; "
"configurable site rate, fox default 0.33). The "
"only non-measured input — power states are all "
"measured per card/model/server at runtime.")
ap.add_argument("--answer-mode", choices=["quote", "claim_lattice"],
default="claim_lattice",
help="STOCK V.1 substrate-ON answer shape for the "
"arborist arm (frozen bench.stock_v1 policy).")
ap.add_argument("--cpu-energy-cmd", default="",
help="optional privileged reader for RAPL energy_uj "
"when it's root-only (PLATYPUS mitigation), e.g. "
@ -457,13 +522,21 @@ def main() -> int:
if arm == "solo":
ans, ctoks = _solo_call(client, mkey, q)
else: # arborist
from arborist.qa.query import (
DEFAULT_QUERY_POLICY, query)
from arborist.qa.query import query
from bench.stock_v1 import (
assert_not_drifted as _assert_stock)
from bench.stock_v1 import policy_for as _stock_policy_for
gold = _gold(shards_dir, it.get("shard", ""),
it["target_root"])
pol = dict(DEFAULT_QUERY_POLICY,
answer_mode="claim_lattice")
if MODELS[mkey].get("reasoning"):
# STOCK V.1 frozen substrate-ON policy (cost is
# measured for the SAME substrate the campaign
# grades). Reasoning refs (phase 3) layer the
# documented JSON overrides and skip the assert.
reasoning = bool(MODELS[mkey].get("reasoning"))
if not reasoning:
_assert_stock(a.answer_mode)
pol = _stock_policy_for(a.answer_mode)
if reasoning:
pol["claim_lattice_json_stop_sequences"] = []
pol["max_tokens"] = 8192
qa_db = Path("/tmp") / f"watt_{ts}_{mkey}.db"
@ -510,6 +583,15 @@ def main() -> int:
if gpu_j and tot_tok else None),
"mean_latency_s": (round(sum(r["latency_s"] for r in rows) / nq, 2)
if nq else None),
# Energy COGS — measured power vs measured warm-idle baseline,
# only price_per_kwh is an operator input. Window timestamps
# let a post-hoc load_monitor cross-ref flag organic-traffic
# contamination under non-isolation (single-slot endpoints).
"energy_cogs": energy_cogs(
gpu_j, st.get("window_s"), tot_tok,
(idle or {}).get("mean_w"), a.price_per_kwh),
"window_start_unix": round(t0, 3),
"window_end_unix": round(t0 + elapsed, 3),
}
cells.append(cell)
with open(outp, "a") as f:
@ -522,6 +604,15 @@ def main() -> int:
f"cpu={cell['cpu_joules_per_question']} "
f"tot={cell['total_joules_per_question']} "
f"lat={cell['mean_latency_s']}s")
cg = cell["energy_cogs"]
if cg.get("available"):
print(f" COGS @${cg['price_per_kwh']}/kWh: "
f"gross ${cg['gross_usd_per_1k_tok']}/1k-tok "
f"(gen {cg['mean_gen_w_measured']}W)"
+ (f" · marginal ${cg['marginal_usd_per_1k_tok']}/1k-tok "
f"(vs warm-idle {cg['warm_idle_w_measured']}W)"
if "marginal_usd_per_1k_tok" in cg else
" · marginal n/a (no idle baseline)"))
for mkey in models:
run_cell("solo", mkey)

View file

@ -98,10 +98,13 @@ waiting), and generation (request burst):
for `$/1k-tok` against API pricing.
**Every power state is MEASURED per (card, model, inference-server) at
runtime — never hardcoded.** `P_warm_idle` comes from a no-request
calibration window; `P_gen` from the burst during actual completions.
Each card × model × server has its own profile. The only operator input
is `$/kWh`, a configurable site flag.
runtime — never hardcoded.** `P_warm_idle` comes from the no-request idle
baseline (`--idle-baseline-s`, default 5 s); `P_gen` from the burst
during actual completions. Each card × model × server has its own
profile. The only operator input is `--price-per-kwh` (default 0.33
USD/kWh, a configurable site rate). `bench/watt_bench.py:energy_cogs`
emits gross + marginal `$/1k-tok` per cell and records the measurement
window (`window_start_unix`/`window_end_unix`) for the cross-ref below.
Because the single-slot endpoints (hermes 3090 / vLLM, qwen 4090 /
llama.cpp) serve **real internet traffic and cannot be isolated**,

59
tests/test_watt_cogs.py Normal file
View file

@ -0,0 +1,59 @@
"""Energy-COGS math for bench/watt_bench.py:energy_cogs (#000057).
Pure function, no GPU / no nvidia-smi. fox 2026-05-21: power states are
MEASURED per card/model/server; the only operator input is price_per_kwh.
These tests pin the joules→kWh→$ arithmetic and the warm-idle marginal
decomposition. See docs/calculator-test-patterns.md.
"""
from bench.watt_bench import energy_cogs
def test_gross_cogs_one_kwh():
# 3.6e6 J == exactly 1 kWh.
c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, tot_tok=1000,
idle_mean_w=100.0, price_per_kwh=0.33)
assert c["available"] is True
assert c["gross_usd"] == 0.33 # 1 kWh * $0.33
assert c["gross_usd_per_1k_tok"] == 0.33 # 0.33 / 1000 * 1000
assert c["mean_gen_w_measured"] == 36000.0 # 3.6e6 J / 100 s
def test_marginal_subtracts_measured_warm_idle():
# warm-idle 100 W over 100 s = 10_000 J of the 3.6e6 J is "keep hot".
c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, tot_tok=1000,
idle_mean_w=100.0, price_per_kwh=0.33)
assert c["warm_idle_joules"] == 10000.0
assert c["marginal_joules"] == 3.6e6 - 10000.0
# marginal < gross because warm-idle energy is removed.
assert c["marginal_usd_per_1k_tok"] < c["gross_usd_per_1k_tok"]
def test_marginal_clamps_to_zero_when_idle_exceeds_load():
# A noisy idle window can read hotter than a short quiet load window.
c = energy_cogs(gpu_joules=5000.0, window_s=100.0, tot_tok=10,
idle_mean_w=100.0, price_per_kwh=0.33)
assert c["marginal_joules"] == 0.0
assert c["marginal_usd"] == 0.0
def test_no_idle_baseline_gives_gross_only():
c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, tot_tok=1000,
idle_mean_w=None, price_per_kwh=0.33)
assert c["available"] is True
assert c["warm_idle_w_measured"] is None
assert "marginal_usd_per_1k_tok" not in c # can't decompose without it
def test_unavailable_when_inputs_missing():
assert energy_cogs(None, 100.0, 1000, 100.0, 0.33)["available"] is False
assert energy_cogs(3.6e6, None, 1000, 100.0, 0.33)["available"] is False
assert energy_cogs(3.6e6, 100.0, 0, 100.0, 0.33)["available"] is False
def test_price_is_the_only_lever():
# Same measured energy, double the rate -> double the dollar COGS.
a = energy_cogs(3.6e6, 100.0, 1000, 100.0, 0.33)
b = energy_cogs(3.6e6, 100.0, 1000, 100.0, 0.66)
# Tolerate the function's 6-decimal display rounding on the ratio.
assert round(b["gross_usd"] / a["gross_usd"], 3) == 2.0
assert round(b["marginal_usd"] / a["marginal_usd"], 3) == 2.0