feat(#000057): energy-COGS layer for watt_bench — marginal vs gross $/1k-tok
fox 2026-05-21: compute cost-of-goods-sold by kWh vs tokens, with the
three power states (idle / warm-idle / generation) MEASURED per
card×model×server — never hardcoded (his 40/127/380 W were illustrative
of one 3090). The only operator input is --price-per-kwh (default 0.33
USD/kWh, a configurable site rate).
energy_cogs() (pure, unit-tested) decomposes measured generation energy
against the measured warm-idle baseline:
* gross — all measured joules over the window (all-in, includes the
warm-idle cost of keeping the model hot, amortized).
* marginal — joules ABOVE warm-idle: what one more request's burst
actually costs (clamped >=0).
kWh = J/3.6e6; $/1k-tok is the unit that compares to API pricing. Both
surface per cell + a COGS print line.
watt_bench's arborist arm now loads the frozen bench.stock_v1 policy
(--answer-mode, drift-guarded on non-reasoning) so cost is measured for
the SAME substrate the campaign grades. Cells record
window_start/end_unix so a post-hoc load_monitor queue-depth cross-ref
can flag organic-traffic contamination on the non-isolated single-slot
endpoints. 6 COGS tests; full suite 2534 passed.
This commit is contained in:
parent
1a7f8eb4ea
commit
1aff09f021
3 changed files with 163 additions and 10 deletions
|
|
@ -355,6 +355,61 @@ def _solo_call(client, mkey: str, question: str) -> tuple[str, int]:
|
|||
return ans, max(1, len(ans) // 4)
|
||||
|
||||
|
||||
def energy_cogs(gpu_joules: float | None, window_s: float | None,
|
||||
tot_tok: int, idle_mean_w: float | None,
|
||||
price_per_kwh: float) -> dict:
|
||||
"""Energy cost-of-goods-sold from MEASURED power — no hardcoded states.
|
||||
|
||||
fox 2026-05-21: idle / warm-idle / generation watts differ for every
|
||||
card × model × inference-server, so every power number here is
|
||||
MEASURED at runtime — ``gpu_joules`` / ``window_s`` / ``mean_gen_w``
|
||||
from the driven window, ``idle_mean_w`` from the no-request idle
|
||||
baseline. The ONLY operator input is ``price_per_kwh`` (a configurable
|
||||
site rate, default 0.33 USD/kWh). Nothing is baked in.
|
||||
|
||||
Decomposition (the three power states the card occupies):
|
||||
|
||||
* warm-idle — model resident, waiting (measured ``idle_mean_w``).
|
||||
* generation — measured mean power over the driven window.
|
||||
* gross COGS — all measured joules over the window, including the
|
||||
warm-idle cost of keeping the model hot; the all-in
|
||||
cost amortized across throughput.
|
||||
* marginal COGS — joules ABOVE the warm-idle baseline: what one more
|
||||
request's generation burst actually costs. Clamped
|
||||
≥0 (a noisy idle window can exceed a quiet load one).
|
||||
|
||||
kWh = J / 3.6e6; ``$/1k-tok`` = $ / tokens × 1000 — the unit that
|
||||
compares to hosted-API pricing. See docs/stock-v1-config.md.
|
||||
"""
|
||||
if not gpu_joules or not tot_tok or not window_s:
|
||||
return {"available": False, "price_per_kwh": price_per_kwh}
|
||||
J_PER_KWH = 3.6e6
|
||||
mean_gen_w = gpu_joules / window_s
|
||||
gross_usd = gpu_joules / J_PER_KWH * price_per_kwh
|
||||
out = {
|
||||
"available": True,
|
||||
"price_per_kwh": price_per_kwh,
|
||||
"warm_idle_w_measured": (round(idle_mean_w, 1)
|
||||
if idle_mean_w is not None else None),
|
||||
"mean_gen_w_measured": round(mean_gen_w, 1),
|
||||
"window_s": round(window_s, 1),
|
||||
"gross_joules": round(gpu_joules, 1),
|
||||
"gross_usd": round(gross_usd, 6),
|
||||
"gross_usd_per_1k_tok": round(gross_usd / tot_tok * 1000, 6),
|
||||
}
|
||||
if idle_mean_w is not None:
|
||||
idle_joules = idle_mean_w * window_s
|
||||
marginal_joules = max(0.0, gpu_joules - idle_joules)
|
||||
marginal_usd = marginal_joules / J_PER_KWH * price_per_kwh
|
||||
out.update({
|
||||
"warm_idle_joules": round(idle_joules, 1),
|
||||
"marginal_joules": round(marginal_joules, 1),
|
||||
"marginal_usd": round(marginal_usd, 6),
|
||||
"marginal_usd_per_1k_tok": round(marginal_usd / tot_tok * 1000, 6),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--fixture", default="bench/qa_questions_stale_map.json")
|
||||
|
|
@ -386,7 +441,17 @@ def main() -> int:
|
|||
"entry's endpoint.")
|
||||
ap.add_argument("--sample-interval", type=float, default=0.5)
|
||||
ap.add_argument("--idle-baseline-s", type=float, default=5.0,
|
||||
help="seconds to sample idle power before driving load")
|
||||
help="seconds to sample idle (warm-idle) power before "
|
||||
"driving load — MEASURED, never hardcoded")
|
||||
ap.add_argument("--price-per-kwh", type=float, default=0.33,
|
||||
help="electricity rate for energy COGS (USD/kWh; "
|
||||
"configurable site rate, fox default 0.33). The "
|
||||
"only non-measured input — power states are all "
|
||||
"measured per card/model/server at runtime.")
|
||||
ap.add_argument("--answer-mode", choices=["quote", "claim_lattice"],
|
||||
default="claim_lattice",
|
||||
help="STOCK V.1 substrate-ON answer shape for the "
|
||||
"arborist arm (frozen bench.stock_v1 policy).")
|
||||
ap.add_argument("--cpu-energy-cmd", default="",
|
||||
help="optional privileged reader for RAPL energy_uj "
|
||||
"when it's root-only (PLATYPUS mitigation), e.g. "
|
||||
|
|
@ -457,13 +522,21 @@ def main() -> int:
|
|||
if arm == "solo":
|
||||
ans, ctoks = _solo_call(client, mkey, q)
|
||||
else: # arborist
|
||||
from arborist.qa.query import (
|
||||
DEFAULT_QUERY_POLICY, query)
|
||||
from arborist.qa.query import query
|
||||
from bench.stock_v1 import (
|
||||
assert_not_drifted as _assert_stock)
|
||||
from bench.stock_v1 import policy_for as _stock_policy_for
|
||||
gold = _gold(shards_dir, it.get("shard", ""),
|
||||
it["target_root"])
|
||||
pol = dict(DEFAULT_QUERY_POLICY,
|
||||
answer_mode="claim_lattice")
|
||||
if MODELS[mkey].get("reasoning"):
|
||||
# STOCK V.1 frozen substrate-ON policy (cost is
|
||||
# measured for the SAME substrate the campaign
|
||||
# grades). Reasoning refs (phase 3) layer the
|
||||
# documented JSON overrides and skip the assert.
|
||||
reasoning = bool(MODELS[mkey].get("reasoning"))
|
||||
if not reasoning:
|
||||
_assert_stock(a.answer_mode)
|
||||
pol = _stock_policy_for(a.answer_mode)
|
||||
if reasoning:
|
||||
pol["claim_lattice_json_stop_sequences"] = []
|
||||
pol["max_tokens"] = 8192
|
||||
qa_db = Path("/tmp") / f"watt_{ts}_{mkey}.db"
|
||||
|
|
@ -510,6 +583,15 @@ def main() -> int:
|
|||
if gpu_j and tot_tok else None),
|
||||
"mean_latency_s": (round(sum(r["latency_s"] for r in rows) / nq, 2)
|
||||
if nq else None),
|
||||
# Energy COGS — measured power vs measured warm-idle baseline,
|
||||
# only price_per_kwh is an operator input. Window timestamps
|
||||
# let a post-hoc load_monitor cross-ref flag organic-traffic
|
||||
# contamination under non-isolation (single-slot endpoints).
|
||||
"energy_cogs": energy_cogs(
|
||||
gpu_j, st.get("window_s"), tot_tok,
|
||||
(idle or {}).get("mean_w"), a.price_per_kwh),
|
||||
"window_start_unix": round(t0, 3),
|
||||
"window_end_unix": round(t0 + elapsed, 3),
|
||||
}
|
||||
cells.append(cell)
|
||||
with open(outp, "a") as f:
|
||||
|
|
@ -522,6 +604,15 @@ def main() -> int:
|
|||
f"cpu={cell['cpu_joules_per_question']} "
|
||||
f"tot={cell['total_joules_per_question']} "
|
||||
f"lat={cell['mean_latency_s']}s")
|
||||
cg = cell["energy_cogs"]
|
||||
if cg.get("available"):
|
||||
print(f" COGS @${cg['price_per_kwh']}/kWh: "
|
||||
f"gross ${cg['gross_usd_per_1k_tok']}/1k-tok "
|
||||
f"(gen {cg['mean_gen_w_measured']}W)"
|
||||
+ (f" · marginal ${cg['marginal_usd_per_1k_tok']}/1k-tok "
|
||||
f"(vs warm-idle {cg['warm_idle_w_measured']}W)"
|
||||
if "marginal_usd_per_1k_tok" in cg else
|
||||
" · marginal n/a (no idle baseline)"))
|
||||
|
||||
for mkey in models:
|
||||
run_cell("solo", mkey)
|
||||
|
|
|
|||
|
|
@ -98,10 +98,13 @@ waiting), and generation (request burst):
|
|||
for `$/1k-tok` against API pricing.
|
||||
|
||||
**Every power state is MEASURED per (card, model, inference-server) at
|
||||
runtime — never hardcoded.** `P_warm_idle` comes from a no-request
|
||||
calibration window; `P_gen` from the burst during actual completions.
|
||||
Each card × model × server has its own profile. The only operator input
|
||||
is `$/kWh`, a configurable site flag.
|
||||
runtime — never hardcoded.** `P_warm_idle` comes from the no-request idle
|
||||
baseline (`--idle-baseline-s`, default 5 s); `P_gen` from the burst
|
||||
during actual completions. Each card × model × server has its own
|
||||
profile. The only operator input is `--price-per-kwh` (default 0.33
|
||||
USD/kWh, a configurable site rate). `bench/watt_bench.py:energy_cogs`
|
||||
emits gross + marginal `$/1k-tok` per cell and records the measurement
|
||||
window (`window_start_unix`/`window_end_unix`) for the cross-ref below.
|
||||
|
||||
Because the single-slot endpoints (hermes 3090 / vLLM, qwen 4090 /
|
||||
llama.cpp) serve **real internet traffic and cannot be isolated**,
|
||||
|
|
|
|||
59
tests/test_watt_cogs.py
Normal file
59
tests/test_watt_cogs.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
"""Energy-COGS math for bench/watt_bench.py:energy_cogs (#000057).
|
||||
|
||||
Pure function, no GPU / no nvidia-smi. fox 2026-05-21: power states are
|
||||
MEASURED per card/model/server; the only operator input is price_per_kwh.
|
||||
These tests pin the joules→kWh→$ arithmetic and the warm-idle marginal
|
||||
decomposition. See docs/calculator-test-patterns.md.
|
||||
"""
|
||||
from bench.watt_bench import energy_cogs
|
||||
|
||||
|
||||
def test_gross_cogs_one_kwh():
|
||||
# 3.6e6 J == exactly 1 kWh.
|
||||
c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, tot_tok=1000,
|
||||
idle_mean_w=100.0, price_per_kwh=0.33)
|
||||
assert c["available"] is True
|
||||
assert c["gross_usd"] == 0.33 # 1 kWh * $0.33
|
||||
assert c["gross_usd_per_1k_tok"] == 0.33 # 0.33 / 1000 * 1000
|
||||
assert c["mean_gen_w_measured"] == 36000.0 # 3.6e6 J / 100 s
|
||||
|
||||
|
||||
def test_marginal_subtracts_measured_warm_idle():
|
||||
# warm-idle 100 W over 100 s = 10_000 J of the 3.6e6 J is "keep hot".
|
||||
c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, tot_tok=1000,
|
||||
idle_mean_w=100.0, price_per_kwh=0.33)
|
||||
assert c["warm_idle_joules"] == 10000.0
|
||||
assert c["marginal_joules"] == 3.6e6 - 10000.0
|
||||
# marginal < gross because warm-idle energy is removed.
|
||||
assert c["marginal_usd_per_1k_tok"] < c["gross_usd_per_1k_tok"]
|
||||
|
||||
|
||||
def test_marginal_clamps_to_zero_when_idle_exceeds_load():
|
||||
# A noisy idle window can read hotter than a short quiet load window.
|
||||
c = energy_cogs(gpu_joules=5000.0, window_s=100.0, tot_tok=10,
|
||||
idle_mean_w=100.0, price_per_kwh=0.33)
|
||||
assert c["marginal_joules"] == 0.0
|
||||
assert c["marginal_usd"] == 0.0
|
||||
|
||||
|
||||
def test_no_idle_baseline_gives_gross_only():
|
||||
c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, tot_tok=1000,
|
||||
idle_mean_w=None, price_per_kwh=0.33)
|
||||
assert c["available"] is True
|
||||
assert c["warm_idle_w_measured"] is None
|
||||
assert "marginal_usd_per_1k_tok" not in c # can't decompose without it
|
||||
|
||||
|
||||
def test_unavailable_when_inputs_missing():
|
||||
assert energy_cogs(None, 100.0, 1000, 100.0, 0.33)["available"] is False
|
||||
assert energy_cogs(3.6e6, None, 1000, 100.0, 0.33)["available"] is False
|
||||
assert energy_cogs(3.6e6, 100.0, 0, 100.0, 0.33)["available"] is False
|
||||
|
||||
|
||||
def test_price_is_the_only_lever():
|
||||
# Same measured energy, double the rate -> double the dollar COGS.
|
||||
a = energy_cogs(3.6e6, 100.0, 1000, 100.0, 0.33)
|
||||
b = energy_cogs(3.6e6, 100.0, 1000, 100.0, 0.66)
|
||||
# Tolerate the function's 6-decimal display rounding on the ratio.
|
||||
assert round(b["gross_usd"] / a["gross_usd"], 3) == 2.0
|
||||
assert round(b["marginal_usd"] / a["marginal_usd"], 3) == 2.0
|
||||
Loading…
Add table
Add a link
Reference in a new issue