engineering/arborist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(#000057): real token usage + cost per input/output separately

Browse source 
fox 2026-05-21: (1) use REAL API token usage, not len//4; (2) the
substrate prefills a large retrieved CONTEXT as INPUT while solo feeds
~nothing, so per-completion-token over-charges the substrate — and per-
TOTAL-token UNDER-charges it (its mix is ~98% cheap prefill tokens).
Measured n=30 qwen-nothink/4090: substrate prefills ~6.6k input tok/query
(claim_lattice) vs solo ~52 — ~127x. Neither single per-token denominator
is honest; prefill (parallel, cheap/tok) and decode (autoregressive,
dear/tok) must be costed separately.

- OpenAICompatibleClient stashes data['usage'] as .last_usage (non-
  invasive; return type unchanged).
- watt_bench captures real prompt_tokens + completion_tokens per call
  (both arms), aggregates per cell, and energy_cogs reports gross +
  marginal per BOTH 1k-total-tok and 1k-completion-tok plus the context
  size. Prints the prompt/completion split.
- 12 tests incl. the prompt-context artifact (per-total cheap, per-
  completion dear). Full suite 2540 passed.

The clean per-input-tok / per-output-tok split rides bench/watt_calibrate
(slope calibration; separate commit once validated live).
This commit is contained in:
russell@unturf.com 2026-05-21 11:41:01 -04:00
parent 32aeb37086
commit 459060c774
No known key found for this signature in database
3 changed files with 131 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

37
tests/test_watt_cogs.py
View file

@ -14,17 +14,31 @@ from bench.watt_probe import classify_power_bands
# ---- energy_cogs -------------------------------------------------------
def test_gross_cogs_one_kwh():
c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, tot_tok=1000,
c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, completion_tok=1000,
idle_mean_w=30.0, price_per_kwh=0.33)
assert c["available"] is True
assert c["gross_usd"] == 0.33 # 1 kWh * $0.33
assert c["gross_usd_per_1k_tok"] == 0.33
assert c["gross_usd_per_1k_total_tok"] == 0.33
assert c["gross_usd_per_1k_completion_tok"] == 0.33
assert c["window_mean_w"] == 36000.0 # blend, honestly labelled
def test_prompt_context_makes_per_total_cheap_per_completion_dear():
# The fox correction: substrate prefills a big CONTEXT (prompt_tok).
# Per TOTAL token processed it is cheap; charging it all to the few
# completion tokens makes it look expensive — that's the artifact.
c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, completion_tok=1000,
idle_mean_w=30.0, price_per_kwh=0.33, prompt_tok=9000)
assert c["prompt_tokens"] == 9000
assert c["total_tokens"] == 10000
assert c["gross_usd_per_1k_total_tok"] == 0.033 # 0.33 / 10000 *1k
assert c["gross_usd_per_1k_completion_tok"] == 0.33 # 10x dearer
assert (c["gross_usd_per_1k_completion_tok"]
> c["gross_usd_per_1k_total_tok"])
def test_marginal_uses_serving_floor_when_present():
# Serving floor (middle-idle 60W) is the right marginal floor, not idle.
c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, tot_tok=1000,
c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, completion_tok=1000,
idle_mean_w=30.0, price_per_kwh=0.33,
serving_floor_w=60.0, gen_draw_w=400.0, gen_duty=0.5)
assert c["marginal_floor"] == "serving"
@ -35,15 +49,16 @@ def test_marginal_uses_serving_floor_when_present():
def test_marginal_falls_back_to_idle_without_serving_floor():
c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, tot_tok=1000,
c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, completion_tok=1000,
idle_mean_w=30.0, price_per_kwh=0.33)
assert c["marginal_floor"] == "idle"
assert c["marginal_joules"] == 3.6e6 - 30.0 * 100.0
assert c["marginal_usd_per_1k_tok"] < c["gross_usd_per_1k_tok"]
assert (c["marginal_usd_per_1k_total_tok"]
< c["gross_usd_per_1k_total_tok"])
def test_marginal_clamps_to_zero_when_floor_exceeds_load():
c = energy_cogs(gpu_joules=5000.0, window_s=100.0, tot_tok=10,
c = energy_cogs(gpu_joules=5000.0, window_s=100.0, completion_tok=10,
idle_mean_w=30.0, price_per_kwh=0.33,
serving_floor_w=100.0)
assert c["marginal_joules"] == 0.0
@ -51,16 +66,18 @@ def test_marginal_clamps_to_zero_when_floor_exceeds_load():
def test_no_floor_gives_gross_only():
c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, tot_tok=1000,
c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, completion_tok=1000,
idle_mean_w=None, price_per_kwh=0.33)
assert c["available"] is True
assert "marginal_usd_per_1k_tok" not in c # no floor -> can't decompose
assert "marginal_usd_per_1k_total_tok" not in c
def test_unavailable_when_inputs_missing():
assert energy_cogs(None, 100.0, 1000, 30.0, 0.33)["available"] is False
assert energy_cogs(3.6e6, None, 1000, 30.0, 0.33)["available"] is False
assert energy_cogs(3.6e6, 100.0, 0, 30.0, 0.33)["available"] is False
# zero completion AND zero prompt -> no tokens at all
assert energy_cogs(3.6e6, 100.0, 0, 30.0, 0.33,
prompt_tok=0)["available"] is False
def test_price_is_the_only_lever():