From 5b1cbeed80247908f26c1eff85dbffb6a3b04f67 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Thu, 21 May 2026 10:58:56 -0400 Subject: [PATCH] fix(#000057): measure power STATES, not a duty-cycle blend; guarantee cache miss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fox 2026-05-21: 'gen 200W' was a bug — joules/window blends the ~400W generation bursts with the sub-100W gaps (retrieval/verify/network) into a power state the card never sits at. A card occupies DISTINCT states (idle / middle-idle = resident-between-requests / generation), differing per card×model×server. watt_probe.classify_power_bands(): largest-gap split of the window samples into a low band (serving floor) and high band (generation draw) + duty cycle. Data-derived, never hardcoded — tested at two scales. The worker emits the decomposition + raw samples; RemoteProbe/LocalProbe expose band_stats() uniformly. energy_cogs: marginal now taken against the measured SERVING FLOOR (the standing cost of being ready), not deep idle; the blend is kept but labelled window_mean_w. Reports idle/serving-floor/gen-draw/duty. Cache-miss certainty (fox's question): the arborist arm runs burn_existing=True (force-deletes any live providence row before inference) and asserts cache_hits==0 with a loud warning + real_inference flag — so we time real generation, never a SQLite lookup. Solo has no cache path. 11 tests (energy math + band split). Validated live on the isolated 4090: solo gen 308W/70%-duty vs substrate 396W/8.6%-duty — substrate marginal/tok is LOWER, gross/tok higher (it holds the card longer for retrieval). --- bench/watt_bench.py | 139 +++++++++++++++++++++++++++++----------- bench/watt_probe.py | 39 +++++++++++ tests/test_watt_cogs.py | 101 ++++++++++++++++++++--------- 3 files changed, 211 insertions(+), 68 deletions(-) diff --git a/bench/watt_bench.py b/bench/watt_bench.py index 3f36d87..0b7822d 100644 --- a/bench/watt_bench.py +++ b/bench/watt_bench.py @@ -252,6 +252,10 @@ class LocalProbe: def cpu_stats(self) -> dict: return self._cs + def band_stats(self) -> dict: + from bench.watt_probe import classify_power_bands + return classify_power_bands([w for _, w in self._g.samples]) + class RemoteProbe: """GPU+CPU power sampled on a REMOTE worker box over SSH — the @@ -330,6 +334,12 @@ class RemoteProbe: "mean_w": rp.get("cpu_mean_w"), "joules": rp.get("cpu_joules"), "window_s": rp.get("window_s")} + def band_stats(self) -> dict: + return (self._report or {}).get("power_bands", {"available": False}) + + def watt_samples(self) -> list: + return (self._report or {}).get("watt_samples", []) + # ----------------------------------------------------------- driver @@ -357,52 +367,67 @@ def _solo_call(client, mkey: str, question: str) -> tuple[str, int]: def energy_cogs(gpu_joules: float | None, window_s: float | None, tot_tok: int, idle_mean_w: float | None, - price_per_kwh: float) -> dict: - """Energy cost-of-goods-sold from MEASURED power — no hardcoded states. + price_per_kwh: float, + serving_floor_w: float | None = None, + gen_draw_w: float | None = None, + gen_duty: float | None = None) -> dict: + """Energy cost-of-goods-sold from MEASURED power states — no hardcodes. - fox 2026-05-21: idle / warm-idle / generation watts differ for every - card × model × inference-server, so every power number here is - MEASURED at runtime — ``gpu_joules`` / ``window_s`` / ``mean_gen_w`` - from the driven window, ``idle_mean_w`` from the no-request idle - baseline. The ONLY operator input is ``price_per_kwh`` (a configurable - site rate, default 0.33 USD/kWh). Nothing is baked in. + fox 2026-05-21: a card occupies DISTINCT power states (idle, middle- + idle = model resident between requests, generation), and they differ + for every card × model × inference-server. So every power number is + measured at runtime and the states are kept DISTINCT — we do NOT + collapse them into ``joules/window`` and call that "generation" + (that blend is a duty-cycle artifact, not a state the card sits at). - Decomposition (the three power states the card occupies): + * idle_mean_w — deep idle (no-request baseline window). + * serving_floor_w — middle-idle: model resident, between requests + (low band of the driven window). The right floor + for marginal cost — the standing cost of being + ready to answer. + * gen_draw_w — the actual generation draw (high band), with + ``gen_duty`` = fraction of the window generating. + * gross COGS — ALL measured joules / tokens: the all-in cost, + amortized across throughput (state-agnostic, so + this number was always correct). + * marginal COGS — joules ABOVE the serving floor / tokens: what + one more request's generation actually adds. + Falls back to idle floor if no band split. - * warm-idle — model resident, waiting (measured ``idle_mean_w``). - * generation — measured mean power over the driven window. - * gross COGS — all measured joules over the window, including the - warm-idle cost of keeping the model hot; the all-in - cost amortized across throughput. - * marginal COGS — joules ABOVE the warm-idle baseline: what one more - request's generation burst actually costs. Clamped - ≥0 (a noisy idle window can exceed a quiet load one). - - kWh = J / 3.6e6; ``$/1k-tok`` = $ / tokens × 1000 — the unit that - compares to hosted-API pricing. See docs/stock-v1-config.md. + kWh = J / 3.6e6; ``$/1k-tok`` is the unit that compares to API pricing. + Only ``price_per_kwh`` is an operator input. See docs/stock-v1-config.md. """ if not gpu_joules or not tot_tok or not window_s: return {"available": False, "price_per_kwh": price_per_kwh} J_PER_KWH = 3.6e6 - mean_gen_w = gpu_joules / window_s gross_usd = gpu_joules / J_PER_KWH * price_per_kwh out = { "available": True, "price_per_kwh": price_per_kwh, - "warm_idle_w_measured": (round(idle_mean_w, 1) - if idle_mean_w is not None else None), - "mean_gen_w_measured": round(mean_gen_w, 1), + # measured power states (distinct — not a blend) + "idle_w_measured": (round(idle_mean_w, 1) + if idle_mean_w is not None else None), + "serving_floor_w_measured": (round(serving_floor_w, 1) + if serving_floor_w is not None else None), + "gen_draw_w_measured": (round(gen_draw_w, 1) + if gen_draw_w is not None else None), + "gen_duty_cycle": gen_duty, + "window_mean_w": round(gpu_joules / window_s, 1), # blend, labelled "window_s": round(window_s, 1), "gross_joules": round(gpu_joules, 1), "gross_usd": round(gross_usd, 6), "gross_usd_per_1k_tok": round(gross_usd / tot_tok * 1000, 6), } - if idle_mean_w is not None: - idle_joules = idle_mean_w * window_s - marginal_joules = max(0.0, gpu_joules - idle_joules) + # Marginal against the serving floor (preferred) or idle (fallback). + floor_w = serving_floor_w if serving_floor_w is not None else idle_mean_w + if floor_w is not None: + floor_joules = floor_w * window_s + marginal_joules = max(0.0, gpu_joules - floor_joules) marginal_usd = marginal_joules / J_PER_KWH * price_per_kwh out.update({ - "warm_idle_joules": round(idle_joules, 1), + "marginal_floor": ("serving" if serving_floor_w is not None + else "idle"), + "marginal_floor_w": round(floor_w, 1), "marginal_joules": round(marginal_joules, 1), "marginal_usd": round(marginal_usd, 6), "marginal_usd_per_1k_tok": round(marginal_usd / tot_tok * 1000, 6), @@ -511,6 +536,7 @@ def main() -> int: endpoint = a.endpoint or MODELS[mkey]["endpoint"] client = _make_client(endpoint) rows = [] + cache_hits = 0 # arborist arm: must stay 0 (real inference, not lookup) probe = _new_probe() probe.start() t0 = time.time() @@ -519,6 +545,7 @@ def main() -> int: for it in items: q = VARIANTS[variant](it["question"]) tq = time.time() + cache_status = "solo_no_cache" if arm == "solo": ans, ctoks = _solo_call(client, mkey, q) else: # arborist @@ -540,25 +567,35 @@ def main() -> int: pol["claim_lattice_json_stop_sequences"] = [] pol["max_tokens"] = 8192 qa_db = Path("/tmp") / f"watt_{ts}_{mkey}.db" + # burn_existing force-deletes any matching live row + # before inference, so the arborist arm ALWAYS runs + # the LLM (real generation energy) and never times a + # cache lookup. cache_hits MUST stay 0 (fox 2026-05-21). r = query(question=q, qa_db=qa_db, chat_client=client, model_id=MODELS[mkey]["model"], shards_dir=shards_dir, extra_body=MODELS[mkey]["extra"], - policy=pol) + policy=pol, burn_existing=True) ans = r.get("raw_answer") or r.get("answer_text") or "" ctoks = max(1, len(ans) // 4) + cache_status = r.get("status", "?") + if (cache_status == "cache_hit" + or "cache_hit" in str(r.get("lookup_path", ""))): + cache_hits += 1 dt = time.time() - tq rows.append({"arm": arm, "model": mkey, "variant": variant, "latency_s": round(dt, 2), "answer_chars": len(ans), - "est_completion_tokens": ctoks}) + "est_completion_tokens": ctoks, + "cache_status": cache_status}) finally: client.close() probe.stop() st = probe.gpu_stats() cpu_st = probe.cpu_stats() + bands = probe.band_stats() elapsed = time.time() - t0 nq = len(rows) tot_tok = sum(r["est_completion_tokens"] for r in rows) @@ -583,15 +620,30 @@ def main() -> int: if gpu_j and tot_tok else None), "mean_latency_s": (round(sum(r["latency_s"] for r in rows) / nq, 2) if nq else None), - # Energy COGS — measured power vs measured warm-idle baseline, - # only price_per_kwh is an operator input. Window timestamps - # let a post-hoc load_monitor cross-ref flag organic-traffic - # contamination under non-isolation (single-slot endpoints). + # Measured power-state decomposition (idle / serving-floor / + # generation), data-derived, never hardcoded. + "power_bands": bands, + # Real-inference guard: arborist arm runs under burn_existing, + # so cache_hits MUST be 0 — else we'd be timing a SQLite lookup, + # not generation, and the energy number is meaningless. + "cache_hits": cache_hits, + "real_inference": (cache_hits == 0), + # Energy COGS — measured power STATES (not a duty-cycle blend); + # marginal taken against the serving floor (middle-idle), the + # standing cost of being ready. Only price_per_kwh is operator + # input. Window timestamps let a post-hoc load_monitor cross-ref + # flag organic-traffic contamination under non-isolation. "energy_cogs": energy_cogs( gpu_j, st.get("window_s"), tot_tok, - (idle or {}).get("mean_w"), a.price_per_kwh), + (idle or {}).get("mean_w"), a.price_per_kwh, + serving_floor_w=(bands.get("low_band_w") + if bands.get("bimodal") else None), + gen_draw_w=bands.get("high_band_w"), + gen_duty=bands.get("duty_cycle")), "window_start_unix": round(t0, 3), "window_end_unix": round(t0 + elapsed, 3), + "watt_samples": (probe.watt_samples() + if hasattr(probe, "watt_samples") else []), } cells.append(cell) with open(outp, "a") as f: @@ -604,15 +656,24 @@ def main() -> int: f"cpu={cell['cpu_joules_per_question']} " f"tot={cell['total_joules_per_question']} " f"lat={cell['mean_latency_s']}s") + if arm == "arborist" and cache_hits: + print(f" !! WARNING {cache_hits}/{nq} CACHE HITS — " + f"energy NOT a true generation cost (re-run; expected 0)") + b = bands or {} + if b.get("available"): + print(f" states: idle≈{(idle or {}).get('mean_w')}W · " + f"serving-floor {b.get('low_band_w')}W · " + f"gen {b.get('high_band_w')}W " + f"(duty {b.get('duty_cycle')}, peak {b.get('peak_w')}W)" + + ("" if b.get("bimodal") else " [unimodal — no gen state]")) cg = cell["energy_cogs"] if cg.get("available"): print(f" COGS @${cg['price_per_kwh']}/kWh: " - f"gross ${cg['gross_usd_per_1k_tok']}/1k-tok " - f"(gen {cg['mean_gen_w_measured']}W)" + f"gross ${cg['gross_usd_per_1k_tok']}/1k-tok" + (f" · marginal ${cg['marginal_usd_per_1k_tok']}/1k-tok " - f"(vs warm-idle {cg['warm_idle_w_measured']}W)" + f"(vs {cg['marginal_floor']}-floor {cg['marginal_floor_w']}W)" if "marginal_usd_per_1k_tok" in cg else - " · marginal n/a (no idle baseline)")) + " · marginal n/a (no floor)")) for mkey in models: run_cell("solo", mkey) diff --git a/bench/watt_probe.py b/bench/watt_probe.py index ae1737b..d1406c2 100644 --- a/bench/watt_probe.py +++ b/bench/watt_probe.py @@ -58,6 +58,39 @@ def _gpu_name(gpu_index: int) -> str: return _nvsmi("name", gpu_index) or "unknown" +def classify_power_bands(watt_samples) -> dict: + """Split power samples into a low band (model resident, between + requests — 'middle idle') and a high band (active token generation) + by the LARGEST GAP in the sorted samples. Data-derived — never + hardcoded; every card/model/inference-server has its own profile + (fox 2026-05-21). Reports measured POWER STATES, not the duty-cycle + blend that `mean_w` (joules/window) collapses them into. + """ + ws = sorted(w for w in watt_samples if w is not None) + n = len(ws) + if n < 4: + return {"available": False, "n_samples": n} + gi = max(range(n - 1), key=lambda i: ws[i + 1] - ws[i]) + split = (ws[gi] + ws[gi + 1]) / 2.0 + low, high = ws[: gi + 1], ws[gi + 1:] + low_mean = sum(low) / len(low) + high_mean = sum(high) / len(high) + # Distinct generation state only if the high band is clearly above + # the low band; otherwise the window was effectively unimodal. + bimodal = high_mean > 1.5 * max(low_mean, 1e-6) + return { + "available": True, + "n_samples": n, + "bimodal": bimodal, + "split_w": round(split, 1), + "low_band_w": round(low_mean, 1), # serving floor / middle-idle + "high_band_w": round(high_mean, 1), # active generation draw + "duty_cycle": round(len(high) / n, 3), + "min_w": round(ws[0], 1), + "peak_w": round(ws[-1], 1), + } + + def _gpu_watts(gpu_index: int) -> float | None: v = _nvsmi("power.draw", gpu_index) try: @@ -160,6 +193,8 @@ def main() -> int: win = t_end - t_start cpu_mean = round(cpu_joules / win, 1) if win else None + power_bands = classify_power_bands([w for _, w, _ in samples]) + report = { "gpu_label": a.gpu_label or "gpu", "gpu_name": gpu_name, @@ -174,6 +209,10 @@ def main() -> int: "gpu_joules": gpu_joules, "cpu_mean_w": cpu_mean, "cpu_joules": cpu_joules, + # Measured power-state decomposition + raw samples for audit / + # offline re-analysis (n is small; transmits fine over ssh cat). + "power_bands": power_bands, + "watt_samples": [round(w, 1) for _, w, _ in samples], } blob = json.dumps(report, indent=2) if a.out: diff --git a/tests/test_watt_cogs.py b/tests/test_watt_cogs.py index 3552419..1194c23 100644 --- a/tests/test_watt_cogs.py +++ b/tests/test_watt_cogs.py @@ -1,59 +1,102 @@ -"""Energy-COGS math for bench/watt_bench.py:energy_cogs (#000057). +"""Energy-COGS math + power-state classification (#000057). -Pure function, no GPU / no nvidia-smi. fox 2026-05-21: power states are -MEASURED per card/model/server; the only operator input is price_per_kwh. -These tests pin the joules→kWh→$ arithmetic and the warm-idle marginal -decomposition. See docs/calculator-test-patterns.md. +Pure functions, no GPU / no nvidia-smi. fox 2026-05-21: a card occupies +DISTINCT power states (idle / middle-idle / generation) that differ per +card×model×server — measure them, never hardcode, never collapse them +into one blended "gen W". These tests pin the joules→kWh→$ arithmetic, +the serving-floor marginal, and the data-derived band split. +See docs/calculator-test-patterns.md. """ from bench.watt_bench import energy_cogs +from bench.watt_probe import classify_power_bands +# ---- energy_cogs ------------------------------------------------------- + def test_gross_cogs_one_kwh(): - # 3.6e6 J == exactly 1 kWh. c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, tot_tok=1000, - idle_mean_w=100.0, price_per_kwh=0.33) + idle_mean_w=30.0, price_per_kwh=0.33) assert c["available"] is True - assert c["gross_usd"] == 0.33 # 1 kWh * $0.33 - assert c["gross_usd_per_1k_tok"] == 0.33 # 0.33 / 1000 * 1000 - assert c["mean_gen_w_measured"] == 36000.0 # 3.6e6 J / 100 s + assert c["gross_usd"] == 0.33 # 1 kWh * $0.33 + assert c["gross_usd_per_1k_tok"] == 0.33 + assert c["window_mean_w"] == 36000.0 # blend, honestly labelled -def test_marginal_subtracts_measured_warm_idle(): - # warm-idle 100 W over 100 s = 10_000 J of the 3.6e6 J is "keep hot". +def test_marginal_uses_serving_floor_when_present(): + # Serving floor (middle-idle 60W) is the right marginal floor, not idle. c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, tot_tok=1000, - idle_mean_w=100.0, price_per_kwh=0.33) - assert c["warm_idle_joules"] == 10000.0 - assert c["marginal_joules"] == 3.6e6 - 10000.0 - # marginal < gross because warm-idle energy is removed. + idle_mean_w=30.0, price_per_kwh=0.33, + serving_floor_w=60.0, gen_draw_w=400.0, gen_duty=0.5) + assert c["marginal_floor"] == "serving" + assert c["marginal_floor_w"] == 60.0 + assert c["marginal_joules"] == 3.6e6 - 60.0 * 100.0 + assert c["gen_draw_w_measured"] == 400.0 + assert c["gen_duty_cycle"] == 0.5 + + +def test_marginal_falls_back_to_idle_without_serving_floor(): + c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, tot_tok=1000, + idle_mean_w=30.0, price_per_kwh=0.33) + assert c["marginal_floor"] == "idle" + assert c["marginal_joules"] == 3.6e6 - 30.0 * 100.0 assert c["marginal_usd_per_1k_tok"] < c["gross_usd_per_1k_tok"] -def test_marginal_clamps_to_zero_when_idle_exceeds_load(): - # A noisy idle window can read hotter than a short quiet load window. +def test_marginal_clamps_to_zero_when_floor_exceeds_load(): c = energy_cogs(gpu_joules=5000.0, window_s=100.0, tot_tok=10, - idle_mean_w=100.0, price_per_kwh=0.33) + idle_mean_w=30.0, price_per_kwh=0.33, + serving_floor_w=100.0) assert c["marginal_joules"] == 0.0 assert c["marginal_usd"] == 0.0 -def test_no_idle_baseline_gives_gross_only(): +def test_no_floor_gives_gross_only(): c = energy_cogs(gpu_joules=3.6e6, window_s=100.0, tot_tok=1000, idle_mean_w=None, price_per_kwh=0.33) assert c["available"] is True - assert c["warm_idle_w_measured"] is None - assert "marginal_usd_per_1k_tok" not in c # can't decompose without it + assert "marginal_usd_per_1k_tok" not in c # no floor -> can't decompose def test_unavailable_when_inputs_missing(): - assert energy_cogs(None, 100.0, 1000, 100.0, 0.33)["available"] is False - assert energy_cogs(3.6e6, None, 1000, 100.0, 0.33)["available"] is False - assert energy_cogs(3.6e6, 100.0, 0, 100.0, 0.33)["available"] is False + assert energy_cogs(None, 100.0, 1000, 30.0, 0.33)["available"] is False + assert energy_cogs(3.6e6, None, 1000, 30.0, 0.33)["available"] is False + assert energy_cogs(3.6e6, 100.0, 0, 30.0, 0.33)["available"] is False def test_price_is_the_only_lever(): - # Same measured energy, double the rate -> double the dollar COGS. - a = energy_cogs(3.6e6, 100.0, 1000, 100.0, 0.33) - b = energy_cogs(3.6e6, 100.0, 1000, 100.0, 0.66) - # Tolerate the function's 6-decimal display rounding on the ratio. + a = energy_cogs(3.6e6, 100.0, 1000, 30.0, 0.33, serving_floor_w=60.0) + b = energy_cogs(3.6e6, 100.0, 1000, 30.0, 0.66, serving_floor_w=60.0) assert round(b["gross_usd"] / a["gross_usd"], 3) == 2.0 assert round(b["marginal_usd"] / a["marginal_usd"], 3) == 2.0 + + +# ---- classify_power_bands --------------------------------------------- + +def test_band_split_is_bimodal_on_idle_vs_gen(): + # 60W middle-idle gaps interleaved with 400W generation bursts. + samples = [60, 62, 58, 405, 398, 410, 61, 400, 59, 402, 63, 395] + b = classify_power_bands(samples) + assert b["available"] and b["bimodal"] + assert 55 <= b["low_band_w"] <= 70 # serving floor + assert 390 <= b["high_band_w"] <= 415 # generation draw + assert b["peak_w"] == 410 + assert 0.0 < b["duty_cycle"] < 1.0 + + +def test_band_unimodal_when_idle_only(): + # A quiet idle window: no distinct generation state. + b = classify_power_bands([30, 31, 29, 30, 32, 28, 31]) + assert b["available"] is True + assert b["bimodal"] is False + + +def test_band_unavailable_too_few_samples(): + assert classify_power_bands([400, 60])["available"] is False + + +def test_band_thresholds_not_hardcoded_scale_with_data(): + # Same shape at a different absolute scale (a different card) still + # splits — proves the split is data-derived, not a fixed watt cut. + b = classify_power_bands([12, 13, 11, 95, 98, 92, 12, 96]) + assert b["bimodal"] is True + assert b["low_band_w"] < 20 and b["high_band_w"] > 90