From ab8df7679229b9d0ec76a70b0bd4f87c576fc99b Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 20 May 2026 14:45:49 -0400 Subject: [PATCH] feat(#000057): CPU wattage (RAPL) in watt_bench + expanded cost/quality matrix doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Toward fox's next goal: score the full serving stack on quality AND cost — {qwen, hermes} × {llama.cpp, vLLM} × {3090, 4090} × {solo, arborist}, measuring CG% + GPU watts + CPU watts + joules/answer per cell. watt_bench.py — adds CpuSampler (Intel RAPL package energy via /sys/class/powercap/intel-rapl:*/energy_uj). RAPL exposes a cumulative microjoule counter, so energy-over-window is an end-minus-start diff (handles wrap) — more accurate than integrating instantaneous power. Sums multi-package. energy_uj is root-only by default (PLATYPUS / CVE-2020-8694), so it degrades to available=False when locked; --cpu-energy-cmd 'sudo cat {path}' supplies a privileged reader when a sudo rule exists. Each cell now reports gpu/cpu/total joules-per- question + gpu joules-per-token; the report records cpu_rapl_available. Verified: graceful degradation when locked; RAPL diff math (1->4 MJ uJ = 3.0 J, exact). benchmark-matrix.md — expands the cost section to the full 16-cell (model × engine × GPU × arm) design, the per-cell metric set (quality + GPU + CPU energy), the serving-stack inventory from 2026-05-20 recon (4090=qwen/llama.cpp, 3090=hermes/vLLM — each box has one engine + one model today), and the buildout gap (vLLM+qwen, llama.cpp+hermes, cross- GPU models). Notes idle-floor asymmetry (hermes/3090 ~127W vs qwen/4090 ~20W) as a real optimizer input. Harness is ready; the serving-config buildout + RAPL perm grant are the remaining (ops, fox-directed) prerequisites to run the full matrix. --- bench/watt_bench.py | 133 ++++++++++++++++++++++++++++++++++++--- docs/benchmark-matrix.md | 68 ++++++++++++++------ 2 files changed, 173 insertions(+), 28 deletions(-) diff --git a/bench/watt_bench.py b/bench/watt_bench.py index 474f9ce..e84105d 100644 --- a/bench/watt_bench.py +++ b/bench/watt_bench.py @@ -132,6 +132,96 @@ class PowerSampler: "joules": round(joules, 1), "window_s": round(window_s, 1)} +# ----------------------------------------------------------- cpu power (RAPL) + +class CpuSampler: + """CPU package energy via Intel RAPL (`/sys/class/powercap/intel-rapl`). + + RAPL exposes a CUMULATIVE energy counter in microjoules + (``energy_uj``); the energy used over a window is just the + end-minus-start diff (handling the wrap at ``max_energy_range_uj``), + so we don't integrate — RAPL gives joules directly, more accurate + than sampling instantaneous power. Mean watts = joules / seconds. + + ``energy_uj`` is root-only by default on most kernels (the + PLATYPUS side-channel mitigation, CVE-2020-8694). When it isn't + readable we degrade to ``available=False`` and report null CPU + energy — the GPU path is unaffected. Pass ``--cpu-energy-cmd`` to + supply a privileged reader (e.g. a sudo rule) when one exists. + + Sums all RAPL packages (multi-socket) under intel-rapl:N.""" + + def __init__(self, energy_cmd: str | None = None): + self.energy_cmd = energy_cmd # optional privileged reader template + self._pkgs = self._discover_packages() + self._t0: float | None = None + self._e0: int | None = None + self.available = self._probe() + + def _discover_packages(self) -> list[Path]: + base = Path("/sys/class/powercap") + if not base.exists(): + return [] + return sorted(p / "energy_uj" for p in base.glob("intel-rapl:*") + if (p / "energy_uj").exists() and ":" in p.name + and p.name.count(":") == 1) # top-level packages only + + def _read_one(self, path: Path) -> int | None: + if self.energy_cmd: + try: + out = subprocess.run( + self.energy_cmd.format(path=str(path)), shell=True, + capture_output=True, text=True, timeout=5).stdout.strip() + return int(out) + except Exception: # noqa: BLE001 + return None + try: + return int(path.read_text().strip()) + except Exception: # noqa: BLE001 + return None + + def _read_total(self) -> int | None: + if not self._pkgs: + return None + vals = [self._read_one(p) for p in self._pkgs] + if any(v is None for v in vals): + return None + return sum(vals) + + def _probe(self) -> bool: + return self._read_total() is not None + + def start(self): + if not self.available: + return + self._e0 = self._read_total() + self._t0 = time.time() + + def stats(self) -> dict: + if not self.available or self._e0 is None: + return {"available": False, "joules": None, "mean_w": None, + "window_s": None, "n_packages": len(self._pkgs)} + e1 = self._read_total() + t1 = time.time() + if e1 is None: + return {"available": False, "joules": None, "mean_w": None, + "window_s": None, "n_packages": len(self._pkgs)} + # energy_uj wraps; if it went down assume a single wrap per pkg. + # max_energy_range_uj is per-package; approximate with the + # observed positive delta when no wrap, else mark suspect. + d_uj = e1 - self._e0 + if d_uj < 0: + # wrap occurred — best-effort: skip rather than report garbage + return {"available": True, "joules": None, "mean_w": None, + "window_s": round(t1 - self._t0, 1), + "n_packages": len(self._pkgs), "note": "counter wrapped"} + joules = d_uj / 1_000_000.0 + window = t1 - self._t0 + return {"available": True, "joules": round(joules, 1), + "mean_w": round(joules / window, 1) if window else None, + "window_s": round(window, 1), "n_packages": len(self._pkgs)} + + # ----------------------------------------------------------- driver def _make_client(endpoint: str): @@ -180,6 +270,13 @@ def main() -> int: ap.add_argument("--sample-interval", type=float, default=0.5) ap.add_argument("--idle-baseline-s", type=float, default=5.0, help="seconds to sample idle power before driving load") + ap.add_argument("--cpu-energy-cmd", default="", + help="optional privileged reader for RAPL energy_uj " + "when it's root-only (PLATYPUS mitigation), e.g. " + "'sudo cat {path}'. {path} is substituted with " + "the intel-rapl energy_uj sysfs path. Omit to " + "read directly (works only if energy_uj is " + "world-readable).") ap.add_argument("--out-dir", default="bench/qa_results") a = ap.parse_args() @@ -193,8 +290,11 @@ def main() -> int: outp.parent.mkdir(parents=True, exist_ok=True) sampler = PowerSampler(a.gpu_index, a.sample_interval) + cpu_probe = CpuSampler(a.cpu_energy_cmd or None) print(f"watt_bench — gpu_label={label!r} detected={sampler.gpu_name!r} " f"nvidia-smi={'yes' if sampler.available else 'NO (energy null)'}") + print(f" cpu RAPL: {'yes' if cpu_probe.available else 'NO (root-only / locked — pass --cpu-energy-cmd)'}" + f" ({len(cpu_probe._pkgs)} package(s))") print(f" fixture={a.fixture} n={len(items)} models={models} " f"variants={variants} arborist_ref={a.arborist_ref or 'none'}") @@ -216,7 +316,9 @@ def main() -> int: client = _make_client(endpoint) rows = [] s = PowerSampler(a.gpu_index, a.sample_interval) + cpu = CpuSampler(a.cpu_energy_cmd or None) s.start() + cpu.start() t0 = time.time() try: for variant in variants: @@ -254,20 +356,29 @@ def main() -> int: client.close() s.stop() st = s.stats() + cpu_st = cpu.stats() elapsed = time.time() - t0 nq = len(rows) tot_tok = sum(r["est_completion_tokens"] for r in rows) + gpu_j = st.get("joules") + cpu_j = cpu_st.get("joules") + total_j = (gpu_j or 0) + (cpu_j or 0) if (gpu_j or cpu_j) else None cell = { "arm": arm, "model": mkey, "variants": variants, "n_questions": nq, "elapsed_s": round(elapsed, 1), - "power": st, + "gpu_power": st, + "cpu_power": cpu_st, "gpu_label": label, "gpu_name": sampler.gpu_name, "idle_baseline": idle, - "joules_per_question": (round(st["joules"] / nq, 1) - if st.get("joules") and nq else None), + "gpu_joules_per_question": (round(gpu_j / nq, 1) + if gpu_j and nq else None), + "cpu_joules_per_question": (round(cpu_j / nq, 1) + if cpu_j and nq else None), + "total_joules_per_question": (round(total_j / nq, 1) + if total_j and nq else None), "est_total_completion_tokens": tot_tok, - "joules_per_token": (round(st["joules"] / tot_tok, 3) - if st.get("joules") and tot_tok else None), + "gpu_joules_per_token": (round(gpu_j / tot_tok, 3) + if gpu_j and tot_tok else None), "mean_latency_s": (round(sum(r["latency_s"] for r in rows) / nq, 2) if nq else None), } @@ -275,10 +386,12 @@ def main() -> int: with open(outp, "a") as f: for r in rows: f.write(json.dumps({**r, "gpu_label": label}) + "\n") - jp = st.get("joules_per_question") - print(f" [{arm}/{mkey}] n={nq} mean={st.get('mean_w')}W " - f"peak={st.get('peak_w')}W J/q={jp} " - f"J/tok={cell['joules_per_token']} " + print(f" [{arm}/{mkey}] n={nq} " + f"gpu={st.get('mean_w')}W/{st.get('peak_w')}peak " + f"cpu={cpu_st.get('mean_w')}W " + f"J/q gpu={cell['gpu_joules_per_question']} " + f"cpu={cell['cpu_joules_per_question']} " + f"tot={cell['total_joules_per_question']} " f"lat={cell['mean_latency_s']}s") for mkey in models: @@ -288,6 +401,8 @@ def main() -> int: report = {"ts": ts, "gpu_label": label, "gpu_name": sampler.gpu_name, "nvidia_smi_available": sampler.available, + "cpu_rapl_available": cpu_probe.available, + "cpu_rapl_packages": len(cpu_probe._pkgs), "idle_baseline": idle, "fixture": a.fixture, "n_items": len(items), "cells": cells} rp = outp.with_suffix(".json").with_name(f"watt_{label}_{ts}.json") diff --git a/docs/benchmark-matrix.md b/docs/benchmark-matrix.md index ec628ce..59280a7 100644 --- a/docs/benchmark-matrix.md +++ b/docs/benchmark-matrix.md @@ -78,29 +78,59 @@ NLI), not Opus-grade reading; JUDGE_ERROR residue is the natural input to a later LLM-batch pass. NLI runs on a GPU (currently the 4090 `ai` box); the rest is pure Python. -## Cost dimension (new — for the constraint optimizer) +## Cost dimension — for the constraint optimizer (expanded goal) -The matrix has been scored on **quality** (CG%) but not yet on **cost**. -Adding the energy axis so the constraint optimizer can trade grounding -fidelity against power: +The next-goal matrix scores **quality AND cost** across the serving +stack, not just the model. Every cell is a deployable configuration; the +optimizer trades grounding-fidelity against energy to pick the config to +ship. -| cost metric | unit | source | -|-------------|------|--------| -| tokens / answer | completion tokens | API usage field | -| latency / answer | seconds | wall-clock per cell | -| **GPU power** | **watts (mean / peak)** | **`nvidia-smi power.draw`, sampled on the inference GPU during the cell** | -| **energy / answer** | **joules** | **∫ power dt / questions** | +**Expanded axes (the full cost/quality matrix):** -Measured per GPU (**3090** and **4090**) so the optimizer knows the -energy cost of each config on each hardware tier. The wattage harness -(`bench/watt_bench.py`) samples `nvidia-smi` on the GPU box while driving -a small representative subset of the matrix. +| axis | values | +|------|--------| +| model | qwen-27B · hermes-8B | +| **engine** | **llama.cpp · vLLM** (where the model/engine combo is supported) | +| **GPU** | **RTX 3090 · RTX 4090** | +| arm | solo · arborist (retrieval) | -**Why it matters:** reasoning chains are the headline cost finding — -`qwen-think` spends 1300-3300 completion tokens/answer vs `qwen-nothink`'s -~50-100 (**20-50× the token cost**), for a workload where -`arborist + qwen-nothink` already lands **82 % CG**. The optimizer should -weight that: grounding-fidelity *per joule*, not per answer. +Cross product = 2 model × 2 engine × 2 GPU × 2 arm = up to **16 +quality+cost cells**. Each cell records: + +| metric | unit | source | +|--------|------|--------| +| quality (CG% / abstain%) | rate | code judge | +| tokens / answer | completion tokens | API usage | +| latency / answer | seconds | wall-clock | +| **GPU energy** | **watts mean/peak · joules · J/answer · J/token** | `nvidia-smi power.draw` on the inference GPU | +| **CPU energy** | **watts mean · joules · J/answer** | Intel RAPL `energy_uj` (package sum) on the box | + +`bench/watt_bench.py` samples both GPU (`nvidia-smi`) and CPU (RAPL) on +the inference box while driving the subset. RAPL `energy_uj` is root-only +by default (PLATYPUS / CVE-2020-8694 mitigation) — pass +`--cpu-energy-cmd 'sudo cat {path}'` with a sudo rule, or relax the +sysfs perm, to capture CPU watts; GPU watts need no special perm. + +**Serving-stack inventory (recon 2026-05-20):** + +| box | host | GPU | engine present | model present | +|-----|------|-----|----------------|---------------| +| 4090 | `ai.foxhop.net:18888` | RTX 4090, i9-14900K (32t) | llama.cpp | qwen-27B gguf | +| 3090 | `3090-ai.foxhop.net:18888` | RTX 3090, i9-12900K (24t) | vLLM (venv) | hermes-8B | + +To fill the matrix, the missing serving configs must be stood up: +vLLM+qwen, llama.cpp+hermes (gguf), and each model on the other GPU. +That's an infra buildout (install engines, fetch models, manage the +single-slot live endpoints), tracked separately from the harness. + +**Why cost matters most here:** reasoning chains are the headline cost +finding — `qwen-think` spends 1300-3300 completion tokens/answer vs +`qwen-nothink`'s ~50-100 (**20-50× the token cost**), for a workload +where `arborist + qwen-nothink` already lands **82 % CG**. And hermes-8B +on a 3090 idles at ~127 W vs qwen-27B on a 4090 at ~20 W — idle floor, +model size, engine, and GPU tier all move the joules-per-grounded-answer +the optimizer cares about. The metric is **grounding-fidelity per joule**, +not per answer. ## Results so far (CG%, all arms on the identical calibrated judge)