From 5260161e6fa44ecb0c4fe9b8f664f8307a703e8b Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 20 May 2026 12:33:58 -0400 Subject: [PATCH] feat(#000057): benchmark matrix doc (for David) + GPU wattage harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two deliverables for the cost/energy axis of the constraint optimizer. docs/benchmark-matrix.md — shareable spec of the control experiment: the question, fixture (386 office-holder Qs with corpus-vintage gold), the 3-model × 3-framing × 2-arm matrix (18 cells), the verdict vocabulary + two reads (accuracy vs grounding-fidelity), the deterministic code judge + its Opus calibration, the results-so-far table, and the NEW cost dimension (tokens / latency / GPU watts / joules-per-answer measured per GPU tier). Self-contained — readable cold by David. bench/watt_bench.py — GPU wattage harness. Samples nvidia-smi power.draw on the inference GPU while driving a small representative subset, reports mean/peak watts, trapezoid-integrated joules, joules-per-question, and joules-per-token. Tags the GPU (--gpu-label 3090|4090) so the optimizer can compare hardware tiers. Idle-baseline sampling separates load draw from idle. Does NOT grade (energy is independent of correctness); saves answers + per-question timing to JSONL for a later quality-per-joule pass via score_with_code_judge. Designed to run ON the GPU box (the orchestrator has no GPU; the 3090/4090 live on the inference boxes). Degrades gracefully when nvidia-smi is absent (energy fields null) so it is testable anywhere. Verified: PowerSampler graceful degradation + trapezoid integration (synthetic 100->200->200W over 2s = 350 J, exact). The headline cost finding the optimizer must weight: qwen-think reasoning = 1300-3300 tokens/answer vs qwen-nothink ~50-100 (20-50x), for a workload where arborist+qwen-nothink already lands 82% CG. The energy numbers will quantify whether reasoning's premium is ever justified — grounding-fidelity per joule, not per answer. --- bench/watt_bench.py | 304 +++++++++++++++++++++++++++++++++++++++ docs/benchmark-matrix.md | 139 ++++++++++++++++++ 2 files changed, 443 insertions(+) create mode 100644 bench/watt_bench.py create mode 100644 docs/benchmark-matrix.md diff --git a/bench/watt_bench.py b/bench/watt_bench.py new file mode 100644 index 0000000..474f9ce --- /dev/null +++ b/bench/watt_bench.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +"""GPU wattage harness for the #000057 benchmark matrix — energy as a +constraint-optimizer axis (fox 2026-05-20). + +The quality matrix (CG%) is scored by `bench/control_sweep.py`. This +harness adds the **cost** side: how much GPU energy does config X cost +to answer a question, on a 3090 vs a 4090? It samples +`nvidia-smi power.draw` on the inference GPU while driving a small +representative subset, and reports mean/peak watts, total joules, +joules-per-question, and joules-per-completion-token. + +**Run it ON the GPU box** (the machine whose GPU serves the model) so +`nvidia-smi` reads the GPU actually doing the inference. The orchestrator +box has no GPU; the 3090/4090 live on the inference boxes. The driven +endpoint can be localhost (same box) or the public URL — the power +draw is the same GPU either way. + +It does NOT grade answers (energy cost is independent of correctness). +Answers + per-question timing are saved to JSONL so a later +`bench/score_with_code_judge.py` pass can compute quality-per-joule. + +Usage (on the GPU box):: + + python -m bench.watt_bench --models qwen-nothink --n 20 \ + --gpu-label 4090 --endpoint http://localhost:8080/v1 + + # arborist arm (retrieval) energy: + python -m bench.watt_bench --arborist-ref qwen-nothink --n 20 \ + --gpu-label 3090 + +Output: `bench/qa_results/watt__.{jsonl,json}` — per-question +rows + an aggregate energy report. Compare two GPUs by running once on +each with the same `--n` and fixture slice. +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import threading +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from bench.control_ab import _SOLO_SYS, _gold # noqa: E402 +from bench.control_sweep import MODELS, VARIANTS # noqa: E402 + + +# ----------------------------------------------------------- power sampler + +class PowerSampler: + """Background thread polling `nvidia-smi --query-gpu=power.draw`. + + Records (timestamp, watts) samples. Integrates to joules via the + trapezoid rule over the sampled window. Degrades gracefully when + nvidia-smi is absent (``available=False``) so the harness runs — + and is testable — anywhere; only the energy fields go null.""" + + def __init__(self, gpu_index: int = 0, interval_s: float = 0.5): + self.gpu_index = gpu_index + self.interval_s = interval_s + self.samples: list[tuple[float, float]] = [] # (t, watts) + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self.available = self._probe() + self.gpu_name = self._gpu_name() if self.available else "no-nvidia-smi" + + def _probe(self) -> bool: + try: + subprocess.run(["nvidia-smi", "--version"], + capture_output=True, timeout=5) + return True + except Exception: # noqa: BLE001 + return False + + def _gpu_name(self) -> str: + try: + out = subprocess.run( + ["nvidia-smi", f"--id={self.gpu_index}", + "--query-gpu=name", "--format=csv,noheader"], + capture_output=True, text=True, timeout=5).stdout.strip() + return out or "unknown" + except Exception: # noqa: BLE001 + return "unknown" + + def _read_watts(self) -> float | None: + try: + out = subprocess.run( + ["nvidia-smi", f"--id={self.gpu_index}", + "--query-gpu=power.draw", "--format=csv,noheader,nounits"], + capture_output=True, text=True, timeout=5).stdout.strip() + return float(out.splitlines()[0]) + except Exception: # noqa: BLE001 + return None + + def _loop(self): + while not self._stop.is_set(): + w = self._read_watts() + if w is not None: + self.samples.append((time.time(), w)) + self._stop.wait(self.interval_s) + + def start(self): + if not self.available: + return + self._thread = threading.Thread(target=self._loop, daemon=True) + self._thread.start() + + def stop(self): + self._stop.set() + if self._thread: + self._thread.join(timeout=5) + + def stats(self) -> dict: + """Mean/peak watts and integrated joules over the sampled window.""" + if not self.samples: + return {"available": False, "n_samples": 0, + "mean_w": None, "peak_w": None, "joules": None, + "window_s": None} + ws = [w for _, w in self.samples] + mean_w = sum(ws) / len(ws) + peak_w = max(ws) + # Trapezoid integration of power over time → joules. + joules = 0.0 + for (t0, w0), (t1, w1) in zip(self.samples, self.samples[1:]): + joules += (w0 + w1) / 2.0 * (t1 - t0) + window_s = self.samples[-1][0] - self.samples[0][0] + return {"available": True, "n_samples": len(self.samples), + "mean_w": round(mean_w, 1), "peak_w": round(peak_w, 1), + "joules": round(joules, 1), "window_s": round(window_s, 1)} + + +# ----------------------------------------------------------- driver + +def _make_client(endpoint: str): + from arborist.qa.client import OpenAICompatibleClient + import os + return OpenAICompatibleClient( + base_url=endpoint, + api_key=os.environ.get("ARBORIST_LLM_API_KEY")) + + +def _solo_call(client, mkey: str, question: str) -> tuple[str, int]: + """One solo completion. Returns (answer, completion_tokens). The + bench client returns only the string; for token counts we re-issue + via a thin httpx call when usage is wanted — but to keep the energy + path simple we estimate tokens as len//4 when usage isn't surfaced.""" + cfg = MODELS[mkey] + ans = client.chat_completion( + [{"role": "system", "content": _SOLO_SYS}, + {"role": "user", "content": question}], + model=cfg["model"], max_tokens=cfg["max_tokens"], + extra_body=cfg["extra"]) + return ans, max(1, len(ans) // 4) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--fixture", default="bench/qa_questions_stale_map.json") + ap.add_argument("--n", type=int, default=20, + help="subset size — small & representative") + ap.add_argument("--models", default="qwen-nothink", + help="comma list for the solo arm (energy of the " + "model alone)") + ap.add_argument("--arborist-ref", default="", + help="if set, also measure the arborist (retrieval) " + "arm energy with this model as synthesis LLM") + ap.add_argument("--variants", default="plain") + ap.add_argument("--shards-dir", + default=str(Path.home() / ".arborist" / "shards")) + ap.add_argument("--gpu-index", type=int, default=0) + ap.add_argument("--gpu-label", default="", + help="human label for the GPU tier (e.g. 3090, 4090) " + "— stamped into the output for cross-GPU compare") + ap.add_argument("--endpoint", default="", + help="override the model endpoint (e.g. localhost on " + "the GPU box). Default = MODELS entry's endpoint.") + ap.add_argument("--sample-interval", type=float, default=0.5) + ap.add_argument("--idle-baseline-s", type=float, default=5.0, + help="seconds to sample idle power before driving load") + ap.add_argument("--out-dir", default="bench/qa_results") + a = ap.parse_args() + + models = [m for m in a.models.split(",") if m in MODELS] + variants = [v for v in a.variants.split(",") if v in VARIANTS] + items = json.loads(Path(a.fixture).read_text())[:a.n] + shards_dir = Path(a.shards_dir) + ts = time.strftime("%Y-%m-%dT%H-%M-%SZ", time.gmtime()) + label = a.gpu_label or "gpu" + outp = Path(a.out_dir) / f"watt_{label}_{ts}.jsonl" + outp.parent.mkdir(parents=True, exist_ok=True) + + sampler = PowerSampler(a.gpu_index, a.sample_interval) + print(f"watt_bench — gpu_label={label!r} detected={sampler.gpu_name!r} " + f"nvidia-smi={'yes' if sampler.available else 'NO (energy null)'}") + print(f" fixture={a.fixture} n={len(items)} models={models} " + f"variants={variants} arborist_ref={a.arborist_ref or 'none'}") + + # Idle baseline. + idle = None + if sampler.available and a.idle_baseline_s > 0: + print(f" sampling idle baseline for {a.idle_baseline_s}s …") + sampler.start() + time.sleep(a.idle_baseline_s) + sampler.stop() + idle = sampler.stats() + print(f" idle: mean={idle['mean_w']}W peak={idle['peak_w']}W") + sampler.samples = [] # reset for the load window + + cells = [] # one entry per (arm, model, variant) measured + + def run_cell(arm: str, mkey: str): + endpoint = a.endpoint or MODELS[mkey]["endpoint"] + client = _make_client(endpoint) + rows = [] + s = PowerSampler(a.gpu_index, a.sample_interval) + s.start() + t0 = time.time() + try: + for variant in variants: + for it in items: + q = VARIANTS[variant](it["question"]) + tq = time.time() + if arm == "solo": + ans, ctoks = _solo_call(client, mkey, q) + else: # arborist + from arborist.qa.query import ( + DEFAULT_QUERY_POLICY, query) + gold = _gold(shards_dir, it.get("shard", ""), + it["target_root"]) + pol = dict(DEFAULT_QUERY_POLICY, + answer_mode="claim_lattice") + if MODELS[mkey].get("reasoning"): + pol["claim_lattice_json_stop_sequences"] = [] + pol["max_tokens"] = 8192 + qa_db = Path("/tmp") / f"watt_{ts}_{mkey}.db" + r = query(question=q, qa_db=qa_db, + chat_client=client, + model_id=MODELS[mkey]["model"], + shards_dir=shards_dir, + extra_body=MODELS[mkey]["extra"], + policy=pol) + ans = r.get("raw_answer") or r.get("answer_text") or "" + ctoks = max(1, len(ans) // 4) + dt = time.time() - tq + rows.append({"arm": arm, "model": mkey, + "variant": variant, + "latency_s": round(dt, 2), + "answer_chars": len(ans), + "est_completion_tokens": ctoks}) + finally: + client.close() + s.stop() + st = s.stats() + elapsed = time.time() - t0 + nq = len(rows) + tot_tok = sum(r["est_completion_tokens"] for r in rows) + cell = { + "arm": arm, "model": mkey, "variants": variants, + "n_questions": nq, "elapsed_s": round(elapsed, 1), + "power": st, + "gpu_label": label, "gpu_name": sampler.gpu_name, + "idle_baseline": idle, + "joules_per_question": (round(st["joules"] / nq, 1) + if st.get("joules") and nq else None), + "est_total_completion_tokens": tot_tok, + "joules_per_token": (round(st["joules"] / tot_tok, 3) + if st.get("joules") and tot_tok else None), + "mean_latency_s": (round(sum(r["latency_s"] for r in rows) / nq, 2) + if nq else None), + } + cells.append(cell) + with open(outp, "a") as f: + for r in rows: + f.write(json.dumps({**r, "gpu_label": label}) + "\n") + jp = st.get("joules_per_question") + print(f" [{arm}/{mkey}] n={nq} mean={st.get('mean_w')}W " + f"peak={st.get('peak_w')}W J/q={jp} " + f"J/tok={cell['joules_per_token']} " + f"lat={cell['mean_latency_s']}s") + + for mkey in models: + run_cell("solo", mkey) + if a.arborist_ref and a.arborist_ref in MODELS: + run_cell("arborist", a.arborist_ref) + + report = {"ts": ts, "gpu_label": label, "gpu_name": sampler.gpu_name, + "nvidia_smi_available": sampler.available, + "idle_baseline": idle, "fixture": a.fixture, + "n_items": len(items), "cells": cells} + rp = outp.with_suffix(".json").with_name(f"watt_{label}_{ts}.json") + rp.write_text(json.dumps(report, indent=2)) + print(f"\n per-question rows: {outp}") + print(f" energy report: {rp}") + if not sampler.available: + print(" NOTE: nvidia-smi unavailable — energy fields are null. " + "Run this ON the GPU box for real wattage.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/benchmark-matrix.md b/docs/benchmark-matrix.md new file mode 100644 index 0000000..ec628ce --- /dev/null +++ b/docs/benchmark-matrix.md @@ -0,0 +1,139 @@ +# #000057 benchmark test matrix + +Shareable spec of the control-experiment benchmark — what we measure, +how, and the cells in the matrix. Companion to the rolling results in +`docs/qa-modes-bench.md` (Addendum 8). + +## The question + +A small language model, asked "who is the president of France?", answers +with the *present-day* office-holder ("Emmanuel Macron") even when the +grounding corpus is a fixed ~2010 vintage (where the answer is Sarkozy). +The model isn't lying — it's reciting its training-data present — but it +fails to recognise it should defer to the supplied source, or abstain +when it has none. The matrix isolates **what fixes that**: model scale, +question framing, retrieval, or reasoning. + +## Fixture + +- **386 office-holder questions** (`bench/qa_questions_stale_map.json`), + each paired with a **fixed corpus-vintage gold article** (~2010-2011, + verified from the artefact). The gold is the single source of truth a + grounded answer must match. +- Adversarial by construction: the corpus answer (Sarkozy) differs from + the model's training-data present (Macron), so a model reciting its + prior is measurably *wrong against the source*. + +## Axes + +| axis | values | what it isolates | +|------|--------|------------------| +| **model** | `hermes` (Hermes-3-8B, vLLM) · `qwen-nothink` (Qwen3.6-27B, llama.cpp, reasoning off) · `qwen-think` (same, reasoning on) | scale (8B vs 27B) and reasoning | +| **framing** | `plain` ("who is the president of France?") · `source_relative` ("According to the reference knowledge base, who is …") · `as_of_corpus` ("As of 2010, who was …") | whether prompt framing alone fixes the drift | +| **arm** | `solo` (model alone — measures the parametric prior) · `arborist` (model + retrieval over the 2010 corpus, `answer_mode=claim_lattice`) | whether retrieval fixes it | + +Full cross product = **3 models × 3 framings × 2 arms = 18 cells**, each +over up to 386 items. (The `arborist` arm currently runs `hermes` and +`qwen-nothink`; `qwen-think` is measurable but cost-prohibitive — see +"Cost dimension".) + +## Metrics + +Every (question, answer, gold) triple gets one verdict from a closed +vocabulary: + +| verdict | meaning | +|---------|---------| +| **CORRECT_GROUNDED** (CG) | answer matches the gold source | +| **WRONG** (W) | gold states something the answer contradicts | +| **FABRICATED** (F) | answer asserts specifics absent from gold | +| **ABSTAINED** (A) | answer honestly declines ("I don't have that") | +| **JUDGE_ERROR** (JE) | judge could not classify deterministically | + +Two reads of the same verdicts: + +- **accuracy** — "did it say the corpus answer?" Penalises a model for + knowing the *current* office-holder. Reported but flagged as + misleading. +- **grounding-fidelity** (the defensible read) — "did the model do the + right conversational thing: answer when grounded, abstain when not?" + W + F are failures (confident ungrounded assertion); A is success on + the questions the model can't ground. **CG% and abstain% are the + headline numbers.** + +## The judge + +Deterministic, no LLM, no quota: `bench/judge_code.py`. Pipeline (first +hit decides): empty/no-gold guard → explicit-abstention regex → +short-answer entity-grounding fast path → NLI contradiction (θ=0.85) → +lexical verifier (quote/span/entity/paraphrase) → WRONG-vs-FABRICATED +tie-break on subject-in-gold. Claim-lattice JSON envelopes are unwrapped +to prose before grading so the `arborist` arm grades on equal terms with +`solo`. + +The judge was calibrated against an Opus (SOTA) reference judge on the +records Opus could grade: agreement CG 13→47 %, WRONG 56→89 %, ABSTAINED +80→95 %. The deterministic verdict is a *proxy* for grounding (lexical + +NLI), not Opus-grade reading; JUDGE_ERROR residue is the natural input +to a later LLM-batch pass. NLI runs on a GPU (currently the 4090 `ai` +box); the rest is pure Python. + +## Cost dimension (new — for the constraint optimizer) + +The matrix has been scored on **quality** (CG%) but not yet on **cost**. +Adding the energy axis so the constraint optimizer can trade grounding +fidelity against power: + +| cost metric | unit | source | +|-------------|------|--------| +| tokens / answer | completion tokens | API usage field | +| latency / answer | seconds | wall-clock per cell | +| **GPU power** | **watts (mean / peak)** | **`nvidia-smi power.draw`, sampled on the inference GPU during the cell** | +| **energy / answer** | **joules** | **∫ power dt / questions** | + +Measured per GPU (**3090** and **4090**) so the optimizer knows the +energy cost of each config on each hardware tier. The wattage harness +(`bench/watt_bench.py`) samples `nvidia-smi` on the GPU box while driving +a small representative subset of the matrix. + +**Why it matters:** reasoning chains are the headline cost finding — +`qwen-think` spends 1300-3300 completion tokens/answer vs `qwen-nothink`'s +~50-100 (**20-50× the token cost**), for a workload where +`arborist + qwen-nothink` already lands **82 % CG**. The optimizer should +weight that: grounding-fidelity *per joule*, not per answer. + +## Results so far (CG%, all arms on the identical calibrated judge) + +| arm / model | plain | source_relative | as_of_corpus | +|-------------|-------|-----------------|--------------| +| solo / hermes | 9 % | 5 % | 18 % | +| solo / qwen-nothink | 7 % | 0 % | 50 % | +| solo / qwen-think | 6 % | 5 % | 44 % | +| arborist / hermes (n=40) | 60 % | 62 % | 25 % | +| **arborist / qwen-nothink** | **82 %** | **65 %** | 50 % | + +Headline findings: (1) **retrieval dominates** — no solo config +approaches the arborist arms; (2) **reasoning doesn't improve raw +correctness** (qwen-think/as_of 44 % vs nothink 50 %) and **breaks +honest-abstention** under source_relative framing (nothink abstains +98 %, think only 62 %); (3) **production answer: arborist + qwen-nothink, +plain framing, reasoning off — 82 % CG.** The cost axis will confirm +whether reasoning's energy premium is ever justified. + +## Reproduce + +``` +# full matrix cell (one model, all framings, both arms) +python -m bench.control_sweep --judge code --models qwen-nothink \ + --arborist-ref qwen-nothink --n 386 --arborist-n 386 --max-workers 1 + +# arborist-only (solo data already collected) +python -m bench.control_sweep ... --skip-solo + +# wattage subset (run ON the GPU box) +python -m bench.watt_bench --models qwen-nothink --n 20 --gpu-label 4090 +``` + +Artifacts land in `bench/qa_results/`; scorecards via +`bench/score_with_code_judge.py`; judge-vs-judge reconciliation via +`bench/analyze_judge_disagreement.py`.