From e227bbc32a3e0cfe08e4892540ba0d8d5b44561c Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Thu, 21 May 2026 09:43:17 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20pin=20STOCK=20V.1=20substrate=20?= =?UTF-8?q?=E2=80=94=20frozen=20substrate-ON=20config=20for=20the=20GPU=20?= =?UTF-8?q?campaign?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before the multi-day campaign (hermes 3090/4090 -> qwen 3090/4090 -> reasoning variants) the substrate-ON treatment arm must NOT drift. It previously inherited DEFAULT_QUERY_POLICY implicitly, so any mid-run edit would silently change what 'substrate-ON' means. bench/stock_v1.py snapshots DEFAULT_QUERY_POLICY + re-asserts the load-bearing pins (answer_mode=quote, crosslang OFF, repair OFF, quantifier caps dry-run, metacognition label-only, soft-preflight OFF, claim ceiling 12, v2-acronym-aware), then hashes the whole effective dict. assert_not_drifted() fails loudly if that hash ever changes — re-pinning is a deliberate fox-gated V.2 bump, never silent. The whole campaign is identified by one governance_policy_hash (5b6ca4c5...aade4e). Non-reasoning + non-distributed are harness axes (reasoning -> phase 3, mesh -> later fork), not policy fields. docs/stock-v1-config.md documents V.1, substrate-OFF (control_ab arm A), the campaign matrix, and the energy-COGS companion (#000057) — whose power states (idle / warm-idle / generation) are MEASURED per card+model+inference-server at runtime, never hardcoded; only $/kWh is an operator flag. --- bench/stock_v1.py | 123 ++++++++++++++++++++++++++++++++++++++++ docs/stock-v1-config.md | 84 +++++++++++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 bench/stock_v1.py create mode 100644 docs/stock-v1-config.md diff --git a/bench/stock_v1.py b/bench/stock_v1.py new file mode 100644 index 0000000..4038174 --- /dev/null +++ b/bench/stock_v1.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""STOCK V.1 — the frozen substrate-ON configuration under test. + +fox 2026-05-21: before the multi-day GPU campaign (hermes 3090/4090 → +qwen 3090/4090 → reasoning variants), pin exactly ONE substrate-ON +config so the treatment arm cannot silently drift across the run. The +control arms (``bench/control_ab.py`` arm A = bare-model-solo, +``bench/control_sweep.py``) and the substrate-ON arm B (the full +``query()`` pipeline) must all measure the SAME substrate every cell, +or the substrate-OFF-vs-ON delta is meaningless. + +What "STOCK V.1" is +------------------- +The full merkle-agi-dag reverse-RAG SQD/Prometheus-σ recursive- +falsification substrate, **non-reasoning** and **non-distributed**: + + * answer_mode = quote (bench leader on raw lexical grounding) + * cross-language guard / sandwich-MT / entity-mask = OFF + (English-only; MT is a separate capability) + * mechanical answer-repair = OFF (one-shot discipline) + * quantifier caps = reported, not applied (dry-run) + * metacognition preflight = label-only (no verdict gating) + * soft-preflight LLM sidecar = OFF (no extra round-trip) + * claim ceiling = 12 + * verifier content-token rules = v2-acronym-aware + * reasoning (inference layer) = OFF -> reasoning variants are phase 3 + * mesh / multi-witness = OFF -> distributed is the later fork + +How the freeze is enforced +-------------------------- +``STOCK_V1_POLICY`` is a deep snapshot of the live +``arborist.qa.query.DEFAULT_QUERY_POLICY`` with the load-bearing knobs +re-asserted explicitly (they ARE the current defaults — this is a +freeze, not a change). We then compute ``governance_policy_hash`` over +the WHOLE effective dict (the same hash that lands in every cache_key) +and assert it equals ``STOCK_V1_GOVERNANCE_HASH``. If anyone edits +``DEFAULT_QUERY_POLICY`` mid-campaign — a pinned knob OR any other +field the hash covers — the assert fails LOUDLY and the harness refuses +to run a drifted substrate. The whole 3-4 day run is therefore +identified by one hash. + +This is the drift guard, not value duplication: we do not hand-copy the +big prompt strings (they would rot against source); we snapshot + pin +the hash. +""" + +from __future__ import annotations + +import copy + +from arborist.qa.keys import governance_policy_hash +from arborist.qa.query import DEFAULT_QUERY_POLICY + +#: The knobs that DEFINE substrate-ON V.1, re-asserted explicitly so the +#: intent is readable at a glance. Every value here equals the current +#: ``DEFAULT_QUERY_POLICY`` default — see the module docstring. +STOCK_V1_PINS: dict = { + "answer_mode": "quote", + "crosslang_guard_enabled": False, + "crosslang_translate_enabled": False, + "crosslang_entity_mask": False, + "repair_enabled": False, + "soft_preflight_enabled": False, + "metacognition_block_on_contradiction": False, + "quantifier_guard_apply_caps": False, + "quantifier_reject_broad": False, + "claim_lattice_max_claims_per_answer": 12, + "content_token_rules": "v2-acronym-aware", +} + +#: Frozen substrate-ON policy: live defaults + explicit pins. +STOCK_V1_POLICY: dict = {**copy.deepcopy(DEFAULT_QUERY_POLICY), **STOCK_V1_PINS} + +#: Inference / harness axes — NOT policy fields (they don't fold into +#: governance_policy_hash). Phase 1+2 hold these; phase 3 flips reasoning. +STOCK_V1_REASONING = False # reasoning variants are campaign phase 3 +STOCK_V1_DISTRIBUTED = False # mesh / multi-witness is the later fork + +#: The one hash that identifies the whole campaign. Pinned 2026-05-21 +#: against DEFAULT_QUERY_POLICY at commit b5cc970. Re-pin (with a fox +#: go) only when a deliberate substrate change is intended. +STOCK_V1_GOVERNANCE_HASH = ( + "5b6ca4c5e754e96b7e2e8af16a8948dec8d8b2a80b9304df21b1f6d494aade4e" +) + + +def _live_governance_hash() -> str: + """The governance_policy_hash of the current frozen policy.""" + return governance_policy_hash(STOCK_V1_POLICY) + + +def assert_not_drifted() -> str: + """Fail loudly if DEFAULT_QUERY_POLICY has drifted from the V.1 pin. + + Returns the live hash on success. The campaign harness calls this + before any substrate-ON run so a mid-campaign edit to + DEFAULT_QUERY_POLICY (pinned knob or otherwise) cannot silently + change what "substrate-ON" means. + """ + live = _live_governance_hash() + if STOCK_V1_GOVERNANCE_HASH == "__PIN_ME__": + return live # un-pinned bootstrap: caller prints + bakes the value + if live != STOCK_V1_GOVERNANCE_HASH: + raise SystemExit( + "STOCK V.1 DRIFT: DEFAULT_QUERY_POLICY no longer hashes to the\n" + f" pinned campaign substrate.\n" + f" pinned: {STOCK_V1_GOVERNANCE_HASH}\n" + f" live: {live}\n" + " Either revert the policy change, or (with a fox go) re-pin\n" + " STOCK_V1_GOVERNANCE_HASH and bump the campaign to V.2." + ) + return live + + +if __name__ == "__main__": + import json + + print("STOCK V.1 substrate-ON config") + print(f" governance_policy_hash : {_live_governance_hash()}") + print(f" reasoning : {STOCK_V1_REASONING}") + print(f" distributed : {STOCK_V1_DISTRIBUTED}") + print(" explicit pins:") + print(json.dumps(STOCK_V1_PINS, indent=4, sort_keys=True)) diff --git a/docs/stock-v1-config.md b/docs/stock-v1-config.md new file mode 100644 index 0000000..4540a07 --- /dev/null +++ b/docs/stock-v1-config.md @@ -0,0 +1,84 @@ +# STOCK V.1 — the frozen substrate under multi-day test + +**Pinned 2026-05-21 (fox).** Before the multi-day GPU campaign, exactly +one substrate-ON configuration is frozen so the treatment arm cannot +silently drift across a 3-4 day run. Source of truth: +[`bench/stock_v1.py`](../bench/stock_v1.py). + +## What V.1 is + +The full merkle-agi-dag reverse-RAG SQD / Prometheus-σ recursive- +falsification substrate, **non-reasoning** and **non-distributed**. + +| knob | value | note | +|---|---|---| +| `answer_mode` | `quote` | bench leader on raw lexical grounding | +| `temperature` / `top_p` / `max_tokens` | `0.1` / `1.0` / `512` | from `DEFAULT_QUERY_POLICY` | +| `repair_enabled` | `False` | one-shot discipline, no self-heal reprompts | +| crosslang guard / translate / entity-mask | **OFF** | English-only; sandwich-MT is a separate capability | +| `quantifier_guard_apply_caps` | `False` | caps reported, not applied (dry-run) | +| `quantifier_reject_broad` | `False` | no preflight rejection | +| `metacognition_*` | label-only | `block_on_contradiction=False`, no verdict gating | +| `soft_preflight_enabled` | `False` | no extra LLM round-trip | +| `claim_lattice_max_claims_per_answer` | `12` | runaway guard | +| `content_token_rules` | `v2-acronym-aware` | verifier token rules | +| `base_version` | `wikitext-base-v1` | prose normalization | +| chunker / norm / schema | `tok-512-v1` / `norm-v1` / `v9.8.0` | versioned defaults | +| **reasoning** (inference layer) | **OFF** | reasoning variants are phase 3 | +| **mesh / multi-witness** | **OFF** | distributed is the later fork | + +**substrate-OFF** = bare model, no retrieval, no verifier — `control_ab` +arm A (question-only, neutral system prompt; gold text supplied +identically to both arms per the §4b ruling). + +## The one hash + +``` +governance_policy_hash = 5b6ca4c5e754e96b7e2e8af16a8948dec8d8b2a80b9304df21b1f6d494aade4e +``` + +This identifies the whole campaign. `bench/stock_v1.py` snapshots the +live `DEFAULT_QUERY_POLICY`, re-asserts the load-bearing pins, hashes +the whole effective dict, and `assert_not_drifted()` fails **loudly** if +that hash ever changes — a mid-campaign edit to `DEFAULT_QUERY_POLICY` +(a pinned knob *or any other field the hash covers*) stops the harness +rather than quietly changing what "substrate-ON" means. Re-pinning is a +deliberate fox-gated bump to V.2, never silent. + +## Campaign matrix + +The substrate (above) is **frozen**; these axes are **swept**: + +1. **hermes-8B**, 3090 + 4090 — ~few days +2. **qwen-27B**, 3090 + 4090 +3. **reasoning variants** — hermes-reasoning + qwen-reasoning + +Models + reasoning flags live in `bench/control_sweep.py:MODELS`. +Quality (CG%) is scored by `control_sweep` / `control_ab`; non-jaggedness +by `bench/jaggedness.py` (#000060). + +## Cost axis — energy COGS (companion, #000057) + +`bench/watt_bench.py` adds the cost side: GPU energy per question / per +completion-token, on each card. Energy COGS decomposes into the power +states the card actually occupies — idle, warm-idle (model resident, +waiting), and generation (request burst): + +- **marginal** `(P_gen − P_warm_idle) · t_gen / tokens` — what one more + token actually costs; +- **gross / amortized** — all-in, including the warm-idle cost of + keeping the model hot, divided across throughput (the COGS-vs- + utilization curve); +- **dollar COGS** — `joules / 3.6e6 → kWh × $/kWh`, then `÷ tokens` + for `$/1k-tok` against API pricing. + +**Every power state is MEASURED per (card, model, inference-server) at +runtime — never hardcoded.** `P_warm_idle` comes from a no-request +calibration window; `P_gen` from the burst during actual completions. +Each card × model × server has its own profile. The only operator input +is `$/kWh`, a configurable site flag. + +Because the single-slot endpoints (hermes 3090 / vLLM, qwen 4090 / +llama.cpp) serve **real internet traffic and cannot be isolated**, +energy attribution cross-references `bench/load_monitor.py` queue-depth ++ req/s so organic-traffic bursts don't get counted as bench cost.