From 1a7f8eb4ea1babc4a86c3885dd4933bac851260b Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Thu, 21 May 2026 10:15:26 -0400 Subject: [PATCH] feat: STOCK V.1 two-mode config family + wire treatment arms to the pin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fox 2026-05-21: characterize substrate-ON under BOTH answer shapes, so answer_mode is a swept axis, not a single pinned value. stock_v1.py now exposes STOCK_V1_POLICIES{quote,claim_lattice} + STOCK_V1_GOVERNANCE_HASHES (quote 5b6ca4c5..., claim_lattice 036a4c79...), policy_for(mode), and assert_not_drifted(mode). Shared pins (crosslang OFF, repair OFF, quantifier dry-run, metacognition label-only, soft-preflight OFF, claim cap 12, v2-acronym-aware) are frozen identically across modes. Wire the treatment arms to the pin (the consumer-side step that makes the freeze real): * control_ab --answer-mode {quote,claim_lattice} * control_sweep --arborist-answer-mode {quote,claim_lattice} Both default claim_lattice (prior behavior), call assert_not_drifted on non-reasoning runs (halts the sweep if DEFAULT_QUERY_POLICY drifts), and load the frozen policy_for(mode) instead of an inline dict(DEFAULT_QUERY_POLICY, ...). Reasoning refs (phase 3) keep their documented JSON overrides and skip the assert by design (different hash). jaggedness is left standalone — it is a mode-agnostic retrieval instrument, coupling it to the answer-policy freeze adds friction with no correctness gain. Full suite 2528 passed. --- bench/control_ab.py | 18 ++++- bench/control_sweep.py | 30 +++++-- bench/stock_v1.py | 168 ++++++++++++++++++++++++---------------- docs/stock-v1-config.md | 45 ++++++++--- 4 files changed, 177 insertions(+), 84 deletions(-) diff --git a/bench/control_ab.py b/bench/control_ab.py index d775a60..7a4988c 100644 --- a/bench/control_ab.py +++ b/bench/control_ab.py @@ -115,6 +115,11 @@ def main() -> int: "ARBORIST_LLM_MODEL", "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic")) ap.add_argument("--out-dir", default="bench/qa_results") + ap.add_argument("--answer-mode", choices=["quote", "claim_lattice"], + default="claim_lattice", + help="STOCK V.1 substrate-ON answer shape (the two-cell " + "config family). Treatment arm B runs the frozen " + "bench.stock_v1 policy for this mode.") ap.add_argument("--judge", choices=sorted(_JUDGES.keys()), default="code", help="which judge to use. 'code' (default, no LLM) is " @@ -125,7 +130,15 @@ def main() -> int: judge, _judge_model_id = _JUDGES[a.judge] from arborist.qa.client import OpenAICompatibleClient - from arborist.qa.query import DEFAULT_QUERY_POLICY, query + from arborist.qa.query import query + + # STOCK V.1 frozen substrate-ON policy. assert_not_drifted halts the + # run if DEFAULT_QUERY_POLICY changed under us, so a mid-campaign edit + # can't silently redefine "substrate-ON". See bench/stock_v1.py. + from bench.stock_v1 import assert_not_drifted as _assert_stock + from bench.stock_v1 import policy_for as _stock_policy_for + _assert_stock(a.answer_mode) + arb_policy = _stock_policy_for(a.answer_mode) items = json.loads(Path(a.fixture).read_text())[:a.n] shards_dir = Path(a.shards_dir) @@ -167,8 +180,7 @@ def main() -> int: try: r = query(question=q, qa_db=qa_db, chat_client=client, model_id=a.model, shards_dir=shards_dir, - policy=dict(DEFAULT_QUERY_POLICY, - answer_mode="claim_lattice")) + policy=arb_policy) arb_raw = r.get("raw_answer") or r.get("answer_text") or "" arb_mode = r.get("audit_mode") except Exception as e: # noqa: BLE001 diff --git a/bench/control_sweep.py b/bench/control_sweep.py index 8605581..9690113 100644 --- a/bench/control_sweep.py +++ b/bench/control_sweep.py @@ -135,7 +135,8 @@ def _process_item(idx: int, it: dict, variants: list[str], models: list[str], shards_dir: Path, arborist_on: bool, arborist_ref: str, ts: str, judge_fn=None, - skip_solo: bool = False) -> list[dict]: + skip_solo: bool = False, + arborist_answer_mode: str = "claim_lattice") -> list[dict]: """All variants × models for ONE fixture item. Self-contained: its own qa_db, its own clients — safe to run concurrently. @@ -149,7 +150,9 @@ def _process_item(idx: int, it: dict, variants: list[str], if judge_fn is None: judge_fn = _judge_code.judge from arborist.qa.client import OpenAICompatibleClient - from arborist.qa.query import DEFAULT_QUERY_POLICY, query + from arborist.qa.query import query + from bench.stock_v1 import assert_not_drifted as _assert_stock + from bench.stock_v1 import policy_for as _stock_policy_for out: list[dict] = [] q0 = it["question"] @@ -207,8 +210,17 @@ def _process_item(idx: int, it: dict, variants: list[str], # are added inside query() via # claim_lattice_structured_output_extras() and # merge with this per-model extras dict. - arb_policy = dict(DEFAULT_QUERY_POLICY, - answer_mode="claim_lattice") + reasoning_ref = bool(MODELS[arborist_ref].get("reasoning")) + # STOCK V.1 frozen substrate-ON policy for this answer + # mode (bench.stock_v1). Non-reasoning runs are + # drift-guarded — a mid-campaign DEFAULT_QUERY_POLICY + # edit halts the sweep rather than silently redefining + # substrate-ON. Reasoning refs (phase 3) layer the + # documented overrides below and diverge from the pin + # by design, so they skip the assert. + if not reasoning_ref: + _assert_stock(arborist_answer_mode) + arb_policy = _stock_policy_for(arborist_answer_mode) # Reasoning models emit a multi-line trace before # the JSON; the claim_lattice "\n\n" runaway-guard # stop sequence (tuned for single-line Hermes JSON) @@ -220,7 +232,6 @@ def _process_item(idx: int, it: dict, variants: list[str], # suppressed, so output is clean single-line JSON — # but the stop must still be cleared or the first # structural newline truncates it.) - reasoning_ref = bool(MODELS[arborist_ref].get("reasoning")) if reasoning_ref: arb_policy["claim_lattice_json_stop_sequences"] = [] # Reasoning burns 1300-3300 completion tokens on @@ -419,6 +430,12 @@ def main() -> int: ap.add_argument("--variants", default="plain,source_relative,as_of_corpus") ap.add_argument("--arborist-ref", default="hermes") + ap.add_argument("--arborist-answer-mode", + choices=["quote", "claim_lattice"], + default="claim_lattice", + help="STOCK V.1 substrate-ON answer shape for the " + "treatment arm (the two-cell config family). " + "Frozen bench.stock_v1 policy per mode.") ap.add_argument("--out-dir", default="bench/qa_results") ap.add_argument("--report-only", default="", help="aggregate a (partial) JSONL with NO spend " @@ -514,7 +531,8 @@ def main() -> int: futs = { ex.submit(_process_item, i, it, variants, models, shards_dir, i <= arb_units, a.arborist_ref, - ts, judge_fn, a.skip_solo): i + ts, judge_fn, a.skip_solo, + a.arborist_answer_mode): i for i, it in enumerate(items, 1) if i not in skip_items } diff --git a/bench/stock_v1.py b/bench/stock_v1.py index 4038174..b245f6f 100644 --- a/bench/stock_v1.py +++ b/bench/stock_v1.py @@ -2,46 +2,64 @@ """STOCK V.1 — the frozen substrate-ON configuration under test. fox 2026-05-21: before the multi-day GPU campaign (hermes 3090/4090 → -qwen 3090/4090 → reasoning variants), pin exactly ONE substrate-ON -config so the treatment arm cannot silently drift across the run. The -control arms (``bench/control_ab.py`` arm A = bare-model-solo, +qwen 3090/4090 → reasoning variants), pin the substrate-ON config so the +treatment arm cannot silently drift across the run. The control arms +(``bench/control_ab.py`` arm A = bare-model-solo, ``bench/control_sweep.py``) and the substrate-ON arm B (the full -``query()`` pipeline) must all measure the SAME substrate every cell, -or the substrate-OFF-vs-ON delta is meaningless. +``query()`` pipeline) must all measure the SAME substrate every cell, or +the substrate-OFF-vs-ON delta is meaningless. -What "STOCK V.1" is -------------------- -The full merkle-agi-dag reverse-RAG SQD/Prometheus-σ recursive- +Two answer modes (fox 2026-05-21) +--------------------------------- +STOCK V.1 is a **two-cell config family**: the substrate is characterized +under BOTH answer shapes, each frozen with its own governance hash — + + * ``quote`` — model writes prose with inline verbatim quotes; + bench leader on raw lexical grounding. + * ``claim_lattice`` — structured JSON claim-lattice (the merkle-agi-dag + four-rung ladder); what control_ab/control_sweep + already drive, and the only shape with phase-3 + reasoning-variant support built. + +``answer_mode`` is therefore a swept axis, not a single pinned value. +Everything else below is frozen identically across both modes. + +What V.1 is (both modes) +------------------------ +The full merkle-agi-dag reverse-RAG SQD / Prometheus-σ recursive- falsification substrate, **non-reasoning** and **non-distributed**: - * answer_mode = quote (bench leader on raw lexical grounding) - * cross-language guard / sandwich-MT / entity-mask = OFF - (English-only; MT is a separate capability) + * cross-language guard / sandwich-MT / entity-mask = OFF (English-only) * mechanical answer-repair = OFF (one-shot discipline) * quantifier caps = reported, not applied (dry-run) * metacognition preflight = label-only (no verdict gating) * soft-preflight LLM sidecar = OFF (no extra round-trip) * claim ceiling = 12 * verifier content-token rules = v2-acronym-aware - * reasoning (inference layer) = OFF -> reasoning variants are phase 3 - * mesh / multi-witness = OFF -> distributed is the later fork + * reasoning (inference layer) = OFF -> reasoning variants are phase 3 + * mesh / multi-witness = OFF -> distributed is the later fork + +Phase 3 (reasoning) note +------------------------ +Reasoning refs (qwen-think) layer documented overrides onto claim_lattice +inside ``control_sweep`` (cleared JSON stop sequences, 8192-token budget +for the reasoning trace, empty-output self-heal). Those overrides change +the policy, so a reasoning run has a DIFFERENT governance hash by +construction — that is correct (it is a different, phase-3 config). The +V.1 drift guard therefore covers only the non-reasoning frozen base; +reasoning runs skip the assert. How the freeze is enforced -------------------------- -``STOCK_V1_POLICY`` is a deep snapshot of the live +``STOCK_V1_POLICIES[mode]`` is a deep snapshot of the live ``arborist.qa.query.DEFAULT_QUERY_POLICY`` with the load-bearing knobs -re-asserted explicitly (they ARE the current defaults — this is a -freeze, not a change). We then compute ``governance_policy_hash`` over -the WHOLE effective dict (the same hash that lands in every cache_key) -and assert it equals ``STOCK_V1_GOVERNANCE_HASH``. If anyone edits -``DEFAULT_QUERY_POLICY`` mid-campaign — a pinned knob OR any other -field the hash covers — the assert fails LOUDLY and the harness refuses -to run a drifted substrate. The whole 3-4 day run is therefore -identified by one hash. - -This is the drift guard, not value duplication: we do not hand-copy the -big prompt strings (they would rot against source); we snapshot + pin -the hash. +re-asserted explicitly (they ARE the current defaults — a freeze, not a +change) plus the mode's ``answer_mode``. ``assert_not_drifted(mode)`` +hashes the whole effective dict and fails LOUDLY if it no longer matches +the pin — a mid-campaign edit to ``DEFAULT_QUERY_POLICY`` stops the +harness rather than quietly changing what "substrate-ON" means. We do +not hand-copy the big prompt strings (they would rot against source); we +snapshot + pin the hash. """ from __future__ import annotations @@ -52,10 +70,10 @@ from arborist.qa.keys import governance_policy_hash from arborist.qa.query import DEFAULT_QUERY_POLICY #: The knobs that DEFINE substrate-ON V.1, re-asserted explicitly so the -#: intent is readable at a glance. Every value here equals the current +#: intent is readable. Every value equals the current #: ``DEFAULT_QUERY_POLICY`` default — see the module docstring. +#: ``answer_mode`` is NOT here — it is the swept axis (see STOCK_V1_MODES). STOCK_V1_PINS: dict = { - "answer_mode": "quote", "crosslang_guard_enabled": False, "crosslang_translate_enabled": False, "crosslang_entity_mask": False, @@ -68,56 +86,76 @@ STOCK_V1_PINS: dict = { "content_token_rules": "v2-acronym-aware", } -#: Frozen substrate-ON policy: live defaults + explicit pins. -STOCK_V1_POLICY: dict = {**copy.deepcopy(DEFAULT_QUERY_POLICY), **STOCK_V1_PINS} +#: The two frozen answer shapes the campaign characterizes. +STOCK_V1_MODES: tuple[str, ...] = ("quote", "claim_lattice") + + +def policy_for(mode: str) -> dict: + """Frozen substrate-ON policy for one answer mode: defaults + pins.""" + if mode not in STOCK_V1_MODES: + raise ValueError(f"unknown STOCK V.1 mode {mode!r}; " + f"expected one of {STOCK_V1_MODES}") + return { + **copy.deepcopy(DEFAULT_QUERY_POLICY), + **STOCK_V1_PINS, + "answer_mode": mode, + } + + +#: Frozen policies, one per mode. +STOCK_V1_POLICIES: dict = {m: policy_for(m) for m in STOCK_V1_MODES} #: Inference / harness axes — NOT policy fields (they don't fold into -#: governance_policy_hash). Phase 1+2 hold these; phase 3 flips reasoning. +#: governance_policy_hash). Phases 1-2 hold these; phase 3 flips reasoning. STOCK_V1_REASONING = False # reasoning variants are campaign phase 3 STOCK_V1_DISTRIBUTED = False # mesh / multi-witness is the later fork -#: The one hash that identifies the whole campaign. Pinned 2026-05-21 -#: against DEFAULT_QUERY_POLICY at commit b5cc970. Re-pin (with a fox -#: go) only when a deliberate substrate change is intended. -STOCK_V1_GOVERNANCE_HASH = ( - "5b6ca4c5e754e96b7e2e8af16a8948dec8d8b2a80b9304df21b1f6d494aade4e" -) +#: The hashes that identify the campaign — one per non-reasoning mode. +#: Pinned 2026-05-21 against DEFAULT_QUERY_POLICY at commit e227bbc. +#: Re-pin (with a fox go) only when a deliberate substrate change lands. +STOCK_V1_GOVERNANCE_HASHES: dict = { + "quote": "5b6ca4c5e754e96b7e2e8af16a8948dec8d8b2a80b9304df21b1f6d494aade4e", + "claim_lattice": "036a4c79fd9d381a9ebf54094091c5f4e892793ae3b6dc56f70b8d4921c47455", +} -def _live_governance_hash() -> str: - """The governance_policy_hash of the current frozen policy.""" - return governance_policy_hash(STOCK_V1_POLICY) +def assert_not_drifted(mode: str | None = None) -> dict: + """Fail loudly if DEFAULT_QUERY_POLICY drifted from the V.1 pin(s). - -def assert_not_drifted() -> str: - """Fail loudly if DEFAULT_QUERY_POLICY has drifted from the V.1 pin. - - Returns the live hash on success. The campaign harness calls this - before any substrate-ON run so a mid-campaign edit to - DEFAULT_QUERY_POLICY (pinned knob or otherwise) cannot silently - change what "substrate-ON" means. + ``mode=None`` checks every mode; pass a single mode to check one. + Returns ``{mode: live_hash}`` on success. Reasoning (phase-3) runs + intentionally diverge and should NOT call this. The campaign harness + calls this before any non-reasoning substrate-ON run. """ - live = _live_governance_hash() - if STOCK_V1_GOVERNANCE_HASH == "__PIN_ME__": - return live # un-pinned bootstrap: caller prints + bakes the value - if live != STOCK_V1_GOVERNANCE_HASH: - raise SystemExit( - "STOCK V.1 DRIFT: DEFAULT_QUERY_POLICY no longer hashes to the\n" - f" pinned campaign substrate.\n" - f" pinned: {STOCK_V1_GOVERNANCE_HASH}\n" - f" live: {live}\n" - " Either revert the policy change, or (with a fox go) re-pin\n" - " STOCK_V1_GOVERNANCE_HASH and bump the campaign to V.2." - ) + modes = STOCK_V1_MODES if mode is None else (mode,) + live = {} + for m in modes: + h = governance_policy_hash(STOCK_V1_POLICIES[m]) + live[m] = h + pinned = STOCK_V1_GOVERNANCE_HASHES.get(m, "__PIN_ME__") + if pinned == "__PIN_ME__": + continue # un-pinned bootstrap: caller prints + bakes the value + if h != pinned: + raise SystemExit( + f"STOCK V.1 DRIFT ({m}): DEFAULT_QUERY_POLICY no longer " + "hashes to the\n pinned campaign substrate.\n" + f" pinned: {pinned}\n" + f" live: {h}\n" + " Either revert the policy change, or (with a fox go) " + "re-pin\n STOCK_V1_GOVERNANCE_HASHES and bump the " + "campaign to V.2." + ) return live if __name__ == "__main__": import json - print("STOCK V.1 substrate-ON config") - print(f" governance_policy_hash : {_live_governance_hash()}") - print(f" reasoning : {STOCK_V1_REASONING}") - print(f" distributed : {STOCK_V1_DISTRIBUTED}") - print(" explicit pins:") + print("STOCK V.1 substrate-ON config (two-mode family)") + print(f" reasoning : {STOCK_V1_REASONING}") + print(f" distributed : {STOCK_V1_DISTRIBUTED}") + for m in STOCK_V1_MODES: + print(f" governance_policy_hash[{m}] = " + f"{governance_policy_hash(STOCK_V1_POLICIES[m])}") + print(" shared pins:") print(json.dumps(STOCK_V1_PINS, indent=4, sort_keys=True)) diff --git a/docs/stock-v1-config.md b/docs/stock-v1-config.md index 4540a07..c35e61e 100644 --- a/docs/stock-v1-config.md +++ b/docs/stock-v1-config.md @@ -10,9 +10,21 @@ silently drift across a 3-4 day run. Source of truth: The full merkle-agi-dag reverse-RAG SQD / Prometheus-σ recursive- falsification substrate, **non-reasoning** and **non-distributed**. +STOCK V.1 is a **two-cell config family** (fox 2026-05-21): the substrate +is characterized under both answer shapes, so `answer_mode` is a **swept +axis**, each cell frozen with its own governance hash — + +- **`quote`** — prose with inline verbatim quotes; bench leader on raw + lexical grounding (~0.54 strict). +- **`claim_lattice`** — structured JSON claim-lattice (the four-rung + ladder); what `control_ab`/`control_sweep` already drive, and the only + shape with phase-3 reasoning-variant support built (~0.42 strict). + +Everything else below is frozen identically across both modes. + | knob | value | note | |---|---|---| -| `answer_mode` | `quote` | bench leader on raw lexical grounding | +| `answer_mode` | **swept**: `quote` \| `claim_lattice` | two-cell family | | `temperature` / `top_p` / `max_tokens` | `0.1` / `1.0` / `512` | from `DEFAULT_QUERY_POLICY` | | `repair_enabled` | `False` | one-shot discipline, no self-heal reprompts | | crosslang guard / translate / entity-mask | **OFF** | English-only; sandwich-MT is a separate capability | @@ -31,19 +43,32 @@ falsification substrate, **non-reasoning** and **non-distributed**. arm A (question-only, neutral system prompt; gold text supplied identically to both arms per the §4b ruling). -## The one hash +## The hashes (one per mode) ``` -governance_policy_hash = 5b6ca4c5e754e96b7e2e8af16a8948dec8d8b2a80b9304df21b1f6d494aade4e +governance_policy_hash[quote] = 5b6ca4c5e754e96b7e2e8af16a8948dec8d8b2a80b9304df21b1f6d494aade4e +governance_policy_hash[claim_lattice] = 036a4c79fd9d381a9ebf54094091c5f4e892793ae3b6dc56f70b8d4921c47455 ``` -This identifies the whole campaign. `bench/stock_v1.py` snapshots the -live `DEFAULT_QUERY_POLICY`, re-asserts the load-bearing pins, hashes -the whole effective dict, and `assert_not_drifted()` fails **loudly** if -that hash ever changes — a mid-campaign edit to `DEFAULT_QUERY_POLICY` -(a pinned knob *or any other field the hash covers*) stops the harness -rather than quietly changing what "substrate-ON" means. Re-pinning is a -deliberate fox-gated bump to V.2, never silent. +These identify the campaign. `bench/stock_v1.py` snapshots the live +`DEFAULT_QUERY_POLICY`, re-asserts the load-bearing pins, sets the mode's +`answer_mode`, hashes the whole effective dict, and +`assert_not_drifted(mode)` fails **loudly** if that hash ever changes — +a mid-campaign edit to `DEFAULT_QUERY_POLICY` (a pinned knob *or any +other field the hash covers*) stops the harness rather than quietly +changing what "substrate-ON" means. Re-pinning is a deliberate fox-gated +bump to V.2, never silent. + +**Phase 3 (reasoning) note.** Reasoning refs (qwen-think) layer +documented overrides onto `claim_lattice` inside `control_sweep` (cleared +JSON stop sequences, 8192-token budget for the reasoning trace, +empty-output self-heal). Those overrides change the policy, so a +reasoning run has a *different* governance hash by construction — correct, +since it is a different phase-3 config. The V.1 drift guard therefore +covers only the non-reasoning frozen base; reasoning runs skip the assert. + +Harnesses select the mode via `--answer-mode` (`control_ab`) / +`--arborist-answer-mode` (`control_sweep`), default `claim_lattice`. ## Campaign matrix