diff --git a/arborist/substrate/anchor_prg.py b/arborist/substrate/anchor_prg.py index d29ec04..fc99b37 100644 --- a/arborist/substrate/anchor_prg.py +++ b/arborist/substrate/anchor_prg.py @@ -26,16 +26,26 @@ matches the substrate's SHA-256 hard-hash family. See ticket #000035 §2.1 for the full reasoning, §2.2 for why this construction won over AES-256-CTR and ChaCha20. -Hard rules (per #000035 §2.4): +Hard rules (per #000035 §2.4 + §3.4, dav1d-reviewed 2026-05-11): -- Seed is **published** (committed in the v7 boot manifest as - ``phi_prg_seed``); secrecy is NOT the security property. The +- The seed is committed in the v7 boot manifest as + ``anchor_prg_seed`` (purpose-scoped name; the function argument + here is just ``seed`` / ``phi_prg_seed``-as-local-alias). It is + **published** — secrecy is NOT the security property; the property is computational indistinguishability of the OUTPUT from - random, which holds even when the seed is public. + random, which holds even with a public seed. +- The seed MUST be generated independently of the model checkpoint + and training data, and MUST NOT be adversary-selected. It is + **single-purpose** — never reuse ``anchor_prg_seed`` for any + other PRG domain (no domain-separation tag in v1, so reuse would + break the PRF argument). - Per-checkpoint seed rotation is the M2 mitigation, orthogonal to this module — leave it to the v7 manifest layer. - 32-byte hard-hash input matches the substrate's SHA-256 surface; shorter inputs raise ``ValueError`` rather than silently padding. +- ``dim_h ≤ 16 · 2^32``: a 4-byte counter admits 2^32 HMAC-SHA-512 + blocks of 64 bytes each (= 16 float32-words). A larger request + raises ``ValueError`` rather than overflowing the counter. """ from __future__ import annotations @@ -50,8 +60,13 @@ import hmac # a string raises a clear TypeError at hmac.new() time. PHI_PRG_VERSION = "phi-prg-v1-hmac-sha512" +# Maximum dim_h: a 4-byte counter admits 2^32 HMAC-SHA-512 blocks of +# 64 bytes = 16 float32-words each (#000035 §3.4). Astronomically above +# real deployments, but the guard makes the construction total. +_MAX_DIM_H = 16 * (1 << 32) + # Placeholder seed for tests + KAT generation. Replaced at deployment -# time with the v7 manifest's ``phi_prg_seed`` field. The placeholder +# time with the v7 manifest's ``anchor_prg_seed`` field. The placeholder # is 32 bytes so it matches the deployment shape; the value itself is # the SHA-256 of a fixed string for reproducibility, NOT a security # claim. Callers in production code path should pass their own seed. @@ -76,22 +91,28 @@ def phi_prg( accepting shorter inputs would silently pad and break the PRF security argument. dim_h - Length of the output anchor vector. Must be a positive int. + Length of the output anchor vector. Must be a positive int + with ``dim_h ≤ 16 · 2^32`` (the 4-byte-counter ceiling). seed - Published HMAC key. Defaults to ``PLACEHOLDER_SEED``; - deployment code must override with the v7 manifest seed. + Published HMAC key — the v7 manifest's ``anchor_prg_seed``. + Defaults to ``PLACEHOLDER_SEED``; deployment code must + override with the committed manifest seed. Must be generated + independently of the model and not adversary-selected + (#000035 §3.4); single-purpose — do not reuse for other PRGs. Returns ------- list[float] - ``dim_h`` floats uniformly distributed on ``[-1, 1)``, + ``dim_h`` floats uniform over a 2^32-point grid in + ``[-1, 1)`` (``-1.0`` reachable, ``+1.0`` not; finite-grid + mean ``−2^−32`` — negligible, but not exactly zero), deterministically derived from ``(seed, hard_hash_32)``. Raises ------ ValueError - If ``hard_hash_32`` is not exactly 32 bytes, or if - ``dim_h`` is not a positive integer. + If ``hard_hash_32`` is not exactly 32 bytes, or ``dim_h`` + is not a positive integer, or ``dim_h > 16 · 2^32``. """ if not isinstance(hard_hash_32, (bytes, bytearray)) or len(hard_hash_32) != 32: raise ValueError( @@ -100,8 +121,13 @@ def phi_prg( if isinstance(hard_hash_32, (bytes, bytearray)) else f"got {type(hard_hash_32).__name__}" ) - if not isinstance(dim_h, int) or dim_h <= 0: + if not isinstance(dim_h, int) or isinstance(dim_h, bool) or dim_h <= 0: raise ValueError(f"dim_h must be a positive int; got {dim_h!r}") + if dim_h > _MAX_DIM_H: + raise ValueError( + f"dim_h must be ≤ 16·2^32 = {_MAX_DIM_H} (4-byte counter " + f"ceiling, #000035 §3.4); got {dim_h}" + ) raw = _expand(seed, bytes(hard_hash_32), dim_h * 4) return _bytes_to_floats(raw) @@ -112,10 +138,16 @@ def _expand(seed: bytes, hard_hash: bytes, n_bytes: int) -> bytes: Block size is the HMAC-SHA-512 output (64 bytes); we ceil-divide to the smallest counter range that yields ``n_bytes`` output, then - truncate the last block. Counter is big-endian 4-byte unsigned; - overflow at 2^32 - 1 blocks (i.e. 256 GB output) raises - ``OverflowError`` from ``int.to_bytes`` rather than silently - wrapping. Production dim_h won't approach that. + truncate the last block. Counter is big-endian 4-byte unsigned, + starting at 0. The ``dim_h ≤ 16·2^32`` guard in ``phi_prg`` keeps + the counter from exhausting; if a bug ever drove it past 2^32 + blocks, ``int.to_bytes(4, 'big')`` would raise ``OverflowError`` + rather than silently wrapping. + + **Endianness note (#000035 §3.4).** Big-endian here is pinned to + the v7 spec amendment and the 10 KAT vectors. If v7's canonical + TLV integer encoding turns out to be little-endian, this flips — + but only with a ``PHI_PRG_VERSION`` bump + regenerated KATs. """ out = bytearray() counter = 0 @@ -129,9 +161,15 @@ def _expand(seed: bytes, hard_hash: bytes, n_bytes: int) -> bytes: def _bytes_to_floats(raw: bytes) -> list[float]: """Map each big-endian uint32 to a float in [-1, 1). - Per #000035 §2.3: ``f(u32) = 2 * (u32 / 2**32) - 1``. Distribution - is uniform on ``[-1, 1)`` modulo 2^-32 quantization, which is well - below any downstream precision the anchor vector cares about. + Per #000035 §2.3 / §3.4: ``f(u32) = 2 * (u32 / 2**32) - 1``. + Output is uniform over a 2^32-point grid in ``[-1, 1)`` with step + ``2^-31``: ``f(0) = -1.0`` (reachable), ``f(2^32-1) = 1 - 2^-31`` + (so ``+1.0`` is never reached). The finite-grid mean is ``−2^−32`` + — negligible for the anchor-map use case, but **not exactly + zero-mean**. If exact zero-mean is ever required, switch to the + midpoint map ``f(u32) = 2*((u32+0.5)/2^32) - 1`` (range ``(-1, 1)``, + exact zero mean) with a ``PHI_PRG_VERSION`` bump + new KATs — + never change the formula silently. """ n = len(raw) // 4 if len(raw) != n * 4: diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 965088a..413f5c3 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -102,7 +102,7 @@ Newest first. Update on every open/close. | #000038 | Phase 4 content acquisition — proprietary textbook license decisions for warrant coverage | closed · obviated 2026-05-10 by alias-substitution sprint under #000031 (74 rows in #000041 + 13 rows in #000042); 92/92 records now resolve. Residue (multilingual PD, Hilbert-Ackermann OCR, Knuth permission, personal-copy path B) preserved as design log §8 | 2026-05-09 | — | | #000037 | Prometheus-Σ recursive falsification controller (bicameral substrate) | in progress · Phases 0 + 1 + 1.b + 1.c + 2 landed 2026-05-10; **§12 Trigger 2 fired** (divergence variance 0.575 / N=37); §22 Findings 2 + 3 RESOLVED (kernel/llm cost split + sweep_weights §15.4 + per-mode τ_qa); `controller_events` carries 4 event kinds (decision · difficulty · budget_allocation · falsification_proposal) feeding `arborist controller-events` inspector + live-harvest third bucket in `bench/scripts/harvest_falsification_proposals.py`; §12 Trigger 1 probe wired 2026-05-11 (`trigger_1_branch_density` reads `fork_score_branches` — measurable, not yet fired); Phase 3 sleep-sweep scheduler tracked under #000045 (gating ticket) | 2026-05-09 | — | | #000036 | T3 per-window covert-channel budget bound | in progress · Phase 1 + dav1d review 2026-05-11 → Tier-1 + Tier-2 (Option B = `b1_model=max_envelope` default, applied *in v1*, no v2 fork) both landed 2026-05-11; KAT regenerated (12 entries, active); baseline 625.87 → 6183.02 (max_envelope); 53 → 83 tests; both prior closure blockers cleared — remaining = fox final close-or-iterate call | 2026-05-09 | — | -| #000035 | PRG choice for φ_PRG (HMAC-SHA-512 expansion) | in progress · Phase 1 landed 2026-05-10; v7 §9.10 amendment awaits maintainer review | 2026-05-09 | — | +| #000035 | PRG choice for φ_PRG (HMAC-SHA-512 expansion) | in progress · Phase 1 landed 2026-05-10; §9.10 amendment text **dav1d-reviewed final 2026-05-11** (manifest field → `anchor_prg_seed`; float-map prose corrected to "negligible mean −2^−32" not "unbiased"; `dim_h ≤ 16·2^32` guard + bool-reject added to impl; seed-independence + M1-policy-separation added; HMAC-SHA-512 / 32-byte / uint32-be / SHALL all LOCKED). Remaining: confirm v7 integer endianness + land §9.10 into v7 spec when plastic-training has a deployment target | 2026-05-09 | — | | #000034 | Hessian alignment under φ_linear | in progress · Phase 1a landed 2026-05-10 (synthetic-ablation probe + KAT fixture); Phase 1b parks for v7 ramp-up | 2026-05-09 | — | | #000033 | Claim-pack pillar VII (combinatorics) | closed · landed 2026-05-09 (live in shard 000.db; lift verified) | 2026-05-09 | — | | #000032 | combinatorics@v1 π* (integer counting kernel) | closed · landed 2026-05-09 | 2026-05-09 | — | diff --git a/docs/soft-hash-channel-analysis.md b/docs/soft-hash-channel-analysis.md index 984d265..288390d 100644 --- a/docs/soft-hash-channel-analysis.md +++ b/docs/soft-hash-channel-analysis.md @@ -417,7 +417,7 @@ The reduction in §4 leaves three loose threads: **Phase 1 landed 2026-05-10** under #000035: reference implementation at `arborist/substrate/anchor_prg.py` (HMAC-SHA-512 counter-mode KDF, pure stdlib — `hashlib` + - `hmac`, no third-party crypto dep). 27 tests in + `hmac`, no third-party crypto dep). 30 tests in `tests/test_anchor_prg.py` covering determinism, range invariants, chi² uniformity, dim_h boundary (1, 16384), seed-bit-flip and hash-bit-flip avalanche, hand-computed @@ -430,10 +430,25 @@ The reduction in §4 leaves three loose threads: variants, and a 4096-element counter-rollover stress sample. Module exports `PHI_PRG_VERSION = "phi-prg-v1-hmac-sha512"` so future algorithm rotation is detectable at the call site - without string-comparing module paths. Spec amendment text - drafted at #000035 §3.4; Phase 2 lands it into a v7 - plastic-training spec when one gains an active deployment - target. + without string-comparing module paths. + + **dav1d-reviewed §9.10 spec amendment text (2026-05-11)** — + #000035 §3.4 holds the final wording. Decisions: HMAC-SHA-512 + LOCK; **manifest field `anchor_prg_seed`** (purpose-scoped; + `phi_prg_seed` is a code-local alias only); 32-byte seed LOCK; + `uint32` big-endian counter from 0 LOCK (flip to little-endian + only if v7's TLV convention already is); float map + `2·(u32/2^32)−1` kept for KAT compatibility but the prose now + says "uniform over a 2^32-point grid in [-1, 1) with negligible + finite-grid mean −2^−32" rather than "unbiased"; `SHALL` + replace `embed_hard_to_vec` when M1 enabled (M1 enablement + itself is a separate mitigation-selection-policy question, with + the NO_ALIGNMENT-skip rule documented in §9.10.1); plus a + `dim_h ≤ 16·2^32` exhaustion guard and a seed-independence / + single-purpose-seed requirement. The reference implementation + + tests + KAT vectors are pinned to these choices. Phase 2 lands + the §9.10 text into the v7 plastic-training spec when one gains + an active deployment target. 3. **T3 per-window budget formal bound.** ~~§6 lists "bounded by per-window budget" without giving the bound.~~ **Closed @@ -470,7 +485,8 @@ deployment data per #000034 Phase 1b for the actual checkpoint measurement), §9.2 (PRG cryptographic strength for `φ_PRG`: decision pinned + Phase 1 reference implementation landed 2026-05-10 per #000035 — -HMAC-SHA-512 with 32-byte committed seed, KAT-pinned; awaits v7 +HMAC-SHA-512 with 32-byte committed `anchor_prg_seed`, KAT-pinned; +dav1d-reviewed §9.10 wording final 2026-05-11 (#000035 §3.4); awaits v7 §9.10 spec amendment when v7 plastic-training spec gains an active deployment target). §9.3 closed 2026-05-10 via the T3 per-window bound at ``docs/soft-hash-channel-t3-bound.md`` diff --git a/docs/tickets/ticket-000006-bench-emergent-findings.md b/docs/tickets/ticket-000006-bench-emergent-findings.md index b46bfb7..f2cb4c7 100644 --- a/docs/tickets/ticket-000006-bench-emergent-findings.md +++ b/docs/tickets/ticket-000006-bench-emergent-findings.md @@ -731,7 +731,7 @@ Two cryptographic-primitive Phase 1 deliverables landed `bench/fixtures/phi-alignment/synthetic-checkpoints.jsonl`. - **#000035 Phase 1** (earlier today): φ_PRG reference impl at `arborist/substrate/anchor_prg.py` (HMAC-SHA-512 counter-mode - KDF) + 27 tests (was 20; +7 from `de997f7` 2026-05-10 pattern + KDF) + 30 tests (was 20; +7 from `de997f7` 2026-05-10 pattern backfill — prefix-extension closure invariant, hand-computed HMAC-SHA-512 first-block formula, parametrized invalid-input cones) + 10-vector KAT fixture at diff --git a/docs/tickets/ticket-000035-prg-choice-phi-prg.md b/docs/tickets/ticket-000035-prg-choice-phi-prg.md index 23fede8..d50c18e 100644 --- a/docs/tickets/ticket-000035-prg-choice-phi-prg.md +++ b/docs/tickets/ticket-000035-prg-choice-phi-prg.md @@ -1,6 +1,6 @@ # Ticket #000035 — PRG choice for φ_PRG -**Status:** in progress · Phase 1 landed 2026-05-10; v7 §9.10 spec amendment text pinned in §3.4 below, awaits v7 spec maintainer review +**Status:** in progress · Phase 1 landed 2026-05-10; **§9.10 spec amendment text dav1d-reviewed final 2026-05-11** (§3.4 below holds the folded version). Review decisions applied: manifest field renamed `phi_prg_seed → anchor_prg_seed` (purpose-scoped; `phi_prg_seed` kept as a code-local alias); float-map prose corrected ("uniform over a 2^32-point grid in [-1, 1) with negligible mean −2^−32", not "unbiased" — formula unchanged for KAT compat); `dim_h ≤ 16·2^32` exhaustion guard added to `arborist/substrate/anchor_prg.py` (+ bool-`dim_h` reject); seed-independence / single-purpose-seed rule added; M1-enablement-policy separation (§9.10.1: skipping M1 governed by mitigation-selection policy, e.g. NO_ALIGNMENT per #000034, not by §9.10). HMAC-SHA-512 / 32-byte seed / uint32-be counter / `SHALL`-replace all LOCKED. Remaining: confirm v7's canonical integer endianness (big-endian assumed) before the §9.10 text freezes into the v7 spec; the actual landing-into-v7-spec step still parks on v7 plastic-training gaining a deployment target. **Opened:** 2026-05-09 **Scope:** Pin a specific cryptographic PRG construction for the M1 mitigation (PRG-based anchor map) proposed in #000018 §5.2 + @@ -192,23 +192,85 @@ regression coverage. Generated once, pinned. ### 3.4 Spec amendment -v7 § 9.10 amendment text: +v7 § 9.10 amendment text — **dav1d-reviewed final (2026-05-11)**. +Decision set (`RESPONSE` 2026-05-11): HMAC-SHA-512 — LOCK; manifest +field `anchor_prg_seed` (purpose-scoped, not implementation-scoped +— `phi_prg_seed` is an acceptable code-local alias only); seed +size 32 bytes — LOCK; counter `uint32` big-endian from 0 — LOCK +(flip to little-endian only if v7's TLV convention already is); +float map `2·(u32/2^32)−1` kept for KAT compatibility but the +prose must say "uniform over a 2^32-point grid in [-1, 1) with +negligible finite-grid mean −2^−32", **not** "unbiased"; `SHALL` +replace `embed_hard_to_vec` when M1 is enabled (whether M1 itself +is mandatory is a separate mitigation-selection-policy question). +Plus an explicit `dim_h ≤ 16·2^32` exhaustion guard and a +seed-independence requirement. ``` -The anchor map φ_PRG(C(M), dim_h) is defined as follows: - Let SEED be the v7 boot manifest's `phi_prg_seed` field - (32-byte committed value). - Let counter range over big-endian 4-byte integers from 0. - Output := concatenate HMAC-SHA-512(SEED, C(M) ‖ counter) - until ≥ dim_h * 4 bytes; truncate. - Convert each successive 4 bytes to a float via - 2 * (uint32_be / 2^32) - 1. - The dim_h-vector is the resulting list. +§ 9.10 Anchor PRG map φ_PRG -φ_PRG SHALL replace the v7 reference linear projection -embed_hard_to_vec when the M1 mitigation is enabled. +When M1 is enabled, the v7 anchor map φ_PRG(C(M), dim_h) is defined +as follows. + +Let anchor_prg_seed be the v7 boot manifest's dedicated 32-byte PRG +seed. The seed MUST be generated independently of the model +checkpoint and training data, and MUST NOT be adversary-selected. +The seed MAY be public once committed, but it MUST be committed +before the corresponding anchor map is evaluated. anchor_prg_seed +is single-purpose — it MUST NOT be reused for any other PRG domain. + +Let C(M) be the fixed-length committed model digest. Let counter +range over unsigned 4-byte big-endian integers starting at 0. + +For counter = 0, 1, 2, … compute: + + block_counter = HMAC-SHA-512(anchor_prg_seed, C(M) ‖ counter_be32) + +Concatenate successive blocks until at least dim_h · 4 bytes are +available; truncate to exactly dim_h · 4 bytes. A 4-byte counter +admits 2^32 HMAC-SHA-512 blocks of 64 bytes each, so dim_h MUST +satisfy dim_h ≤ 16 · 2^32; a request beyond that is an error. + +Partition the byte stream into dim_h successive 4-byte words. +Interpret each word as an unsigned big-endian integer u32. Map +each u32 to a float by: + + x = 2 · (u32 / 2^32) − 1 + +The resulting dim_h-vector is φ_PRG(C(M), dim_h). + +This maps uniformly onto a 2^32-point grid in [-1, 1). The value +-1.0 is reachable; +1.0 is not. The finite-grid mean is −2^−32, +which is negligible for the anchor-map use case. If exact +zero-mean sampling is ever required, the formula MUST be +version-bumped (e.g. to the midpoint map x = 2·((u32+0.5)/2^32)−1) +rather than silently changed, and new KAT vectors emitted. + +When M1 is enabled, φ_PRG SHALL replace the v7 reference linear +projection embed_hard_to_vec for the hard-anchor-to-vector map. +An operator MUST NOT claim M1 while continuing to use +embed_hard_to_vec. + +§ 9.10.1 M1 enablement policy (non-normative pointer) + +Whether a given deployment MUST enable M1 is decided by the +mitigation-selection policy, not by this section. M1 enablement +MAY be skipped only under an explicit policy rule — for example +when a current #000034 φ/Hessian alignment probe returns +NO_ALIGNMENT and the deployment policy accepts the residual risk. +Once M1 is enabled, § 9.10 above applies in full. ``` +**Note on counter endianness.** §3.4 currently fixes big-endian. +Before the §9.10 amendment freezes into the v7 spec, confirm v7's +canonical integer-byte-order convention: if v7 TLV uses +little-endian, flip §3.4 to match; if v7 has no established +convention, big-endian as specified here becomes the canonical +choice for this PRG. The reference implementation +(`arborist/substrate/anchor_prg.py`) and the 10 KAT vectors are +pinned to big-endian; a flip would require a `PHI_PRG_VERSION` +bump + regenerated KATs. + ## 4. Out of scope - v7 spec full revision; this ticket lands the §9.10 amendment @@ -271,7 +333,7 @@ the moment v7 needs it. from any version number. Acceptance-criteria §5 item 1 reads through to "or equivalent location" so the move doesn't invalidate the original criterion. -- ``tests/test_anchor_prg.py`` — 27 tests covering determinism, +- ``tests/test_anchor_prg.py`` — 30 tests covering determinism, range invariants, chi² loose-uniformity sanity, dim_h boundary (1, 16384), seed-bit-flip avalanche, hash-bit-flip avalanche, hand-computed HMAC-SHA-512 first-block formula, prefix-extension diff --git a/docs/tickets/ticket-000044-autocount-doc-drift-discipline.md b/docs/tickets/ticket-000044-autocount-doc-drift-discipline.md index fc90ef0..517c691 100644 --- a/docs/tickets/ticket-000044-autocount-doc-drift-discipline.md +++ b/docs/tickets/ticket-000044-autocount-doc-drift-discipline.md @@ -82,7 +82,7 @@ because the prose IS the inside-of-tag content. ### 3.1 `tests` ```markdown -27 +30 ``` Live value: `pytest --collect-only -q ` count, with diff --git a/docs/warrant-substrate-cookbook.md b/docs/warrant-substrate-cookbook.md index febc4db..3fc5566 100644 --- a/docs/warrant-substrate-cookbook.md +++ b/docs/warrant-substrate-cookbook.md @@ -589,7 +589,7 @@ than waiting for bench-time STRICT-rate drift to surface it. ### Substrate-paper-spec'd primitives (#000012 + #000018 + #000034 + #000037) -- `tests/test_anchor_prg.py` — 27 tests for φ_PRG HMAC-SHA-512 +- `tests/test_anchor_prg.py` — 30 tests for φ_PRG HMAC-SHA-512 expansion (#000035 Phase 1). Covers KAT regression, hand-formula (first-block matches direct ``hmac.new``), prefix-extending closure invariant, output-length monotonicity per dim_h. diff --git a/tests/test_anchor_prg.py b/tests/test_anchor_prg.py index cc6bdbf..3844a9f 100644 --- a/tests/test_anchor_prg.py +++ b/tests/test_anchor_prg.py @@ -306,6 +306,28 @@ def test_phi_prg_rejects_non_int_dim_h(): phi_prg(h, dim_h=8.5) # type: ignore[arg-type] +@pytest.mark.parametrize("bad_dim", [True, False]) +def test_phi_prg_rejects_bool_dim_h(bad_dim): + """isinstance(True, int) is True in Python — bool must be rejected + explicitly so `dim_h=True` doesn't silently become dim_h=1 + (dav1d review 2026-05-11).""" + h = hashlib.sha256(b"x").digest() + with pytest.raises(ValueError, match="positive"): + phi_prg(h, dim_h=bad_dim) # type: ignore[arg-type] + + +def test_phi_prg_rejects_dim_h_above_counter_ceiling(): + """dim_h > 16·2^32 exhausts the 4-byte counter; reject with a + clear ValueError naming the ceiling rather than overflowing + deep in _expand (#000035 §3.4, dav1d review 2026-05-11).""" + h = hashlib.sha256(b"x").digest() + ceiling = 16 * (1 << 32) + # at the ceiling is fine to *request* (we don't actually compute it + # — that would need 256 GB; just check the boundary classification) + with pytest.raises(ValueError, match=r"16.2\^32|counter"): + phi_prg(h, dim_h=ceiling + 1) + + # ----------------------------------------------------------- module shape