diff --git a/arborist/substrate/anchor_prg.py b/arborist/substrate/anchor_prg.py index fc99b37..30a16ae 100644 --- a/arborist/substrate/anchor_prg.py +++ b/arborist/substrate/anchor_prg.py @@ -12,13 +12,21 @@ Construction — HMAC-SHA-512 in NIST SP 800-108 KDF counter mode: i = 0 out = b"" while len(out) < n_bytes: - out += HMAC-SHA-512(SEED, C(M) || i.to_bytes(4, 'big')) + out += HMAC-SHA-512(SEED, C(M) || i.to_bytes(4, 'little')) i += 1 return out[:n_bytes] - Float conversion (uint32_be → [-1, 1)): + Float conversion (uint32_le → [-1, 1)): f(u32) := 2 * (u32 / 2**32) - 1 +Byte order is **little-endian** throughout — matching v7's canonical +TLV encoding (verified 2026-05-11 against ``merkle-agi-dag_v7.txt`` +§A1: TLV length prefixes ``to_bytes(4, "little")``, ``enc_int`` is +``to_bytes(8, "little")``, tensors are `` bytes: Block size is the HMAC-SHA-512 output (64 bytes); we ceil-divide to the smallest counter range that yields ``n_bytes`` output, then - truncate the last block. Counter is big-endian 4-byte unsigned, - starting at 0. The ``dim_h ≤ 16·2^32`` guard in ``phi_prg`` keeps - the counter from exhausting; if a bug ever drove it past 2^32 - blocks, ``int.to_bytes(4, 'big')`` would raise ``OverflowError`` - rather than silently wrapping. - - **Endianness note (#000035 §3.4).** Big-endian here is pinned to - the v7 spec amendment and the 10 KAT vectors. If v7's canonical - TLV integer encoding turns out to be little-endian, this flips — - but only with a ``PHI_PRG_VERSION`` bump + regenerated KATs. + truncate the last block. Counter is **little-endian** 4-byte + unsigned, starting at 0 — matching v7's TLV integer encoding + (#000035 §3.4, dav1d review 2026-05-11). The ``dim_h ≤ 16·2^32`` + guard in ``phi_prg`` keeps the counter from exhausting; if a bug + ever drove it past 2^32 blocks, ``int.to_bytes(4, 'little')`` + would raise ``OverflowError`` rather than silently wrapping. """ out = bytearray() counter = 0 while len(out) < n_bytes: - msg = hard_hash + counter.to_bytes(4, "big") + msg = hard_hash + counter.to_bytes(4, "little") out += hmac.new(seed, msg, hashlib.sha512).digest() counter += 1 return bytes(out[:n_bytes]) def _bytes_to_floats(raw: bytes) -> list[float]: - """Map each big-endian uint32 to a float in [-1, 1). + """Map each little-endian uint32 to a float in [-1, 1). - Per #000035 §2.3 / §3.4: ``f(u32) = 2 * (u32 / 2**32) - 1``. - Output is uniform over a 2^32-point grid in ``[-1, 1)`` with step - ``2^-31``: ``f(0) = -1.0`` (reachable), ``f(2^32-1) = 1 - 2^-31`` - (so ``+1.0`` is never reached). The finite-grid mean is ``−2^−32`` - — negligible for the anchor-map use case, but **not exactly - zero-mean**. If exact zero-mean is ever required, switch to the - midpoint map ``f(u32) = 2*((u32+0.5)/2^32) - 1`` (range ``(-1, 1)``, - exact zero mean) with a ``PHI_PRG_VERSION`` bump + new KATs — - never change the formula silently. + Per #000035 §2.3 / §3.4: ``f(u32) = 2 * (u32 / 2**32) - 1``. Words + are read **little-endian** to match v7's TLV integer convention + (dav1d review 2026-05-11). Output is uniform over a 2^32-point grid + in ``[-1, 1)`` with step ``2^-31``: ``f(0) = -1.0`` (reachable), + ``f(2^32-1) = 1 - 2^-31`` (so ``+1.0`` is never reached). The + finite-grid mean is ``−2^−32`` — negligible for the anchor-map use + case, but **not exactly zero-mean**. If exact zero-mean is ever + required, switch to the midpoint map ``f(u32) = 2*((u32+0.5)/2**32) + - 1`` (range ``(-1, 1)``, exact zero mean) with a ``PHI_PRG_VERSION`` + bump + new KATs — never change the formula silently. """ n = len(raw) // 4 if len(raw) != n * 4: @@ -180,7 +185,7 @@ def _bytes_to_floats(raw: bytes) -> list[float]: floats: list[float] = [] inv_2_32 = 1.0 / (1 << 32) for i in range(n): - u32 = int.from_bytes(raw[4 * i : 4 * (i + 1)], "big") + u32 = int.from_bytes(raw[4 * i : 4 * (i + 1)], "little") floats.append(2.0 * (u32 * inv_2_32) - 1.0) return floats diff --git a/bench/fixtures/phi-prg/known-answer-tests.jsonl b/bench/fixtures/phi-prg/known-answer-tests.jsonl index 6e351cd..a579035 100644 --- a/bench/fixtures/phi-prg/known-answer-tests.jsonl +++ b/bench/fixtures/phi-prg/known-answer-tests.jsonl @@ -1,16 +1,17 @@ -# arborist v7 phi_prg known-answer tests — version phi-prg-v1-hmac-sha512 +# arborist v7 phi_prg known-answer tests — version phi-prg-v1-hmac-sha512-le # Pinned (seed, hard_hash, dim_h) → SHA-256 of raw byte output -# (HMAC-SHA-512 counter-mode expansion before float conversion). -# Algorithm change MUST bump PHI_PRG_VERSION and create a new -# fixture file; do not overwrite — old runs replay against old data. - -{"label": "placeholder-seed/zero-hash/dim_h=1", "seed_hex": "ef532720a49159beb6816d98e13a162bac63c531b631bd1adb0fcca96b467ff3", "hard_hash_hex": "0000000000000000000000000000000000000000000000000000000000000000", "dim_h": 1, "output_sha256": "93618e085f1afae3368cabb57b328f2f01a81cb45c768e02264b13d5ec52732a", "output_bytes": 4} -{"label": "placeholder-seed/zero-hash/dim_h=8", "seed_hex": "ef532720a49159beb6816d98e13a162bac63c531b631bd1adb0fcca96b467ff3", "hard_hash_hex": "0000000000000000000000000000000000000000000000000000000000000000", "dim_h": 8, "output_sha256": "b7e7af7180105100e94fcb4361799e2820d78f9cf2636643dea2a367ea16beb8", "output_bytes": 32} -{"label": "placeholder-seed/zero-hash/dim_h=32", "seed_hex": "ef532720a49159beb6816d98e13a162bac63c531b631bd1adb0fcca96b467ff3", "hard_hash_hex": "0000000000000000000000000000000000000000000000000000000000000000", "dim_h": 32, "output_sha256": "537bf81bdb0bc6300ffe9e9853ecf42a19ee4ea10be11d76624dd83d5528d55a", "output_bytes": 128} -{"label": "placeholder-seed/all-ones-hash/dim_h=16", "seed_hex": "ef532720a49159beb6816d98e13a162bac63c531b631bd1adb0fcca96b467ff3", "hard_hash_hex": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "dim_h": 16, "output_sha256": "2466a75eda007980acea20ed9f1d8676c36700b6da15220455268151a0f6072c", "output_bytes": 64} -{"label": "seed=A/hash=B/dim_h=64", "seed_hex": "14ab2dab0d3cddeaa58ec70632d3ff4f5de2c514004a92144e260dfe384d912e", "hard_hash_hex": "a23cb10b94660f062b467f313a9dc9d84f2dc2748c3627c661082dda7f55e3bb", "dim_h": 64, "output_sha256": "c6b107e73d4b4f23f3ee8991ab953dc4ecc5af35405bf5a97ce66a0fd41c4245", "output_bytes": 256} -{"label": "seed=A/hash=B'/dim_h=64 (one-bit-flip from prior)", "seed_hex": "14ab2dab0d3cddeaa58ec70632d3ff4f5de2c514004a92144e260dfe384d912e", "hard_hash_hex": "223cb10b94660f062b467f313a9dc9d84f2dc2748c3627c661082dda7f55e3bb", "dim_h": 64, "output_sha256": "a29cbf930ae89cd10c816fc5dd03b97354ee0a303e62e92822f9331c42b94f96", "output_bytes": 256} -{"label": "seed=A'/hash=B/dim_h=64 (one-bit-flip seed)", "seed_hex": "94ab2dab0d3cddeaa58ec70632d3ff4f5de2c514004a92144e260dfe384d912e", "hard_hash_hex": "a23cb10b94660f062b467f313a9dc9d84f2dc2748c3627c661082dda7f55e3bb", "dim_h": 64, "output_sha256": "ce1b60357bbd3354f65a16c1e51bd12de186bbb7765a728fae04fdbb09973f31", "output_bytes": 256} -{"label": "block-boundary/dim_h=16", "seed_hex": "cfd60c2bda64ebcefbb23a5b28d98269c9c4f8b8ac77f6f9ca7a0f4865b10f58", "hard_hash_hex": "724cd966a7bfe78ba802877510ffb90c67f385a1d3135e4e1b8a1b38f744c6da", "dim_h": 16, "output_sha256": "75d1eb90b4d385b8d475fa53539eac129b0abfab9e784c0f3be5cf2738c40dec", "output_bytes": 64} -{"label": "block-boundary/dim_h=17", "seed_hex": "cfd60c2bda64ebcefbb23a5b28d98269c9c4f8b8ac77f6f9ca7a0f4865b10f58", "hard_hash_hex": "724cd966a7bfe78ba802877510ffb90c67f385a1d3135e4e1b8a1b38f744c6da", "dim_h": 17, "output_sha256": "3d9226cff6da50e5610cceba54b191223b2dfd5bbbdd3bd0f807062762fb3827", "output_bytes": 68} -{"label": "stress/dim_h=4096", "seed_hex": "0ddd62c311f88ebe2d4f6cd5d9d1374474dfd645e012043648dd966a71785c95", "hard_hash_hex": "e605ede3d9d0d13c6d7d32c5c424b998677eef0689a0d9f0fa4ebd1bb4307cb9", "dim_h": 4096, "output_sha256": "5072d05b17eb4f4b3356bfc66f772330337dfcc1b3bb8fa87fc0ae0568e9a387", "output_bytes": 16384} +# (HMAC-SHA-512 little-endian counter-mode expansion, before float +# conversion). Regenerated 2026-05-11 after the big-endian → little- +# endian counter flip (dav1d review; PHI_PRG_VERSION carries -le). +# Algorithm change MUST bump PHI_PRG_VERSION and create a new fixture +# file; do not overwrite — old runs replay against old data. +{"label": "placeholder-seed/zero-hash/dim_h=1", "version": "phi-prg-v1-hmac-sha512-le", "seed_hex": "ef532720a49159beb6816d98e13a162bac63c531b631bd1adb0fcca96b467ff3", "hard_hash_hex": "0000000000000000000000000000000000000000000000000000000000000000", "dim_h": 1, "output_sha256": "93618e085f1afae3368cabb57b328f2f01a81cb45c768e02264b13d5ec52732a", "output_bytes": 4} +{"label": "placeholder-seed/zero-hash/dim_h=8", "version": "phi-prg-v1-hmac-sha512-le", "seed_hex": "ef532720a49159beb6816d98e13a162bac63c531b631bd1adb0fcca96b467ff3", "hard_hash_hex": "0000000000000000000000000000000000000000000000000000000000000000", "dim_h": 8, "output_sha256": "b7e7af7180105100e94fcb4361799e2820d78f9cf2636643dea2a367ea16beb8", "output_bytes": 32} +{"label": "placeholder-seed/zero-hash/dim_h=32", "version": "phi-prg-v1-hmac-sha512-le", "seed_hex": "ef532720a49159beb6816d98e13a162bac63c531b631bd1adb0fcca96b467ff3", "hard_hash_hex": "0000000000000000000000000000000000000000000000000000000000000000", "dim_h": 32, "output_sha256": "1b5e89f20777000d0051daffb79aaf93e1e8bba7d3a77d5f6b4354561800d365", "output_bytes": 128} +{"label": "placeholder-seed/all-ones-hash/dim_h=16", "version": "phi-prg-v1-hmac-sha512-le", "seed_hex": "ef532720a49159beb6816d98e13a162bac63c531b631bd1adb0fcca96b467ff3", "hard_hash_hex": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "dim_h": 16, "output_sha256": "2466a75eda007980acea20ed9f1d8676c36700b6da15220455268151a0f6072c", "output_bytes": 64} +{"label": "seed=A/hash=B/dim_h=64", "version": "phi-prg-v1-hmac-sha512-le", "seed_hex": "14ab2dab0d3cddeaa58ec70632d3ff4f5de2c514004a92144e260dfe384d912e", "hard_hash_hex": "a23cb10b94660f062b467f313a9dc9d84f2dc2748c3627c661082dda7f55e3bb", "dim_h": 64, "output_sha256": "3c45b49e21fa5c13b847325b3a974de2545ef578b3856466b3760e5d8dc25a11", "output_bytes": 256} +{"label": "seed=A/hash=B'/dim_h=64 (one-bit-flip from prior)", "version": "phi-prg-v1-hmac-sha512-le", "seed_hex": "14ab2dab0d3cddeaa58ec70632d3ff4f5de2c514004a92144e260dfe384d912e", "hard_hash_hex": "223cb10b94660f062b467f313a9dc9d84f2dc2748c3627c661082dda7f55e3bb", "dim_h": 64, "output_sha256": "0cea076d0f088f3888312ec33c8fa60602ad7a5e4a2fd5c78584737d613b9888", "output_bytes": 256} +{"label": "seed=A'/hash=B/dim_h=64 (one-bit-flip seed)", "version": "phi-prg-v1-hmac-sha512-le", "seed_hex": "94ab2dab0d3cddeaa58ec70632d3ff4f5de2c514004a92144e260dfe384d912e", "hard_hash_hex": "a23cb10b94660f062b467f313a9dc9d84f2dc2748c3627c661082dda7f55e3bb", "dim_h": 64, "output_sha256": "ab0350482a8734c40ddfb965eb8f4996a8cd4c896d419f292dc66863a7c014bf", "output_bytes": 256} +{"label": "block-boundary/dim_h=16", "version": "phi-prg-v1-hmac-sha512-le", "seed_hex": "cfd60c2bda64ebcefbb23a5b28d98269c9c4f8b8ac77f6f9ca7a0f4865b10f58", "hard_hash_hex": "724cd966a7bfe78ba802877510ffb90c67f385a1d3135e4e1b8a1b38f744c6da", "dim_h": 16, "output_sha256": "75d1eb90b4d385b8d475fa53539eac129b0abfab9e784c0f3be5cf2738c40dec", "output_bytes": 64} +{"label": "block-boundary/dim_h=17", "version": "phi-prg-v1-hmac-sha512-le", "seed_hex": "cfd60c2bda64ebcefbb23a5b28d98269c9c4f8b8ac77f6f9ca7a0f4865b10f58", "hard_hash_hex": "724cd966a7bfe78ba802877510ffb90c67f385a1d3135e4e1b8a1b38f744c6da", "dim_h": 17, "output_sha256": "abce601ee72fdc7ca1bf1ee4aba58cc879a6e83d3d7e41e996db29cd30b0d3d4", "output_bytes": 68} +{"label": "stress/dim_h=4096", "version": "phi-prg-v1-hmac-sha512-le", "seed_hex": "0ddd62c311f88ebe2d4f6cd5d9d1374474dfd645e012043648dd966a71785c95", "hard_hash_hex": "e605ede3d9d0d13c6d7d32c5c424b998677eef0689a0d9f0fa4ebd1bb4307cb9", "dim_h": 4096, "output_sha256": "ff01f8bf2c516d2c54f595338c981cd853b0cff077ecf8eefdb5cf0dae69e55c", "output_bytes": 16384} diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 79d810b..fac4f8d 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -102,7 +102,7 @@ Newest first. Update on every open/close. | #000038 | Phase 4 content acquisition — proprietary textbook license decisions for warrant coverage | closed · obviated 2026-05-10 by alias-substitution sprint under #000031 (74 rows in #000041 + 13 rows in #000042); 92/92 records now resolve. Residue (multilingual PD, Hilbert-Ackermann OCR, Knuth permission, personal-copy path B) preserved as design log §8 | 2026-05-09 | — | | #000037 | Prometheus-Σ recursive falsification controller (bicameral substrate) | in progress · Phases 0 + 1 + 1.b + 1.c + 2 landed 2026-05-10; **§12 Trigger 2 fired** (divergence variance 0.575 / N=37); §22 Findings 2 + 3 RESOLVED (kernel/llm cost split + sweep_weights §15.4 + per-mode τ_qa); `controller_events` carries 4 event kinds (decision · difficulty · budget_allocation · falsification_proposal) feeding `arborist controller-events` inspector + live-harvest third bucket in `bench/scripts/harvest_falsification_proposals.py`; §12 Trigger 1 probe wired 2026-05-11 (`trigger_1_branch_density` reads `fork_score_branches` — measurable, not yet fired); Phase 3 sleep-sweep scheduler tracked under #000045 (gating ticket) | 2026-05-09 | — | | #000036 | T3 per-window covert-channel budget bound | in progress · Phase 1 + dav1d review 2026-05-11 → Tier-1 + Tier-2 (Option B = `b1_model=max_envelope` default, applied *in v1*, no v2 fork) both landed 2026-05-11; KAT regenerated (12 entries, active); baseline 625.87 → 6183.02 (max_envelope); 53 → 83 tests; both prior closure blockers cleared — remaining = fox final close-or-iterate call | 2026-05-09 | — | -| #000035 | PRG choice for φ_PRG (HMAC-SHA-512 expansion) | in progress · Phase 1 landed 2026-05-10; §9.10 amendment text **dav1d-reviewed final 2026-05-11** (manifest field → `anchor_prg_seed`; float-map prose corrected to "negligible mean −2^−32" not "unbiased"; `dim_h ≤ 16·2^32` guard + bool-reject added to impl; seed-independence + M1-policy-separation added; HMAC-SHA-512 / 32-byte / uint32-be / SHALL all LOCKED). Remaining: confirm v7 integer endianness + land §9.10 into v7 spec when plastic-training has a deployment target | 2026-05-09 | — | +| #000035 | PRG choice for φ_PRG (HMAC-SHA-512 expansion) | in progress · Phase 1 landed 2026-05-10; §9.10 amendment text **dav1d-reviewed final + endianness resolved 2026-05-11** (manifest field → `anchor_prg_seed`; v7 TLV confirmed little-endian → counter + uint32-word reads flipped big→little, `PHI_PRG_VERSION → -le`, 10 KATs regenerated; float-map prose corrected to "negligible mean −2^−32" not "unbiased"; `dim_h ≤ 16·2^32` guard + bool-reject added; seed-independence + M1-policy-separation added; HMAC-SHA-512 / 32-byte / SHALL all LOCKED; 31 tests). Remaining: land §9.10 into v7 spec when plastic-training has a deployment target (exogenous) | 2026-05-09 | — | | #000034 | Hessian alignment under φ_linear | in progress · Phase 1a landed 2026-05-10 (synthetic-ablation probe + KAT fixture); Phase 1b parks for v7 ramp-up | 2026-05-09 | — | | #000033 | Claim-pack pillar VII (combinatorics) | closed · landed 2026-05-09 (live in shard 000.db; lift verified) | 2026-05-09 | — | | #000032 | combinatorics@v1 π* (integer counting kernel) | closed · landed 2026-05-09 | 2026-05-09 | — | diff --git a/docs/soft-hash-channel-analysis.md b/docs/soft-hash-channel-analysis.md index 288390d..725da56 100644 --- a/docs/soft-hash-channel-analysis.md +++ b/docs/soft-hash-channel-analysis.md @@ -417,7 +417,7 @@ The reduction in §4 leaves three loose threads: **Phase 1 landed 2026-05-10** under #000035: reference implementation at `arborist/substrate/anchor_prg.py` (HMAC-SHA-512 counter-mode KDF, pure stdlib — `hashlib` + - `hmac`, no third-party crypto dep). 30 tests in + `hmac`, no third-party crypto dep). 31 tests in `tests/test_anchor_prg.py` covering determinism, range invariants, chi² uniformity, dim_h boundary (1, 16384), seed-bit-flip and hash-bit-flip avalanche, hand-computed @@ -428,7 +428,7 @@ The reduction in §4 leaves three loose threads: covering block-boundary cases (dim_h=16 = one HMAC block; dim_h=17 = two blocks with truncation), one-bit-flip variants, and a 4096-element counter-rollover stress sample. - Module exports `PHI_PRG_VERSION = "phi-prg-v1-hmac-sha512"` + Module exports `PHI_PRG_VERSION = "phi-prg-v1-hmac-sha512-le"` so future algorithm rotation is detectable at the call site without string-comparing module paths. @@ -436,19 +436,24 @@ The reduction in §4 leaves three loose threads: #000035 §3.4 holds the final wording. Decisions: HMAC-SHA-512 LOCK; **manifest field `anchor_prg_seed`** (purpose-scoped; `phi_prg_seed` is a code-local alias only); 32-byte seed LOCK; - `uint32` big-endian counter from 0 LOCK (flip to little-endian - only if v7's TLV convention already is); float map - `2·(u32/2^32)−1` kept for KAT compatibility but the prose now - says "uniform over a 2^32-point grid in [-1, 1) with negligible - finite-grid mean −2^−32" rather than "unbiased"; `SHALL` - replace `embed_hard_to_vec` when M1 enabled (M1 enablement - itself is a separate mitigation-selection-policy question, with - the NO_ALIGNMENT-skip rule documented in §9.10.1); plus a - `dim_h ≤ 16·2^32` exhaustion guard and a seed-independence / - single-purpose-seed requirement. The reference implementation + - tests + KAT vectors are pinned to these choices. Phase 2 lands - the §9.10 text into the v7 plastic-training spec when one gains - an active deployment target. + **`uint32` little-endian counter + uint32-word reads from 0** — + matching v7's canonical TLV integer encoding (`merkle-agi-dag_v7.txt` + §A1: TLV length prefixes + `enc_int` are `to_bytes(…, "little")`, + tensors are `30 tests (was 20; +7 from `de997f7` 2026-05-10 pattern + KDF) + 31 tests (was 20; +7 from `de997f7` 2026-05-10 pattern backfill — prefix-extension closure invariant, hand-computed HMAC-SHA-512 first-block formula, parametrized invalid-input cones) + 10-vector KAT fixture at diff --git a/docs/tickets/ticket-000035-prg-choice-phi-prg.md b/docs/tickets/ticket-000035-prg-choice-phi-prg.md index d50c18e..3754f92 100644 --- a/docs/tickets/ticket-000035-prg-choice-phi-prg.md +++ b/docs/tickets/ticket-000035-prg-choice-phi-prg.md @@ -1,6 +1,6 @@ # Ticket #000035 — PRG choice for φ_PRG -**Status:** in progress · Phase 1 landed 2026-05-10; **§9.10 spec amendment text dav1d-reviewed final 2026-05-11** (§3.4 below holds the folded version). Review decisions applied: manifest field renamed `phi_prg_seed → anchor_prg_seed` (purpose-scoped; `phi_prg_seed` kept as a code-local alias); float-map prose corrected ("uniform over a 2^32-point grid in [-1, 1) with negligible mean −2^−32", not "unbiased" — formula unchanged for KAT compat); `dim_h ≤ 16·2^32` exhaustion guard added to `arborist/substrate/anchor_prg.py` (+ bool-`dim_h` reject); seed-independence / single-purpose-seed rule added; M1-enablement-policy separation (§9.10.1: skipping M1 governed by mitigation-selection policy, e.g. NO_ALIGNMENT per #000034, not by §9.10). HMAC-SHA-512 / 32-byte seed / uint32-be counter / `SHALL`-replace all LOCKED. Remaining: confirm v7's canonical integer endianness (big-endian assumed) before the §9.10 text freezes into the v7 spec; the actual landing-into-v7-spec step still parks on v7 plastic-training gaining a deployment target. +**Status:** in progress · Phase 1 landed 2026-05-10; **§9.10 spec amendment text dav1d-reviewed final 2026-05-11** (§3.4 below holds the folded version). Review decisions applied: manifest field renamed `phi_prg_seed → anchor_prg_seed` (purpose-scoped; `phi_prg_seed` kept as a code-local alias); float-map prose corrected ("uniform over a 2^32-point grid in [-1, 1) with negligible mean −2^−32", not "unbiased"); `dim_h ≤ 16·2^32` exhaustion guard added to `arborist/substrate/anchor_prg.py` (+ bool-`dim_h` reject); seed-independence / single-purpose-seed rule added; M1-enablement-policy separation (§9.10.1: skipping M1 governed by mitigation-selection policy, e.g. NO_ALIGNMENT per #000034, not by §9.10). **Endianness RESOLVED 2026-05-11**: v7's canonical TLV integer encoding is little-endian (`merkle-agi-dag_v7.txt` §A1), so per dav1d's review rule the φ_PRG counter + uint32-word reads were flipped big-endian → little-endian; `PHI_PRG_VERSION → phi-prg-v1-hmac-sha512-le`; the 10 KAT vectors regenerated; `test_anchor_prg.py` updated (30 → 31). HMAC-SHA-512 / 32-byte seed / `SHALL`-replace all LOCKED. Remaining: land §9.10 into the v7 plastic-training spec when that spec gains a deployment target (exogenous). **Opened:** 2026-05-09 **Scope:** Pin a specific cryptographic PRG construction for the M1 mitigation (PRG-based anchor map) proposed in #000018 §5.2 + @@ -192,25 +192,34 @@ regression coverage. Generated once, pinned. ### 3.4 Spec amendment -v7 § 9.10 amendment text — **dav1d-reviewed final (2026-05-11)**. -Decision set (`RESPONSE` 2026-05-11): HMAC-SHA-512 — LOCK; manifest +v7 § 9.10 amendment text — **dav1d-reviewed final (2026-05-11), +little-endian variant**. Decision set (`RESPONSE` 2026-05-11) + +endianness resolution (2026-05-11): HMAC-SHA-512 — LOCK; manifest field `anchor_prg_seed` (purpose-scoped, not implementation-scoped — `phi_prg_seed` is an acceptable code-local alias only); seed -size 32 bytes — LOCK; counter `uint32` big-endian from 0 — LOCK -(flip to little-endian only if v7's TLV convention already is); -float map `2·(u32/2^32)−1` kept for KAT compatibility but the -prose must say "uniform over a 2^32-point grid in [-1, 1) with +size 32 bytes — LOCK; counter `uint32` **little-endian** from 0, +and the uint32-word interpretation is also little-endian — matching +v7's canonical TLV integer encoding (verified against +`merkle-agi-dag_v7.txt` §A1: TLV length prefixes +`to_bytes(4, "little")`, `enc_int` is `to_bytes(8, "little")`, +quantized tensors are `30 tests covering determinism, +- ``tests/test_anchor_prg.py`` — 31 tests covering determinism, range invariants, chi² loose-uniformity sanity, dim_h boundary (1, 16384), seed-bit-flip avalanche, hash-bit-flip avalanche, hand-computed HMAC-SHA-512 first-block formula, prefix-extension diff --git a/docs/tickets/ticket-000044-autocount-doc-drift-discipline.md b/docs/tickets/ticket-000044-autocount-doc-drift-discipline.md index 517c691..54b0278 100644 --- a/docs/tickets/ticket-000044-autocount-doc-drift-discipline.md +++ b/docs/tickets/ticket-000044-autocount-doc-drift-discipline.md @@ -82,7 +82,7 @@ because the prose IS the inside-of-tag content. ### 3.1 `tests` ```markdown -30 +31 ``` Live value: `pytest --collect-only -q ` count, with diff --git a/docs/warrant-substrate-cookbook.md b/docs/warrant-substrate-cookbook.md index 3fc5566..2d678c8 100644 --- a/docs/warrant-substrate-cookbook.md +++ b/docs/warrant-substrate-cookbook.md @@ -589,7 +589,7 @@ than waiting for bench-time STRICT-rate drift to surface it. ### Substrate-paper-spec'd primitives (#000012 + #000018 + #000034 + #000037) -- `tests/test_anchor_prg.py` — 30 tests for φ_PRG HMAC-SHA-512 +- `tests/test_anchor_prg.py` — 31 tests for φ_PRG HMAC-SHA-512 expansion (#000035 Phase 1). Covers KAT regression, hand-formula (first-block matches direct ``hmac.new``), prefix-extending closure invariant, output-length monotonicity per dim_h. diff --git a/tests/test_anchor_prg.py b/tests/test_anchor_prg.py index 3844a9f..09621e2 100644 --- a/tests/test_anchor_prg.py +++ b/tests/test_anchor_prg.py @@ -77,8 +77,21 @@ def test_bytes_to_floats_max_uint32_just_below_one(): def test_bytes_to_floats_midpoint_maps_to_zero(): - # 0x80000000 → 2 * 0.5 - 1 = 0.0. - assert _bytes_to_floats(b"\x80\x00\x00\x00") == [0.0] + # u32 = 2^31 → 2 * 0.5 - 1 = 0.0. Words are read little-endian + # (matches v7's TLV convention), so the byte representation of + # 2^31 is b"\x00\x00\x00\x80", not b"\x80\x00\x00\x00". + assert int.from_bytes(b"\x00\x00\x00\x80", "little") == 1 << 31 + assert _bytes_to_floats(b"\x00\x00\x00\x80") == [0.0] + + +def test_bytes_to_floats_reads_little_endian(): + """Sanity-pin the endianness: 0x00000001 in little-endian byte + layout is b"\x01\x00\x00\x00" → u32=1 → 2*(1/2^32)-1 ≈ -1.0. + The big-endian reading (u32 = 0x01000000 = 2^24) would give a + very different float — this test catches an accidental flip back.""" + out = _bytes_to_floats(b"\x01\x00\x00\x00") + assert out[0] == pytest.approx(2.0 * (1 / 2**32) - 1.0, abs=1e-12) + assert out[0] < -0.999999 # ----------------------------------------------------------- chi² sanity @@ -215,33 +228,37 @@ def test_phi_prg_output_is_prefix_extending(): def test_phi_prg_first_block_matches_direct_hmac(): """First HMAC-SHA-512 block of phi_prg's output should match - a direct hmac.new(seed, hard_hash + b'\\x00\\x00\\x00\\x00', - sha512).digest() invocation. Hand-computed against the - function's spec (#000035 §3.1 + module §1). + a direct hmac.new(seed, hard_hash + counter_le_4, sha512).digest() + invocation. Hand-computed against the function's spec (#000035 + §3.1/§3.4 + module §1). Counter and uint32-word reads are both + little-endian (matches v7's TLV convention; dav1d review + 2026-05-11) — at counter=0 the bytes are b'\\x00\\x00\\x00\\x00' + regardless of endianness, but the float-word interpretation is + the part this test pins as little-endian. Pattern from fox's `test_b1_exact_formula`: don't rely on KAT regression alone — compute the first-principles math in the test file and assert exact agreement. Catches algorithm - drift that KAT regenerated against a buggy version would - miss. + drift that KAT regenerated against a buggy version would miss. """ import hmac seed = hashlib.sha256(b"hand-formula-seed").digest() h = hashlib.sha256(b"hand-formula-hash").digest() - # Spec: out = HMAC-SHA-512(seed, hard_hash || counter_be_4) for + # Spec: out = HMAC-SHA-512(seed, hard_hash || counter_le_4) for # counter = 0, 1, 2, ...; concatenated; truncated to dim_h * 4 bytes. expected_block_0 = hmac.new( - seed, h + (0).to_bytes(4, "big"), hashlib.sha512 + seed, h + (0).to_bytes(4, "little"), hashlib.sha512 ).digest() # First block is 64 bytes = 16 uint32s = 16 floats. dim_h=16 # consumes exactly the first block. floats = phi_prg(h, dim_h=16, seed=seed) - # Convert expected_block_0 to floats per §2.3 spec. + # Convert expected_block_0 to floats per §3.4 spec: uint32 read + # little-endian, then 2*(u32/2^32)-1. expected_floats = [] for i in range(16): - u32 = int.from_bytes(expected_block_0[4 * i : 4 * (i + 1)], "big") + u32 = int.from_bytes(expected_block_0[4 * i : 4 * (i + 1)], "little") expected_floats.append(2.0 * (u32 / 2 ** 32) - 1.0) for j, (got, exp) in enumerate(zip(floats, expected_floats)): assert got == pytest.approx(exp, abs=1e-12), ( @@ -332,12 +349,16 @@ def test_phi_prg_rejects_dim_h_above_counter_ceiling(): def test_module_exports_version_string(): - assert PHI_PRG_VERSION == "phi-prg-v1-hmac-sha512" + # The "-le" suffix records the 2026-05-11 big-endian → little-endian + # counter flip (dav1d review; matches v7's TLV convention). A future + # endianness or formula change MUST bump this token + regen the KATs. + assert PHI_PRG_VERSION == "phi-prg-v1-hmac-sha512-le" # versioned-default discipline (calculator-test-patterns.md §2): # "v1" substring present so future major-version rotations # (v2-blake3-expansion etc.) are detectable at the call site # without string-comparing module paths. assert "v1" in PHI_PRG_VERSION + assert PHI_PRG_VERSION.endswith("-le") def test_placeholder_seed_is_32_bytes(): @@ -391,6 +412,14 @@ def test_phi_prg_known_answer_tests(): f"KAT mismatch for label={kat.get('label')!r}: " f"expected {kat['output_sha256']}, got {digest}" ) + # Version pin (present in 2026-05-11+ fixtures): a KAT recorded + # under a different PHI_PRG_VERSION must not be silently replayed + # against the current algorithm. + if "version" in kat: + assert kat["version"] == PHI_PRG_VERSION, ( + f"KAT version mismatch for label={kat.get('label')!r}: " + f"fixture {kat['version']}, module {PHI_PRG_VERSION}" + ) # Also assert the float list has the expected length so a # bug in _bytes_to_floats truncation is caught. assert len(out_floats) == dim_h