From 27f468c5b799b247cb22215863be8843c08b73a3 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Mon, 20 Apr 2026 10:57:39 -0400 Subject: [PATCH] portal-rng: Python + C impls of xoshiro256** + portal state capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Zoe's contribution question: does our portal preserve RNG state so a simulation can continue in another process with the same random stream? Answer today: no — no RNG existed. Answer now (Python + C): yes, bit-identical. - New builtins: random-seed!, random, random-int, random-state, random-state! - xoshiro256** (Blackman & Vigna 2018) — deterministic, portable, no libc rand - State = 4 x u64; portal-v1 JSON gains 'rng' field with 8 x u32 halves - Python and C produce bit-identical streams (verified: seed=42, 10 draws) - Asm impl + cross-impl tests + whitepaper note: next commits Ticket: docs/tickets/0001-portal-rng.md --- c/portal.c | 127 +++++++++++++++++++++++++ docs/tickets/0001-portal-rng.md | 161 ++++++++++++++++++++++++++++++++ lumbda.py | 78 ++++++++++++++++ 3 files changed, 366 insertions(+) create mode 100644 docs/tickets/0001-portal-rng.md diff --git a/c/portal.c b/c/portal.c index 092ceda..e2382b4 100644 --- a/c/portal.c +++ b/c/portal.c @@ -16,6 +16,56 @@ __thread const char *g_portal_checkpoint_path = NULL; +/* ═══════════════════════════════════════════════════════════════════════════ + * xoshiro256** — deterministic, portable PRNG shared with Python and asm. + * Portal serializes these 4 words so simulations continue across processes + * with a bit-identical random stream. Reference: Blackman & Vigna 2018. + * ═══════════════════════════════════════════════════════════════════════════ */ + +static uint64_t g_rng_state[4] = {0, 0, 0, 0}; + +static uint64_t rng_rotl(uint64_t x, int k) { + return (x << k) | (x >> (64 - k)); +} + +static uint64_t rng_splitmix64_step(uint64_t *z) { + *z += 0x9e3779b97f4a7c15ULL; + uint64_t r = *z; + r = (r ^ (r >> 30)) * 0xbf58476d1ce4e5b9ULL; + r = (r ^ (r >> 27)) * 0x94d049bb133111ebULL; + return r ^ (r >> 31); +} + +static void rng_seed(uint64_t k) { + uint64_t z = k; + for (int i = 0; i < 4; i++) g_rng_state[i] = rng_splitmix64_step(&z); +} + +static uint64_t rng_next(void) { + uint64_t result = rng_rotl(g_rng_state[1] * 5, 7) * 9; + uint64_t t = g_rng_state[1] << 17; + g_rng_state[2] ^= g_rng_state[0]; + g_rng_state[3] ^= g_rng_state[1]; + g_rng_state[1] ^= g_rng_state[2]; + g_rng_state[0] ^= g_rng_state[3]; + g_rng_state[2] ^= t; + g_rng_state[3] = rng_rotl(g_rng_state[3], 45); + return result; +} + +void rng_get_halves(uint32_t out[8]) { + for (int i = 0; i < 4; i++) { + out[2 * i] = (uint32_t)(g_rng_state[i] & 0xffffffffULL); + out[2 * i + 1] = (uint32_t)(g_rng_state[i] >> 32); + } +} + +void rng_set_halves(const uint32_t in[8]) { + for (int i = 0; i < 4; i++) { + g_rng_state[i] = ((uint64_t)in[2 * i + 1] << 32) | (uint64_t)in[2 * i]; + } +} + /* ═══════════════════════════════════════════════════════════════════════════ * Minimal JSON writer — writes directly to FILE* * ═══════════════════════════════════════════════════════════════════════════ */ @@ -190,6 +240,18 @@ void portal_save(Env *env, const char *path, FullCont *continuation) { fputs("{\"format\":\"lumbda-portal-v1\",\n", fp); + /* RNG state — xoshiro256** as 8 × u32 halves (low, high, low, high, ...) */ + { + uint32_t halves[8]; + rng_get_halves(halves); + fputs("\"rng\":{\"algo\":\"xoshiro256**\",\"state\":[", fp); + for (int i = 0; i < 8; i++) { + if (i > 0) fputc(',', fp); + fprintf(fp, "%u", halves[i]); + } + fputs("]},\n", fp); + } + /* Environment bindings (user-defined only from global) */ fputs("\"env\":", fp); json_write_env_bindings(fp, env, (env->global == env)); @@ -625,6 +687,19 @@ bool portal_resume(const char *path, Env *base_env, Env **out_env, FullCont **ou return false; } + /* Restore RNG state if present (absent = pre-RNG portal file, skip) */ + JsonNode *rng_node = json_obj_get(root, "rng"); + if (rng_node && rng_node->type == JT_OBJECT) { + JsonNode *state_arr = json_obj_get(rng_node, "state"); + if (state_arr && state_arr->type == JT_ARRAY && state_arr->array.count == 8) { + uint32_t halves[8]; + for (int i = 0; i < 8; i++) { + halves[i] = (uint32_t)json_int(state_arr->array.items[i]); + } + rng_set_halves(halves); + } + } + /* Merge environment bindings into base_env */ JsonNode *env_node = json_obj_get(root, "env"); if (env_node && env_node->type == JT_OBJECT) { @@ -679,7 +754,59 @@ static Value builtin_portal_checkpoint(Value *args, int nargs, Env *env) { return VAL_VOID; } +static Value builtin_random_seed_bang(Value *args, int nargs, Env *env) { + (void)env; + if (nargs != 1) lisp_error("random-seed!: expected 1 arg"); + rng_seed((uint64_t)as_number_int(args[0])); + return VAL_VOID; +} + +static Value builtin_random(Value *args, int nargs, Env *env) { + (void)args; (void)env; + if (nargs != 0) lisp_error("random: expected 0 args"); + return make_double((double)(rng_next() >> 11) / (double)(1ULL << 53)); +} + +static Value builtin_random_int(Value *args, int nargs, Env *env) { + (void)env; + if (nargs != 1) lisp_error("random-int: expected 1 arg"); + int64_t n = as_number_int(args[0]); + if (n <= 0) lisp_error("random-int: n must be positive, got %lld", (long long)n); + return VAL_INT((int64_t)(rng_next() % (uint64_t)n)); +} + +static Value builtin_random_state(Value *args, int nargs, Env *env) { + (void)args; (void)env; + if (nargs != 0) lisp_error("random-state: expected 0 args"); + uint32_t halves[8]; + rng_get_halves(halves); + Value list = VAL_NIL; + for (int i = 7; i >= 0; i--) { + list = cons(VAL_INT((int64_t)halves[i]), list); + } + return list; +} + +static Value builtin_random_state_bang(Value *args, int nargs, Env *env) { + (void)env; + if (nargs != 1) lisp_error("random-state!: expected 1 arg (list of 8 ints)"); + uint32_t halves[8]; + Value lst = args[0]; + for (int i = 0; i < 8; i++) { + if (!IS_PAIR(lst)) lisp_error("random-state!: list too short"); + halves[i] = (uint32_t)as_number_int(CAR(lst)); + lst = CDR(lst); + } + rng_set_halves(halves); + return VAL_VOID; +} + void register_portal_builtins(Env *env) { env_define(env, intern("portal-checkpoint!"), VAL_BUILTIN(builtin_portal_checkpoint)); env_define(env, intern("portal-save!"), VAL_BUILTIN(builtin_portal_checkpoint)); + env_define(env, intern("random-seed!"), VAL_BUILTIN(builtin_random_seed_bang)); + env_define(env, intern("random"), VAL_BUILTIN(builtin_random)); + env_define(env, intern("random-int"), VAL_BUILTIN(builtin_random_int)); + env_define(env, intern("random-state"), VAL_BUILTIN(builtin_random_state)); + env_define(env, intern("random-state!"), VAL_BUILTIN(builtin_random_state_bang)); } diff --git a/docs/tickets/0001-portal-rng.md b/docs/tickets/0001-portal-rng.md new file mode 100644 index 0000000..bc83c5c --- /dev/null +++ b/docs/tickets/0001-portal-rng.md @@ -0,0 +1,161 @@ +# 0001 — Portal preserves RNG state across processes + +**Status:** open +**Reporter:** Zoe (via fox) +**Implementer:** blackops +**Opened:** 2026-04-20 + +## Problem + +Our portal serializes environment bindings and full continuations. It does +not capture any random-number-generator state — because no RNG builtin +exists in any of our three impls today. A simulation that draws randoms, +saves mid-run, and resumes in another process will diverge from a +single-process baseline the moment it calls `(random)`. + +Zoe flagged this as a research contribution: *"does portal tech keep the +random seed so we can transfer random entropy between processes when +continuing a simulation?"* Answer today: no. Answer after this ticket: +yes, bit-for-bit identical across Python ↔ C ↔ asm. + +## Goals + +1. Pick one deterministic, portable PRNG and bind it in all three impls. +2. Expose a minimal builtin API: seed, draw float, draw bounded int, + introspect/set state. +3. Extend portal-v1 so save/resume round-trips RNG state. +4. Prove cross-impl reproducibility with a test: seed `k`, draw `N` + values, save, clear, resume in any impl, draw `M` more — full stream + matches a single-process Python baseline. + +## Non-goals + +- Statistical quality beyond xoshiro256\*\*'s documented properties. +- Thread-local RNG (one global stream per process; matches current + `__thread` portal checkpoint discipline in C). +- Cryptographic strength. +- Rejection-sampling uniformity for `(random-int n)` — plain modulo, + bias acceptable for `n << 2^64`. + +## Algorithm — xoshiro256\*\* + +State: four `uint64_t` words `s[0..3]`. + +``` +rotl(x, k) = (x << k) | (x >> (64 - k)) + +next(): + result = rotl(s[1] * 5, 7) * 9 + t = s[1] << 17 + s[2] ^= s[0] + s[3] ^= s[1] + s[1] ^= s[2] + s[0] ^= s[3] + s[2] ^= t + s[3] = rotl(s[3], 45) + return result +``` + +Seeding uses splitmix64 from a single 64-bit seed: + +``` +splitmix64(&z): + z += 0x9e3779b97f4a7c15 + z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9 + z = (z ^ (z >> 27)) * 0x94d049bb133111eb + return z ^ (z >> 31) + +seed(k): + z = k + for i in 0..3: s[i] = splitmix64(&z) +``` + +Chosen because: +- Public domain reference (Blackman & Vigna, 2018). +- Identical bit output on any machine — no libc, no FPU, no `rand()`. +- Four u64 words → same wire format across Python, C, asm. +- Small code footprint — asm port is ~40 instructions. + +## Builtin API + +| builtin | args | returns | +|---|---|---| +| `(random-seed! k)` | int `k` | void — reseeds from `k` via splitmix64 | +| `(random)` | — | float in `[0.0, 1.0)` — top 53 bits / 2^53 | +| `(random-int n)` | positive int `n` | int in `[0, n)` — raw modulo | +| `(random-state)` | — | list of 8 ints: `(w0_lo w0_hi w1_lo w1_hi w2_lo w2_hi w3_lo w3_hi)` each in `[0, 2^32)` | +| `(random-state! s)` | list of 8 ints | void — restores state from the list | + +Rationale for the 8-int representation: each 64-bit word split into low +and high 32-bit halves. Fits Lumbda's 48-bit C int and 61-bit asm int +without widening. Portable over the S-expression portal too. + +Default seed at interpreter startup: `0` (all impls). Deterministic from +process start means tests don't need to seed explicitly unless they +want a specific stream. + +## Portal format changes + +### JSON (Python + C) — lumbda-portal-v1 + +Add optional top-level key `"rng"` to the existing `lumbda-portal-v1` +envelope. No magic bump required — absent field means "assume seed 0". + +```json +{ + "format": "lumbda-portal-v1", + "env": {...}, + "continuation": ..., + "rng": { + "algo": "xoshiro256**", + "state": [w0_lo, w0_hi, w1_lo, w1_hi, w2_lo, w2_hi, w3_lo, w3_hi] + } +} +``` + +### Asm binary — LUMBDAB2 + +Bump magic from `LUMBDAB1` to `LUMBDAB2`. Extend `PORTAL_HDR_SIZE` from +48 to 80 bytes: append 32 bytes of raw RNG state (4 × u64, little-endian, +native layout). + +Old `LUMBDAB1` files rejected during resume. (Portal is not yet +externally released, so no migration burden.) + +### Asm text (GC build) — `;; lumbda-portal v1` + +Append one comment line after the header: + +``` +;; lumbda-portal v1 +;; rng xoshiro256** w0_lo w0_hi w1_lo w1_hi w2_lo w2_hi w3_lo w3_hi +... +``` + +Parser recognizes the `;; rng ` prefix and calls `random-state!` +internally. + +## Test plan + +1. **Unit** — seed `42`, first 5 draws match a hardcoded Python baseline + bit-for-bit. Asserted in all three impls. +2. **Functional** — `tests/functional.lsp` gets determinism tests. +3. **Portal round-trip** — `tests/portal-rng-save.lsp` seeds, draws 50, + saves, exits. `tests/portal-rng-load.lsp` resumes, draws 50 more, + compares against full 100-draw baseline. +4. **Cross-impl** — wire into `tests/portal-cross-test.sh`. Python saves, + C resumes, next draws match. All 9 producer×consumer cells. +5. **MOAD scan** — `~/git/unmoad.com/unmoad` on all changed files. + +## Deliverables + +- [ ] `docs/tickets/0001-portal-rng.md` (this ticket) +- [ ] Python: `lumbda.py` — xoshiro256\*\* + 5 builtins + portal hooks +- [ ] C: `c/builtins.c` + `c/portal.c` + `c/lumbda.h` — same +- [ ] Asm: `asm/lumbda.s` — same, plus LUMBDAB2 header +- [ ] Shared tests: `tests/functional.lsp` determinism block +- [ ] Cross tests: `tests/portal-rng-save.lsp`, `tests/portal-rng-load.lsp` +- [ ] `tests/portal-cross-test.sh` — new RNG cells +- [ ] Asm test: `asm/test.sh` determinism checks +- [ ] `make test-all` green +- [ ] `unmoad` clean on all changed files diff --git a/lumbda.py b/lumbda.py index 14600d6..f213e8a 100644 --- a/lumbda.py +++ b/lumbda.py @@ -1959,6 +1959,73 @@ def load_compiled(path, env): return CompiledProc(code, params, rest, env, data.get('name')) +############################################################################### +# xoshiro256** — deterministic, portable PRNG shared with C and asm impls. +# Portal serializes this state so simulations continue across processes with +# a bit-identical random stream. Reference: Blackman & Vigna 2018. +############################################################################### + +_MASK64 = (1 << 64) - 1 +_rng_state = [0, 0, 0, 0] + + +def _rng_splitmix64_step(z): + """Returns (output, next_counter). Reference: Vigna splitmix64. + The persistent counter advances only by the constant; the mixing + is on a local copy.""" + z = (z + 0x9e3779b97f4a7c15) & _MASK64 + r = z + r = ((r ^ (r >> 30)) * 0xbf58476d1ce4e5b9) & _MASK64 + r = ((r ^ (r >> 27)) * 0x94d049bb133111eb) & _MASK64 + return (r ^ (r >> 31)) & _MASK64, z + + +def _rng_seed(k): + z = k & _MASK64 + for i in range(4): + _rng_state[i], z = _rng_splitmix64_step(z) + + +def _rng_next(): + s = _rng_state + v = (s[1] * 5) & _MASK64 + result = ((((v << 7) & _MASK64) | (v >> 57)) * 9) & _MASK64 + t = (s[1] << 17) & _MASK64 + s[2] ^= s[0] + s[3] ^= s[1] + s[1] ^= s[2] + s[0] ^= s[3] + s[2] ^= t + s[3] = (((s[3] << 45) & _MASK64) | (s[3] >> 19)) & _MASK64 + return result + + +def _rng_random_float(): + return (_rng_next() >> 11) / (1 << 53) + + +def _rng_random_int(n): + if n <= 0: raise LispErr(f'random-int: n must be positive, got {n}') + return _rng_next() % n + + +def _rng_state_to_halves(): + out = [] + for w in _rng_state: + out.append(w & 0xffffffff) + out.append((w >> 32) & 0xffffffff) + return out + + +def _rng_state_from_halves(halves): + if len(halves) != 8: + raise LispErr('random-state!: expected list of 8 integers') + for i in range(4): + lo = halves[2 * i] & 0xffffffff + hi = halves[2 * i + 1] & 0xffffffff + _rng_state[i] = (hi << 32) | lo + + ############################################################################### # Portal — Serialize/resume full machine state across machines ############################################################################### @@ -2221,6 +2288,7 @@ def portal_save(env, path, continuation=None): 'env': ser.serialize_env(env), 'continuation': ser.serialize_continuation(continuation) if continuation else None, 'auto_compile': _auto_compile[0], + 'rng': {'algo': 'xoshiro256**', 'state': _rng_state_to_halves()}, } state['objects'] = ser.finalize() with open(path, 'w') as f: @@ -2239,6 +2307,9 @@ def portal_resume(path, base_env=None): des = _PortalDeserializer(base_env) des.rebuild_objects(state.get('objects', [])) _auto_compile[0] = state.get('auto_compile', False) + rng = state.get('rng') + if rng and 'state' in rng: + _rng_state_from_halves(rng['state']) cont = None if state.get('continuation'): cont = des.deserialize_value(state['continuation']) @@ -3472,6 +3543,13 @@ def make_global_env(): d(S('portal-resume'), lambda a, _: _portal_resume_builtin(_str_val(a[0]), g)) d(S('portal-checkpoint!'), lambda a, _: _portal_checkpoint.__setitem__(0, _str_val(a[0])) or VOID) + # ── Random (xoshiro256**) — portal-serialized across all three impls ──── + d(S('random-seed!'), lambda a, _: _rng_seed(int(_num(a[0]))) or VOID) + d(S('random'), lambda a, _: _rng_random_float()) + d(S('random-int'), lambda a, _: _rng_random_int(int(_num(a[0])))) + d(S('random-state'), lambda a, _: _P(_rng_state_to_halves())) + d(S('random-state!'), lambda a, _: _rng_state_from_halves([int(_num(x)) for x in _L(a[0])]) or VOID) + def _portal_resume_builtin(path, env): """Resume from portal file, merging into current env.""" _, cont = portal_resume(path, env)