portal-rng: Python + C impls of xoshiro256** + portal state capture
Zoe's contribution question: does our portal preserve RNG state so a simulation can continue in another process with the same random stream? Answer today: no — no RNG existed. Answer now (Python + C): yes, bit-identical. - New builtins: random-seed!, random, random-int, random-state, random-state! - xoshiro256** (Blackman & Vigna 2018) — deterministic, portable, no libc rand - State = 4 x u64; portal-v1 JSON gains 'rng' field with 8 x u32 halves - Python and C produce bit-identical streams (verified: seed=42, 10 draws) - Asm impl + cross-impl tests + whitepaper note: next commits Ticket: docs/tickets/0001-portal-rng.md
This commit is contained in:
parent
6e9d3ea52f
commit
27f468c5b7
3 changed files with 366 additions and 0 deletions
127
c/portal.c
127
c/portal.c
|
|
@ -16,6 +16,56 @@
|
|||
|
||||
__thread const char *g_portal_checkpoint_path = NULL;
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════════════
|
||||
* xoshiro256** — deterministic, portable PRNG shared with Python and asm.
|
||||
* Portal serializes these 4 words so simulations continue across processes
|
||||
* with a bit-identical random stream. Reference: Blackman & Vigna 2018.
|
||||
* ═══════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
static uint64_t g_rng_state[4] = {0, 0, 0, 0};
|
||||
|
||||
static uint64_t rng_rotl(uint64_t x, int k) {
|
||||
return (x << k) | (x >> (64 - k));
|
||||
}
|
||||
|
||||
static uint64_t rng_splitmix64_step(uint64_t *z) {
|
||||
*z += 0x9e3779b97f4a7c15ULL;
|
||||
uint64_t r = *z;
|
||||
r = (r ^ (r >> 30)) * 0xbf58476d1ce4e5b9ULL;
|
||||
r = (r ^ (r >> 27)) * 0x94d049bb133111ebULL;
|
||||
return r ^ (r >> 31);
|
||||
}
|
||||
|
||||
static void rng_seed(uint64_t k) {
|
||||
uint64_t z = k;
|
||||
for (int i = 0; i < 4; i++) g_rng_state[i] = rng_splitmix64_step(&z);
|
||||
}
|
||||
|
||||
static uint64_t rng_next(void) {
|
||||
uint64_t result = rng_rotl(g_rng_state[1] * 5, 7) * 9;
|
||||
uint64_t t = g_rng_state[1] << 17;
|
||||
g_rng_state[2] ^= g_rng_state[0];
|
||||
g_rng_state[3] ^= g_rng_state[1];
|
||||
g_rng_state[1] ^= g_rng_state[2];
|
||||
g_rng_state[0] ^= g_rng_state[3];
|
||||
g_rng_state[2] ^= t;
|
||||
g_rng_state[3] = rng_rotl(g_rng_state[3], 45);
|
||||
return result;
|
||||
}
|
||||
|
||||
void rng_get_halves(uint32_t out[8]) {
|
||||
for (int i = 0; i < 4; i++) {
|
||||
out[2 * i] = (uint32_t)(g_rng_state[i] & 0xffffffffULL);
|
||||
out[2 * i + 1] = (uint32_t)(g_rng_state[i] >> 32);
|
||||
}
|
||||
}
|
||||
|
||||
void rng_set_halves(const uint32_t in[8]) {
|
||||
for (int i = 0; i < 4; i++) {
|
||||
g_rng_state[i] = ((uint64_t)in[2 * i + 1] << 32) | (uint64_t)in[2 * i];
|
||||
}
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════════════
|
||||
* Minimal JSON writer — writes directly to FILE*
|
||||
* ═══════════════════════════════════════════════════════════════════════════ */
|
||||
|
|
@ -190,6 +240,18 @@ void portal_save(Env *env, const char *path, FullCont *continuation) {
|
|||
|
||||
fputs("{\"format\":\"lumbda-portal-v1\",\n", fp);
|
||||
|
||||
/* RNG state — xoshiro256** as 8 × u32 halves (low, high, low, high, ...) */
|
||||
{
|
||||
uint32_t halves[8];
|
||||
rng_get_halves(halves);
|
||||
fputs("\"rng\":{\"algo\":\"xoshiro256**\",\"state\":[", fp);
|
||||
for (int i = 0; i < 8; i++) {
|
||||
if (i > 0) fputc(',', fp);
|
||||
fprintf(fp, "%u", halves[i]);
|
||||
}
|
||||
fputs("]},\n", fp);
|
||||
}
|
||||
|
||||
/* Environment bindings (user-defined only from global) */
|
||||
fputs("\"env\":", fp);
|
||||
json_write_env_bindings(fp, env, (env->global == env));
|
||||
|
|
@ -625,6 +687,19 @@ bool portal_resume(const char *path, Env *base_env, Env **out_env, FullCont **ou
|
|||
return false;
|
||||
}
|
||||
|
||||
/* Restore RNG state if present (absent = pre-RNG portal file, skip) */
|
||||
JsonNode *rng_node = json_obj_get(root, "rng");
|
||||
if (rng_node && rng_node->type == JT_OBJECT) {
|
||||
JsonNode *state_arr = json_obj_get(rng_node, "state");
|
||||
if (state_arr && state_arr->type == JT_ARRAY && state_arr->array.count == 8) {
|
||||
uint32_t halves[8];
|
||||
for (int i = 0; i < 8; i++) {
|
||||
halves[i] = (uint32_t)json_int(state_arr->array.items[i]);
|
||||
}
|
||||
rng_set_halves(halves);
|
||||
}
|
||||
}
|
||||
|
||||
/* Merge environment bindings into base_env */
|
||||
JsonNode *env_node = json_obj_get(root, "env");
|
||||
if (env_node && env_node->type == JT_OBJECT) {
|
||||
|
|
@ -679,7 +754,59 @@ static Value builtin_portal_checkpoint(Value *args, int nargs, Env *env) {
|
|||
return VAL_VOID;
|
||||
}
|
||||
|
||||
static Value builtin_random_seed_bang(Value *args, int nargs, Env *env) {
|
||||
(void)env;
|
||||
if (nargs != 1) lisp_error("random-seed!: expected 1 arg");
|
||||
rng_seed((uint64_t)as_number_int(args[0]));
|
||||
return VAL_VOID;
|
||||
}
|
||||
|
||||
static Value builtin_random(Value *args, int nargs, Env *env) {
|
||||
(void)args; (void)env;
|
||||
if (nargs != 0) lisp_error("random: expected 0 args");
|
||||
return make_double((double)(rng_next() >> 11) / (double)(1ULL << 53));
|
||||
}
|
||||
|
||||
static Value builtin_random_int(Value *args, int nargs, Env *env) {
|
||||
(void)env;
|
||||
if (nargs != 1) lisp_error("random-int: expected 1 arg");
|
||||
int64_t n = as_number_int(args[0]);
|
||||
if (n <= 0) lisp_error("random-int: n must be positive, got %lld", (long long)n);
|
||||
return VAL_INT((int64_t)(rng_next() % (uint64_t)n));
|
||||
}
|
||||
|
||||
static Value builtin_random_state(Value *args, int nargs, Env *env) {
|
||||
(void)args; (void)env;
|
||||
if (nargs != 0) lisp_error("random-state: expected 0 args");
|
||||
uint32_t halves[8];
|
||||
rng_get_halves(halves);
|
||||
Value list = VAL_NIL;
|
||||
for (int i = 7; i >= 0; i--) {
|
||||
list = cons(VAL_INT((int64_t)halves[i]), list);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
static Value builtin_random_state_bang(Value *args, int nargs, Env *env) {
|
||||
(void)env;
|
||||
if (nargs != 1) lisp_error("random-state!: expected 1 arg (list of 8 ints)");
|
||||
uint32_t halves[8];
|
||||
Value lst = args[0];
|
||||
for (int i = 0; i < 8; i++) {
|
||||
if (!IS_PAIR(lst)) lisp_error("random-state!: list too short");
|
||||
halves[i] = (uint32_t)as_number_int(CAR(lst));
|
||||
lst = CDR(lst);
|
||||
}
|
||||
rng_set_halves(halves);
|
||||
return VAL_VOID;
|
||||
}
|
||||
|
||||
void register_portal_builtins(Env *env) {
|
||||
env_define(env, intern("portal-checkpoint!"), VAL_BUILTIN(builtin_portal_checkpoint));
|
||||
env_define(env, intern("portal-save!"), VAL_BUILTIN(builtin_portal_checkpoint));
|
||||
env_define(env, intern("random-seed!"), VAL_BUILTIN(builtin_random_seed_bang));
|
||||
env_define(env, intern("random"), VAL_BUILTIN(builtin_random));
|
||||
env_define(env, intern("random-int"), VAL_BUILTIN(builtin_random_int));
|
||||
env_define(env, intern("random-state"), VAL_BUILTIN(builtin_random_state));
|
||||
env_define(env, intern("random-state!"), VAL_BUILTIN(builtin_random_state_bang));
|
||||
}
|
||||
|
|
|
|||
161
docs/tickets/0001-portal-rng.md
Normal file
161
docs/tickets/0001-portal-rng.md
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
# 0001 — Portal preserves RNG state across processes
|
||||
|
||||
**Status:** open
|
||||
**Reporter:** Zoe (via fox)
|
||||
**Implementer:** blackops
|
||||
**Opened:** 2026-04-20
|
||||
|
||||
## Problem
|
||||
|
||||
Our portal serializes environment bindings and full continuations. It does
|
||||
not capture any random-number-generator state — because no RNG builtin
|
||||
exists in any of our three impls today. A simulation that draws randoms,
|
||||
saves mid-run, and resumes in another process will diverge from a
|
||||
single-process baseline the moment it calls `(random)`.
|
||||
|
||||
Zoe flagged this as a research contribution: *"does portal tech keep the
|
||||
random seed so we can transfer random entropy between processes when
|
||||
continuing a simulation?"* Answer today: no. Answer after this ticket:
|
||||
yes, bit-for-bit identical across Python ↔ C ↔ asm.
|
||||
|
||||
## Goals
|
||||
|
||||
1. Pick one deterministic, portable PRNG and bind it in all three impls.
|
||||
2. Expose a minimal builtin API: seed, draw float, draw bounded int,
|
||||
introspect/set state.
|
||||
3. Extend portal-v1 so save/resume round-trips RNG state.
|
||||
4. Prove cross-impl reproducibility with a test: seed `k`, draw `N`
|
||||
values, save, clear, resume in any impl, draw `M` more — full stream
|
||||
matches a single-process Python baseline.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Statistical quality beyond xoshiro256\*\*'s documented properties.
|
||||
- Thread-local RNG (one global stream per process; matches current
|
||||
`__thread` portal checkpoint discipline in C).
|
||||
- Cryptographic strength.
|
||||
- Rejection-sampling uniformity for `(random-int n)` — plain modulo,
|
||||
bias acceptable for `n << 2^64`.
|
||||
|
||||
## Algorithm — xoshiro256\*\*
|
||||
|
||||
State: four `uint64_t` words `s[0..3]`.
|
||||
|
||||
```
|
||||
rotl(x, k) = (x << k) | (x >> (64 - k))
|
||||
|
||||
next():
|
||||
result = rotl(s[1] * 5, 7) * 9
|
||||
t = s[1] << 17
|
||||
s[2] ^= s[0]
|
||||
s[3] ^= s[1]
|
||||
s[1] ^= s[2]
|
||||
s[0] ^= s[3]
|
||||
s[2] ^= t
|
||||
s[3] = rotl(s[3], 45)
|
||||
return result
|
||||
```
|
||||
|
||||
Seeding uses splitmix64 from a single 64-bit seed:
|
||||
|
||||
```
|
||||
splitmix64(&z):
|
||||
z += 0x9e3779b97f4a7c15
|
||||
z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9
|
||||
z = (z ^ (z >> 27)) * 0x94d049bb133111eb
|
||||
return z ^ (z >> 31)
|
||||
|
||||
seed(k):
|
||||
z = k
|
||||
for i in 0..3: s[i] = splitmix64(&z)
|
||||
```
|
||||
|
||||
Chosen because:
|
||||
- Public domain reference (Blackman & Vigna, 2018).
|
||||
- Identical bit output on any machine — no libc, no FPU, no `rand()`.
|
||||
- Four u64 words → same wire format across Python, C, asm.
|
||||
- Small code footprint — asm port is ~40 instructions.
|
||||
|
||||
## Builtin API
|
||||
|
||||
| builtin | args | returns |
|
||||
|---|---|---|
|
||||
| `(random-seed! k)` | int `k` | void — reseeds from `k` via splitmix64 |
|
||||
| `(random)` | — | float in `[0.0, 1.0)` — top 53 bits / 2^53 |
|
||||
| `(random-int n)` | positive int `n` | int in `[0, n)` — raw modulo |
|
||||
| `(random-state)` | — | list of 8 ints: `(w0_lo w0_hi w1_lo w1_hi w2_lo w2_hi w3_lo w3_hi)` each in `[0, 2^32)` |
|
||||
| `(random-state! s)` | list of 8 ints | void — restores state from the list |
|
||||
|
||||
Rationale for the 8-int representation: each 64-bit word split into low
|
||||
and high 32-bit halves. Fits Lumbda's 48-bit C int and 61-bit asm int
|
||||
without widening. Portable over the S-expression portal too.
|
||||
|
||||
Default seed at interpreter startup: `0` (all impls). Deterministic from
|
||||
process start means tests don't need to seed explicitly unless they
|
||||
want a specific stream.
|
||||
|
||||
## Portal format changes
|
||||
|
||||
### JSON (Python + C) — lumbda-portal-v1
|
||||
|
||||
Add optional top-level key `"rng"` to the existing `lumbda-portal-v1`
|
||||
envelope. No magic bump required — absent field means "assume seed 0".
|
||||
|
||||
```json
|
||||
{
|
||||
"format": "lumbda-portal-v1",
|
||||
"env": {...},
|
||||
"continuation": ...,
|
||||
"rng": {
|
||||
"algo": "xoshiro256**",
|
||||
"state": [w0_lo, w0_hi, w1_lo, w1_hi, w2_lo, w2_hi, w3_lo, w3_hi]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Asm binary — LUMBDAB2
|
||||
|
||||
Bump magic from `LUMBDAB1` to `LUMBDAB2`. Extend `PORTAL_HDR_SIZE` from
|
||||
48 to 80 bytes: append 32 bytes of raw RNG state (4 × u64, little-endian,
|
||||
native layout).
|
||||
|
||||
Old `LUMBDAB1` files rejected during resume. (Portal is not yet
|
||||
externally released, so no migration burden.)
|
||||
|
||||
### Asm text (GC build) — `;; lumbda-portal v1`
|
||||
|
||||
Append one comment line after the header:
|
||||
|
||||
```
|
||||
;; lumbda-portal v1
|
||||
;; rng xoshiro256** w0_lo w0_hi w1_lo w1_hi w2_lo w2_hi w3_lo w3_hi
|
||||
...
|
||||
```
|
||||
|
||||
Parser recognizes the `;; rng ` prefix and calls `random-state!`
|
||||
internally.
|
||||
|
||||
## Test plan
|
||||
|
||||
1. **Unit** — seed `42`, first 5 draws match a hardcoded Python baseline
|
||||
bit-for-bit. Asserted in all three impls.
|
||||
2. **Functional** — `tests/functional.lsp` gets determinism tests.
|
||||
3. **Portal round-trip** — `tests/portal-rng-save.lsp` seeds, draws 50,
|
||||
saves, exits. `tests/portal-rng-load.lsp` resumes, draws 50 more,
|
||||
compares against full 100-draw baseline.
|
||||
4. **Cross-impl** — wire into `tests/portal-cross-test.sh`. Python saves,
|
||||
C resumes, next draws match. All 9 producer×consumer cells.
|
||||
5. **MOAD scan** — `~/git/unmoad.com/unmoad` on all changed files.
|
||||
|
||||
## Deliverables
|
||||
|
||||
- [ ] `docs/tickets/0001-portal-rng.md` (this ticket)
|
||||
- [ ] Python: `lumbda.py` — xoshiro256\*\* + 5 builtins + portal hooks
|
||||
- [ ] C: `c/builtins.c` + `c/portal.c` + `c/lumbda.h` — same
|
||||
- [ ] Asm: `asm/lumbda.s` — same, plus LUMBDAB2 header
|
||||
- [ ] Shared tests: `tests/functional.lsp` determinism block
|
||||
- [ ] Cross tests: `tests/portal-rng-save.lsp`, `tests/portal-rng-load.lsp`
|
||||
- [ ] `tests/portal-cross-test.sh` — new RNG cells
|
||||
- [ ] Asm test: `asm/test.sh` determinism checks
|
||||
- [ ] `make test-all` green
|
||||
- [ ] `unmoad` clean on all changed files
|
||||
78
lumbda.py
78
lumbda.py
|
|
@ -1959,6 +1959,73 @@ def load_compiled(path, env):
|
|||
return CompiledProc(code, params, rest, env, data.get('name'))
|
||||
|
||||
|
||||
###############################################################################
|
||||
# xoshiro256** — deterministic, portable PRNG shared with C and asm impls.
|
||||
# Portal serializes this state so simulations continue across processes with
|
||||
# a bit-identical random stream. Reference: Blackman & Vigna 2018.
|
||||
###############################################################################
|
||||
|
||||
_MASK64 = (1 << 64) - 1
|
||||
_rng_state = [0, 0, 0, 0]
|
||||
|
||||
|
||||
def _rng_splitmix64_step(z):
|
||||
"""Returns (output, next_counter). Reference: Vigna splitmix64.
|
||||
The persistent counter advances only by the constant; the mixing
|
||||
is on a local copy."""
|
||||
z = (z + 0x9e3779b97f4a7c15) & _MASK64
|
||||
r = z
|
||||
r = ((r ^ (r >> 30)) * 0xbf58476d1ce4e5b9) & _MASK64
|
||||
r = ((r ^ (r >> 27)) * 0x94d049bb133111eb) & _MASK64
|
||||
return (r ^ (r >> 31)) & _MASK64, z
|
||||
|
||||
|
||||
def _rng_seed(k):
|
||||
z = k & _MASK64
|
||||
for i in range(4):
|
||||
_rng_state[i], z = _rng_splitmix64_step(z)
|
||||
|
||||
|
||||
def _rng_next():
|
||||
s = _rng_state
|
||||
v = (s[1] * 5) & _MASK64
|
||||
result = ((((v << 7) & _MASK64) | (v >> 57)) * 9) & _MASK64
|
||||
t = (s[1] << 17) & _MASK64
|
||||
s[2] ^= s[0]
|
||||
s[3] ^= s[1]
|
||||
s[1] ^= s[2]
|
||||
s[0] ^= s[3]
|
||||
s[2] ^= t
|
||||
s[3] = (((s[3] << 45) & _MASK64) | (s[3] >> 19)) & _MASK64
|
||||
return result
|
||||
|
||||
|
||||
def _rng_random_float():
|
||||
return (_rng_next() >> 11) / (1 << 53)
|
||||
|
||||
|
||||
def _rng_random_int(n):
|
||||
if n <= 0: raise LispErr(f'random-int: n must be positive, got {n}')
|
||||
return _rng_next() % n
|
||||
|
||||
|
||||
def _rng_state_to_halves():
|
||||
out = []
|
||||
for w in _rng_state:
|
||||
out.append(w & 0xffffffff)
|
||||
out.append((w >> 32) & 0xffffffff)
|
||||
return out
|
||||
|
||||
|
||||
def _rng_state_from_halves(halves):
|
||||
if len(halves) != 8:
|
||||
raise LispErr('random-state!: expected list of 8 integers')
|
||||
for i in range(4):
|
||||
lo = halves[2 * i] & 0xffffffff
|
||||
hi = halves[2 * i + 1] & 0xffffffff
|
||||
_rng_state[i] = (hi << 32) | lo
|
||||
|
||||
|
||||
###############################################################################
|
||||
# Portal — Serialize/resume full machine state across machines
|
||||
###############################################################################
|
||||
|
|
@ -2221,6 +2288,7 @@ def portal_save(env, path, continuation=None):
|
|||
'env': ser.serialize_env(env),
|
||||
'continuation': ser.serialize_continuation(continuation) if continuation else None,
|
||||
'auto_compile': _auto_compile[0],
|
||||
'rng': {'algo': 'xoshiro256**', 'state': _rng_state_to_halves()},
|
||||
}
|
||||
state['objects'] = ser.finalize()
|
||||
with open(path, 'w') as f:
|
||||
|
|
@ -2239,6 +2307,9 @@ def portal_resume(path, base_env=None):
|
|||
des = _PortalDeserializer(base_env)
|
||||
des.rebuild_objects(state.get('objects', []))
|
||||
_auto_compile[0] = state.get('auto_compile', False)
|
||||
rng = state.get('rng')
|
||||
if rng and 'state' in rng:
|
||||
_rng_state_from_halves(rng['state'])
|
||||
cont = None
|
||||
if state.get('continuation'):
|
||||
cont = des.deserialize_value(state['continuation'])
|
||||
|
|
@ -3472,6 +3543,13 @@ def make_global_env():
|
|||
d(S('portal-resume'), lambda a, _: _portal_resume_builtin(_str_val(a[0]), g))
|
||||
d(S('portal-checkpoint!'), lambda a, _: _portal_checkpoint.__setitem__(0, _str_val(a[0])) or VOID)
|
||||
|
||||
# ── Random (xoshiro256**) — portal-serialized across all three impls ────
|
||||
d(S('random-seed!'), lambda a, _: _rng_seed(int(_num(a[0]))) or VOID)
|
||||
d(S('random'), lambda a, _: _rng_random_float())
|
||||
d(S('random-int'), lambda a, _: _rng_random_int(int(_num(a[0]))))
|
||||
d(S('random-state'), lambda a, _: _P(_rng_state_to_halves()))
|
||||
d(S('random-state!'), lambda a, _: _rng_state_from_halves([int(_num(x)) for x in _L(a[0])]) or VOID)
|
||||
|
||||
def _portal_resume_builtin(path, env):
|
||||
"""Resume from portal file, merging into current env."""
|
||||
_, cont = portal_resume(path, env)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue