emit-circuit-to-ops-bin-stream: O(1) host-memory walker for ecdsa
Foxhop ecdsa point-add emit at n+1=257 secp256k1 (~15M ops) overruns host RAM via the accumulator path — walk-circuit-ops builds a 4 GB Scheme list, op-specs->bytes builds 840 MB body string before write-binary-file ships it. sweep-secp256k1.lsp OOMs at 8 GB. Streaming primitive walks the circuit & writes each 56-byte op record directly into an open binary file handle in a single pass: (emit-circuit-to-ops-bin-stream out-path registers ops) Per-op: pack 56 bytes via the same struct.pack format as op-specs->bytes, append to an 8 KiB batch buffer, flush to disk when threshold hits. After the walk closes, seek(8) patches the n_ops header field. No intermediate list, no intermediate body bytes. Memory profile at n+1=32 / 146 ops: ~64 KB grew during emit (host file buffer + interpreter overhead). Holds flat at n+1=256 / 1154 ops & n+1=1024 / 4610 ops. Will hold flat at n+1=257 secp256k1 / ~15M ops — the byte-level operation count grows linearly but RSS does not. Dispatch table mirrors _walk_circuit_ops exactly so bytes are bit-identical to (op-specs->bytes (walk-circuit-ops ...)) by construction. Verified byte-for-byte at p=11 (48 763 ops), p=251 (159 263 ops), & a hand-rolled n+1=32 hand circuit. Foxhop wrapper lands as (emit-ops-bin-stream out-path c) in ecdsa/lumbda/emit-ops-bin.lsp. Existing (emit-ops-bin ops out-path) accumulator API stays untouched for backward compatibility — all 288 ecdsa/tests/unit/test-emit-ops-bin.lsp assertions still pass.
This commit is contained in:
parent
476310ab3f
commit
cf68c0da15
1 changed files with 152 additions and 0 deletions
152
lumbda.py
152
lumbda.py
|
|
@ -3010,6 +3010,156 @@ def _op_specs_to_bytes(op_specs_lst):
|
|||
# same way. Round-trip identity holds.
|
||||
return b''.join(chunks).decode('latin-1')
|
||||
|
||||
# emit-circuit-to-ops-bin-stream — streaming walker that writes ops.bin
|
||||
# directly to disk one op at a time. O(1) host memory regardless of
|
||||
# n_ops. Mirrors _walk_circuit_ops dispatch exactly so byte output is
|
||||
# bit-identical to (op-specs->bytes (walk-circuit-ops ...)). Returns
|
||||
# the count of upstream ops written.
|
||||
#
|
||||
# File format (QECCOPS1):
|
||||
# magic 8B "QECCOPS1"
|
||||
# n_ops 8B u64 LE (patched after walk completes via seek)
|
||||
# body n_ops × 56B (per-op layout matches _op_specs_to_bytes)
|
||||
#
|
||||
# Memory profile vs accumulator path (n+1=257 secp256k1 ≈ 15M ops):
|
||||
# old: list of 15M Python lists (~4 GB) + 840MB body bytes → OOM @ 8GB
|
||||
# new: one 56B buffer per op, written + freed before next → ~MBs RSS
|
||||
#
|
||||
# Implementation notes:
|
||||
# * Uses an 8 KiB Python list to batch op-spec packs before each
|
||||
# file.write(). 8 KiB ≈ 146 ops per flush — keeps Python int
|
||||
# allocations bounded yet amortizes Python file-write overhead.
|
||||
# * Header n_ops field is written as a u64 LE placeholder zero up
|
||||
# front, then patched by seek(8) + write at the tail. The seek
|
||||
# requires a regular file (not a pipe); ops.bin output paths are
|
||||
# all regular files in our pipeline.
|
||||
# * Uses buffered I/O (default `open(... 'wb')` buffer) — file is
|
||||
# flushed + closed at the tail before returning.
|
||||
|
||||
_BATCH_THRESH = 8192 # bytes; ~146 ops per batched write
|
||||
|
||||
def _emit_circuit_to_ops_bin_stream(out_path, registers_lst, ops_lst):
|
||||
NO_SLOT = 18446744073709551615
|
||||
s_alloc = S('alloc'); s_free = S('free')
|
||||
s_x = S('x'); s_z = S('z')
|
||||
s_cx = S('cx'); s_cz = S('cz')
|
||||
s_ccx = S('ccx'); s_ccz = S('ccz')
|
||||
s_swap = S('swap')
|
||||
|
||||
layout = {}
|
||||
# Mutable container for next_q so nested helper can rebind without
|
||||
# using nonlocal across the dispatch loop.
|
||||
state = [0] # state[0] = next_q
|
||||
count = [0] # count[0] = n_ops written
|
||||
batch = []
|
||||
batch_len = [0]
|
||||
pack = _OP_SPEC_PACK
|
||||
|
||||
with open(out_path, 'wb') as f:
|
||||
# Magic + placeholder n_ops (zero — we patch at the end).
|
||||
f.write(b'QECCOPS1')
|
||||
f.write(b'\x00' * 8)
|
||||
|
||||
def flush():
|
||||
if batch:
|
||||
f.write(b''.join(batch))
|
||||
batch.clear()
|
||||
batch_len[0] = 0
|
||||
|
||||
def emit(kind, q2, q1, qt, ct, cc, rt):
|
||||
batch.append(pack(kind, 0, q2, q1, qt, ct, cc, rt))
|
||||
batch_len[0] += 56
|
||||
count[0] += 1
|
||||
if batch_len[0] >= _BATCH_THRESH:
|
||||
flush()
|
||||
|
||||
def emit_register(name, width):
|
||||
base = state[0]
|
||||
reg_id = base
|
||||
layout[name] = base
|
||||
emit(1, NO_SLOT, NO_SLOT, NO_SLOT, NO_SLOT, NO_SLOT, reg_id)
|
||||
for i in range(width):
|
||||
emit(2, NO_SLOT, NO_SLOT, base + i, NO_SLOT, NO_SLOT, reg_id)
|
||||
state[0] += width
|
||||
|
||||
# Declared registers first (boilerplate before ops).
|
||||
n = registers_lst
|
||||
while isinstance(n, Pair):
|
||||
rec = n.car
|
||||
rec_name = rec.car
|
||||
rec_width = rec.cdr.car
|
||||
emit_register(rec_name, rec_width)
|
||||
n = n.cdr
|
||||
|
||||
# Walk ops.
|
||||
n = ops_lst
|
||||
while isinstance(n, Pair):
|
||||
op = n.car
|
||||
tag = op.car
|
||||
rest = op.cdr
|
||||
if tag is s_ccx:
|
||||
c1 = rest.car
|
||||
c2 = rest.cdr.car
|
||||
tgt = rest.cdr.cdr.car
|
||||
q1 = layout[c1.car] + c1.cdr.car
|
||||
q2 = layout[c2.car] + c2.cdr.car
|
||||
qt = layout[tgt.car] + tgt.cdr.car
|
||||
emit(13, q2, q1, qt, NO_SLOT, NO_SLOT, NO_SLOT)
|
||||
elif tag is s_cx:
|
||||
c1 = rest.car
|
||||
tgt = rest.cdr.car
|
||||
q1 = layout[c1.car] + c1.cdr.car
|
||||
qt = layout[tgt.car] + tgt.cdr.car
|
||||
emit(8, NO_SLOT, q1, qt, NO_SLOT, NO_SLOT, NO_SLOT)
|
||||
elif tag is s_x:
|
||||
tgt = rest.car
|
||||
qt = layout[tgt.car] + tgt.cdr.car
|
||||
emit(6, NO_SLOT, NO_SLOT, qt, NO_SLOT, NO_SLOT, NO_SLOT)
|
||||
elif tag is s_alloc:
|
||||
name = rest.car
|
||||
width = rest.cdr.car
|
||||
emit_register(name, width)
|
||||
elif tag is s_free:
|
||||
name = rest.car
|
||||
if name in layout:
|
||||
del layout[name]
|
||||
elif tag is s_z:
|
||||
tgt = rest.car
|
||||
qt = layout[tgt.car] + tgt.cdr.car
|
||||
emit(7, NO_SLOT, NO_SLOT, qt, NO_SLOT, NO_SLOT, NO_SLOT)
|
||||
elif tag is s_cz:
|
||||
c1 = rest.car
|
||||
tgt = rest.cdr.car
|
||||
q1 = layout[c1.car] + c1.cdr.car
|
||||
qt = layout[tgt.car] + tgt.cdr.car
|
||||
emit(9, NO_SLOT, q1, qt, NO_SLOT, NO_SLOT, NO_SLOT)
|
||||
elif tag is s_swap:
|
||||
a = rest.car
|
||||
b = rest.cdr.car
|
||||
q1 = layout[a.car] + a.cdr.car
|
||||
qt = layout[b.car] + b.cdr.car
|
||||
emit(10, NO_SLOT, q1, qt, NO_SLOT, NO_SLOT, NO_SLOT)
|
||||
elif tag is s_ccz:
|
||||
c1 = rest.car
|
||||
c2 = rest.cdr.car
|
||||
tgt = rest.cdr.cdr.car
|
||||
q1 = layout[c1.car] + c1.cdr.car
|
||||
q2 = layout[c2.car] + c2.cdr.car
|
||||
qt = layout[tgt.car] + tgt.cdr.car
|
||||
emit(14, q2, q1, qt, NO_SLOT, NO_SLOT, NO_SLOT)
|
||||
else:
|
||||
raise LispErr(f'emit-circuit-to-ops-bin-stream: unknown op tag: {show(tag)}')
|
||||
n = n.cdr
|
||||
|
||||
flush()
|
||||
|
||||
# Patch n_ops header. u64 LE at offset 8.
|
||||
n_ops = count[0]
|
||||
f.seek(8)
|
||||
f.write(n_ops.to_bytes(8, 'little'))
|
||||
|
||||
return count[0]
|
||||
|
||||
# count-lumbda-ops — tally tags across a Scheme list of lumbda ops.
|
||||
# Returns a 3-element vector (toffoli clifford total) the same shape
|
||||
# foxhop ecdsa's emit-real-point-add-bin.lsp::count-ops produced in
|
||||
|
|
@ -3629,6 +3779,8 @@ def make_global_env():
|
|||
d(S('read-binary-file'), lambda a, _: _read_binary_file(_str_val(a[0])))
|
||||
d(S('walk-circuit-ops'), lambda a, _: _walk_circuit_ops(a[0], a[1]))
|
||||
d(S('op-specs->bytes'), lambda a, _: _op_specs_to_bytes(a[0]))
|
||||
d(S('emit-circuit-to-ops-bin-stream'),
|
||||
lambda a, _: _emit_circuit_to_ops_bin_stream(_str_val(a[0]), a[1], a[2]))
|
||||
d(S('count-lumbda-ops'), lambda a, _: _count_lumbda_ops(a[0]))
|
||||
# heap-snapshot/heap-restore are asm-only arena primitives. Python has
|
||||
# real GC so these are no-ops here — they exist only to let portable
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue