diff --git a/lumbda.py b/lumbda.py index 72e49f9..1a62c58 100644 --- a/lumbda.py +++ b/lumbda.py @@ -3010,6 +3010,156 @@ def _op_specs_to_bytes(op_specs_lst): # same way. Round-trip identity holds. return b''.join(chunks).decode('latin-1') +# emit-circuit-to-ops-bin-stream — streaming walker that writes ops.bin +# directly to disk one op at a time. O(1) host memory regardless of +# n_ops. Mirrors _walk_circuit_ops dispatch exactly so byte output is +# bit-identical to (op-specs->bytes (walk-circuit-ops ...)). Returns +# the count of upstream ops written. +# +# File format (QECCOPS1): +# magic 8B "QECCOPS1" +# n_ops 8B u64 LE (patched after walk completes via seek) +# body n_ops × 56B (per-op layout matches _op_specs_to_bytes) +# +# Memory profile vs accumulator path (n+1=257 secp256k1 ≈ 15M ops): +# old: list of 15M Python lists (~4 GB) + 840MB body bytes → OOM @ 8GB +# new: one 56B buffer per op, written + freed before next → ~MBs RSS +# +# Implementation notes: +# * Uses an 8 KiB Python list to batch op-spec packs before each +# file.write(). 8 KiB ≈ 146 ops per flush — keeps Python int +# allocations bounded yet amortizes Python file-write overhead. +# * Header n_ops field is written as a u64 LE placeholder zero up +# front, then patched by seek(8) + write at the tail. The seek +# requires a regular file (not a pipe); ops.bin output paths are +# all regular files in our pipeline. +# * Uses buffered I/O (default `open(... 'wb')` buffer) — file is +# flushed + closed at the tail before returning. + +_BATCH_THRESH = 8192 # bytes; ~146 ops per batched write + +def _emit_circuit_to_ops_bin_stream(out_path, registers_lst, ops_lst): + NO_SLOT = 18446744073709551615 + s_alloc = S('alloc'); s_free = S('free') + s_x = S('x'); s_z = S('z') + s_cx = S('cx'); s_cz = S('cz') + s_ccx = S('ccx'); s_ccz = S('ccz') + s_swap = S('swap') + + layout = {} + # Mutable container for next_q so nested helper can rebind without + # using nonlocal across the dispatch loop. + state = [0] # state[0] = next_q + count = [0] # count[0] = n_ops written + batch = [] + batch_len = [0] + pack = _OP_SPEC_PACK + + with open(out_path, 'wb') as f: + # Magic + placeholder n_ops (zero — we patch at the end). + f.write(b'QECCOPS1') + f.write(b'\x00' * 8) + + def flush(): + if batch: + f.write(b''.join(batch)) + batch.clear() + batch_len[0] = 0 + + def emit(kind, q2, q1, qt, ct, cc, rt): + batch.append(pack(kind, 0, q2, q1, qt, ct, cc, rt)) + batch_len[0] += 56 + count[0] += 1 + if batch_len[0] >= _BATCH_THRESH: + flush() + + def emit_register(name, width): + base = state[0] + reg_id = base + layout[name] = base + emit(1, NO_SLOT, NO_SLOT, NO_SLOT, NO_SLOT, NO_SLOT, reg_id) + for i in range(width): + emit(2, NO_SLOT, NO_SLOT, base + i, NO_SLOT, NO_SLOT, reg_id) + state[0] += width + + # Declared registers first (boilerplate before ops). + n = registers_lst + while isinstance(n, Pair): + rec = n.car + rec_name = rec.car + rec_width = rec.cdr.car + emit_register(rec_name, rec_width) + n = n.cdr + + # Walk ops. + n = ops_lst + while isinstance(n, Pair): + op = n.car + tag = op.car + rest = op.cdr + if tag is s_ccx: + c1 = rest.car + c2 = rest.cdr.car + tgt = rest.cdr.cdr.car + q1 = layout[c1.car] + c1.cdr.car + q2 = layout[c2.car] + c2.cdr.car + qt = layout[tgt.car] + tgt.cdr.car + emit(13, q2, q1, qt, NO_SLOT, NO_SLOT, NO_SLOT) + elif tag is s_cx: + c1 = rest.car + tgt = rest.cdr.car + q1 = layout[c1.car] + c1.cdr.car + qt = layout[tgt.car] + tgt.cdr.car + emit(8, NO_SLOT, q1, qt, NO_SLOT, NO_SLOT, NO_SLOT) + elif tag is s_x: + tgt = rest.car + qt = layout[tgt.car] + tgt.cdr.car + emit(6, NO_SLOT, NO_SLOT, qt, NO_SLOT, NO_SLOT, NO_SLOT) + elif tag is s_alloc: + name = rest.car + width = rest.cdr.car + emit_register(name, width) + elif tag is s_free: + name = rest.car + if name in layout: + del layout[name] + elif tag is s_z: + tgt = rest.car + qt = layout[tgt.car] + tgt.cdr.car + emit(7, NO_SLOT, NO_SLOT, qt, NO_SLOT, NO_SLOT, NO_SLOT) + elif tag is s_cz: + c1 = rest.car + tgt = rest.cdr.car + q1 = layout[c1.car] + c1.cdr.car + qt = layout[tgt.car] + tgt.cdr.car + emit(9, NO_SLOT, q1, qt, NO_SLOT, NO_SLOT, NO_SLOT) + elif tag is s_swap: + a = rest.car + b = rest.cdr.car + q1 = layout[a.car] + a.cdr.car + qt = layout[b.car] + b.cdr.car + emit(10, NO_SLOT, q1, qt, NO_SLOT, NO_SLOT, NO_SLOT) + elif tag is s_ccz: + c1 = rest.car + c2 = rest.cdr.car + tgt = rest.cdr.cdr.car + q1 = layout[c1.car] + c1.cdr.car + q2 = layout[c2.car] + c2.cdr.car + qt = layout[tgt.car] + tgt.cdr.car + emit(14, q2, q1, qt, NO_SLOT, NO_SLOT, NO_SLOT) + else: + raise LispErr(f'emit-circuit-to-ops-bin-stream: unknown op tag: {show(tag)}') + n = n.cdr + + flush() + + # Patch n_ops header. u64 LE at offset 8. + n_ops = count[0] + f.seek(8) + f.write(n_ops.to_bytes(8, 'little')) + + return count[0] + # count-lumbda-ops — tally tags across a Scheme list of lumbda ops. # Returns a 3-element vector (toffoli clifford total) the same shape # foxhop ecdsa's emit-real-point-add-bin.lsp::count-ops produced in @@ -3629,6 +3779,8 @@ def make_global_env(): d(S('read-binary-file'), lambda a, _: _read_binary_file(_str_val(a[0]))) d(S('walk-circuit-ops'), lambda a, _: _walk_circuit_ops(a[0], a[1])) d(S('op-specs->bytes'), lambda a, _: _op_specs_to_bytes(a[0])) + d(S('emit-circuit-to-ops-bin-stream'), + lambda a, _: _emit_circuit_to_ops_bin_stream(_str_val(a[0]), a[1], a[2])) d(S('count-lumbda-ops'), lambda a, _: _count_lumbda_ops(a[0])) # heap-snapshot/heap-restore are asm-only arena primitives. Python has # real GC so these are no-ops here — they exist only to let portable