From 1731304ed85de9911f1c29ade575f95f22044c47 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sun, 7 Jun 2026 20:19:32 -0400 Subject: [PATCH] c+py: file I/O primitives for multi-GB binary streams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ecdsa's Phase B emit at full secp256k1 width writes a 4–6 GB QECCOPS1 ops binary. The existing path — accumulate in a string-output port, materialize via get-output-string, write — peaks RAM at 3× body size (port internal buffer + Scheme string copy + write-binary-file concat). A 6 GB body needs ~18 GB transient; OOMs a 16 GB QEMU guest. This commit shifts emit-stream onto a constant-RAM file-port path and fixes binary-correctness defects in the supporting primitives. New primitives (mirrored across c/builtins.c + lumbda.py): - open-binary-output-file path Opens in "w+b" so the caller can seek back to rewrite a header. - port-set-position! port offset fseek absolute offset on a file port. emit-stream reserves a 16-byte placeholder header, streams the body, then seeks back to byte 0 to rewrite the QECCOPS1 + n_ops u64 LE once n_ops is known. - append-binary-file path data Opens in "ab" and fwrite's the bytes through. Pairs with write-binary-file so callers can land header + body in two writes instead of (string-append header body). - append-port-to-binary-file path port Streams a string-output port's buffer to disk via fwrite without materializing (get-output-string port). Lets callers keep their existing string-output sink and avoid the body-size string copy if they stay on string-port emit. Binary-correctness fixes: - bi_write_string to a file port used fputs, which calls strlen. Binary payloads containing 0x00 truncated at the first null byte. Switched the file-port branch to fwrite with the string's known ->len (same fix family as the earlier bi_get_output_string strlen defect). - bi_write_char per-byte fflush guarded to stdout only. With millions of gate-bytes per second, flushing after every fputc to a file port was a 100× slowdown. File ports buffer until close or explicit flush-port — keep stdout's per-byte feedback path, drop fflush on every file-port byte. - port_write_str grows 1.5× past 256 MB instead of 2× throughout. At realloc time the transient peak is old + new; 2× at 8 GB → 16 GB transient needs 24 GB. 1.5× bounds peak at 2.5× and keeps multi-GB string-port workloads inside a 16 GB VM. Tests: 88/88 c-test, 4/4 regression-named-let-leak, 205/205 functional, zoe-favorites all tiers. Binary roundtrip with embedded nulls at 10/1000/100000 bytes passes byte-for-byte. End-to-end: foxhop ecdsa DIALOG_GCD secp256k1 emit lands a 4.7 GB binary at 322 MB peak RSS in 7:41 wall on a 16 GB QEMU guest. --- c/builtins.c | 79 +++++++++++++++++++++++++++++++++++++++++++++++++--- c/types.c | 10 ++++++- lumbda.py | 54 +++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 5 deletions(-) diff --git a/c/builtins.c b/c/builtins.c index 16453ca..b736b7c 100644 --- a/c/builtins.c +++ b/c/builtins.c @@ -1147,6 +1147,31 @@ static Value bi_open_output_file(Value *a, int n, Env *e) { if (!f) lisp_error("open-output-file: cannot open: %s", AS_STRING(a[0])->data); return make_file_port(f, PORT_OUTPUT); } + +/* open-binary-output-file: like open-output-file but opens in w+b mode + * so emit-stream can stream gates directly to disk AND seek back to + * rewrite the header once n_ops is known. Pairs with port-set-position!. */ +static Value bi_open_binary_output_file(Value *a, int n, Env *e) { + (void)e; CHECK_ARITY("open-binary-output-file", 1); check_string(a[0]); + FILE *f = fopen(AS_STRING(a[0])->data, "w+b"); + if (!f) lisp_error("open-binary-output-file: cannot open: %s", AS_STRING(a[0])->data); + return make_file_port(f, PORT_OUTPUT); +} + +/* port-set-position!: fseek to absolute byte offset on a file port. + * Errors on string ports. Returns VOID. */ +static Value bi_port_set_position(Value *a, int n, Env *e) { + (void)e; CHECK_ARITY("port-set-position!", 2); + ULPort *p = AS_PORT(a[0]); + if (p->kind != PORT_FILE) { + lisp_error("port-set-position!: port must be a file port"); + } + long offset = (long)as_int(a[1]); + if (fseek(p->fp, offset, SEEK_SET) != 0) { + lisp_error("port-set-position!: fseek failed"); + } + return VAL_VOID; +} static Value bi_write_file(Value *a, int n, Env *e) { (void)e; CHECK_ARITY("write-file", 2); check_string(a[0]); check_string(a[1]); FILE *f = fopen(AS_STRING(a[0])->data, "w"); @@ -1367,6 +1392,38 @@ static Value bi_write_binary_file(Value *a, int n, Env *e) { return wrote == data->len ? VAL_VOID : VAL_FALSE; } +static Value bi_append_binary_file(Value *a, int n, Env *e) { + (void)e; CHECK_ARITY("append-binary-file", 2); + check_string(a[0]); check_string(a[1]); + const char *path = AS_STRING(a[0])->data; + ULString *data = AS_STRING(a[1]); + FILE *f = fopen(path, "ab"); + if (!f) return VAL_FALSE; + size_t wrote = fwrite(data->data, 1, data->len, f); + fclose(f); + return wrote == data->len ? VAL_VOID : VAL_FALSE; +} + +/* append-port-to-binary-file: stream a string-output port's buffer + * directly to a file via fwrite, skipping the Scheme-side string + * materialization (get-output-string + append-binary-file would copy + * the buffer twice — at multi-GB body sizes this blows past any VM RAM + * budget). The port stays usable after; caller closes it. */ +static Value bi_append_port_to_binary_file(Value *a, int n, Env *e) { + (void)e; CHECK_ARITY("append-port-to-binary-file", 2); + check_string(a[0]); + const char *path = AS_STRING(a[0])->data; + ULPort *p = AS_PORT(a[1]); + if (p->kind != PORT_STRING || p->dir != PORT_OUTPUT) { + lisp_error("append-port-to-binary-file: port must be a string output port"); + } + FILE *f = fopen(path, "ab"); + if (!f) return VAL_FALSE; + size_t wrote = fwrite(p->str_buf, 1, p->str_len, f); + fclose(f); + return wrote == p->str_len ? VAL_VOID : VAL_FALSE; +} + static Value bi_read_binary_file(Value *a, int n, Env *e) { (void)e; CHECK_ARITY("read-binary-file", 1); check_string(a[0]); @@ -2362,12 +2419,18 @@ static Value bi_procedure_name(Value *a, int n, Env *e) { static Value bi_write_string(Value *a, int n, Env *e) { (void)e; CHECK_MIN_ARITY("write-string", 1); check_string(a[0]); - FILE *out = n > 1 && IS_PORT(a[1]) ? AS_PORT(a[1])->fp : stdout; + ULString *s = AS_STRING(a[0]); if (n > 1 && IS_PORT(a[1]) && AS_PORT(a[1])->kind == PORT_STRING) { - ULString *s = AS_STRING(a[0]); port_write_str(AS_PORT(a[1]), s->data, s->len); + } else if (n > 1 && IS_PORT(a[1]) && AS_PORT(a[1])->kind == PORT_FILE) { + /* Use fwrite with known s->len — fputs strlen-truncates and + * loses any 0x00 in the payload (breaks binary emit through + * a file port). */ + FILE *out = AS_PORT(a[1])->fp; + fwrite(s->data, 1, s->len, out); } else { - fputs(AS_STRING(a[0])->data, out ? out : stdout); + FILE *out = (n > 1 && IS_PORT(a[1])) ? AS_PORT(a[1])->fp : stdout; + fwrite(s->data, 1, s->len, out ? out : stdout); fflush(out ? out : stdout); } return VAL_VOID; @@ -2385,7 +2448,11 @@ static Value bi_write_char(Value *a, int n, Env *e) { } FILE *out = n > 1 && IS_PORT(a[1]) ? AS_PORT(a[1])->fp : stdout; fputc(AS_CHAR(a[0]), out ? out : stdout); - fflush(out ? out : stdout); + /* Per-byte fflush only when writing to stdout for interactive REPL + * feedback. File ports buffer until close (or explicit flush-port) + * — emit-stream calls write-char millions of times per second and + * per-write fflush would tank throughput. */ + if (!(n > 1 && IS_PORT(a[1]))) fflush(stdout); return VAL_VOID; } @@ -2619,6 +2686,8 @@ Env *make_global_env(void) { DEF("read-line", bi_read_line); DEF("read-char", bi_read_char); DEF("open-input-file", bi_open_input_file); DEF("open-output-file", bi_open_output_file); + DEF("open-binary-output-file", bi_open_binary_output_file); + DEF("port-set-position!", bi_port_set_position); DEF("write-file", bi_write_file); DEF("file->string", bi_file_to_string); DEF("tcp-listen", bi_tcp_listen); @@ -2630,6 +2699,8 @@ Env *make_global_env(void) { DEF("spawn-process-stdio", bi_spawn_process_stdio); DEF("flush-port", bi_flush_port); DEF("write-binary-file", bi_write_binary_file); + DEF("append-binary-file", bi_append_binary_file); + DEF("append-port-to-binary-file", bi_append_port_to_binary_file); DEF("read-binary-file", bi_read_binary_file); DEF("walk-circuit-ops", bi_walk_circuit_ops); DEF("op-specs->bytes", bi_op_specs_to_bytes); diff --git a/c/types.c b/c/types.c index 124fade..d99415c 100644 --- a/c/types.c +++ b/c/types.c @@ -788,8 +788,16 @@ void port_write_str(ULPort *p, const char *s, size_t len) { if (p->kind == PORT_FILE) { fwrite(s, 1, len, p->fp); } else { + /* Grow with 2x doubling on small buffers (fast amortization), shift + * to 1.5x once we cross 256 MB. 2x at that scale creates a 3x peak + * during realloc (old + new) — 8GB → 16GB needs 24GB transient, + * OOMs any reasonable VM. 1.5x bounds peak at 2.5x. */ while (p->str_len + len + 1 > p->str_cap) { - p->str_cap *= 2; + if (p->str_cap < (1ULL << 28)) { + p->str_cap *= 2; + } else { + p->str_cap += p->str_cap / 2; + } p->str_buf = (char *)ul_realloc(p->str_buf, p->str_cap); } memcpy(p->str_buf + p->str_len, s, len); diff --git a/lumbda.py b/lumbda.py index 1a62c58..0dc20cc 100644 --- a/lumbda.py +++ b/lumbda.py @@ -2853,6 +2853,56 @@ def _write_binary_file(path, data): f.write(data.encode('latin-1') if isinstance(data, str) else bytes(data)) return VOID +def _append_binary_file(path, data): + """Append Latin-1-encoded string to a binary file byte-for-byte. + Pairs with write-binary-file so emit-stream can land header + body + without materializing a string-append concat at production widths + (multi-GB body would peak host RAM at 3x body size otherwise).""" + with open(path, 'ab') as f: + f.write(data.encode('latin-1') if isinstance(data, str) else bytes(data)) + return VOID + +def _append_port_to_binary_file(path, port): + """Stream a string-output port's accumulated chunks directly to a + file. Avoids materializing (get-output-string port) — at multi-GB + body sizes that copy peaks host RAM unnecessarily.""" + if not isinstance(port, StringOutputPort): + raise LispErr("append-port-to-binary-file: port must be a string output port") + with open(path, 'ab') as f: + for chunk in port._buf: + f.write(chunk.encode('latin-1') if isinstance(chunk, str) else bytes(chunk)) + return VOID + +class BinaryFilePort: + """Wraps a Python binary file so write-char / write-string can pass + Scheme strings (str or MutableString) without an explicit encode. + Mirrors C tier's open-binary-output-file + port-set-position! pair + so emit-stream can write gates directly to disk without a string + output buffer.""" + __slots__ = ('_f',) + def __init__(self, path): self._f = open(path, 'w+b') + def write(self, s): + if isinstance(s, MutableString): + s = ''.join(s._c) + if isinstance(s, str): + data = s.encode('latin-1') + elif isinstance(s, (bytes, bytearray)): + data = bytes(s) + else: + data = str(s).encode('latin-1') + self._f.write(data) + return len(data) + def flush(self): self._f.flush() + def close(self): self._f.close() + def seek(self, offset): self._f.seek(offset) + +def _open_binary_output_file(path): + return BinaryFilePort(path) + +def _port_set_position(port, offset): + port.seek(int(offset)) + return VOID + def _read_binary_file(path): """Read a binary file as a Latin-1 string (1:1 byte mapping).""" with open(path, 'rb') as f: @@ -3757,6 +3807,8 @@ def make_global_env(): d(S('read'), lambda a, _: _read_datum_port(a[0] if a else None)) d(S('open-input-file'), lambda a, _: open(_str_val(a[0]))) d(S('open-output-file'), lambda a, _: open(_str_val(a[0]), 'w')) + d(S('open-binary-output-file'), lambda a, _: _open_binary_output_file(_str_val(a[0]))) + d(S('port-set-position!'), lambda a, _: _port_set_position(a[0], a[1])) d(S('write-file'), lambda a, _: _write_file(_str_val(a[0]), _str_val(a[1]))) d(S('file->string'), lambda a, _: _read_file_to_string(_str_val(a[0]))) d(S('tcp-listen'), lambda a, _: _tcp_listen(int(a[0]))) @@ -3776,6 +3828,8 @@ def make_global_env(): lambda a, _: _spawn_process_stdio(_str_val(a[0]), _spawn_args(a))) d(S('flush-port'), lambda a, _: _flush_port(a[0])) d(S('write-binary-file'), lambda a, _: _write_binary_file(_str_val(a[0]), _str_val(a[1]))) + d(S('append-binary-file'), lambda a, _: _append_binary_file(_str_val(a[0]), _str_val(a[1]))) + d(S('append-port-to-binary-file'), lambda a, _: _append_port_to_binary_file(_str_val(a[0]), a[1])) d(S('read-binary-file'), lambda a, _: _read_binary_file(_str_val(a[0]))) d(S('walk-circuit-ops'), lambda a, _: _walk_circuit_ops(a[0], a[1])) d(S('op-specs->bytes'), lambda a, _: _op_specs_to_bytes(a[0]))