c+py: file I/O primitives for multi-GB binary streams
ecdsa's Phase B emit at full secp256k1 width writes a 4–6 GB QECCOPS1 ops binary. The existing path — accumulate in a string-output port, materialize via get-output-string, write — peaks RAM at 3× body size (port internal buffer + Scheme string copy + write-binary-file concat). A 6 GB body needs ~18 GB transient; OOMs a 16 GB QEMU guest. This commit shifts emit-stream onto a constant-RAM file-port path and fixes binary-correctness defects in the supporting primitives. New primitives (mirrored across c/builtins.c + lumbda.py): - open-binary-output-file path Opens in "w+b" so the caller can seek back to rewrite a header. - port-set-position! port offset fseek absolute offset on a file port. emit-stream reserves a 16-byte placeholder header, streams the body, then seeks back to byte 0 to rewrite the QECCOPS1 + n_ops u64 LE once n_ops is known. - append-binary-file path data Opens in "ab" and fwrite's the bytes through. Pairs with write-binary-file so callers can land header + body in two writes instead of (string-append header body). - append-port-to-binary-file path port Streams a string-output port's buffer to disk via fwrite without materializing (get-output-string port). Lets callers keep their existing string-output sink and avoid the body-size string copy if they stay on string-port emit. Binary-correctness fixes: - bi_write_string to a file port used fputs, which calls strlen. Binary payloads containing 0x00 truncated at the first null byte. Switched the file-port branch to fwrite with the string's known ->len (same fix family as the earlier bi_get_output_string strlen defect). - bi_write_char per-byte fflush guarded to stdout only. With millions of gate-bytes per second, flushing after every fputc to a file port was a 100× slowdown. File ports buffer until close or explicit flush-port — keep stdout's per-byte feedback path, drop fflush on every file-port byte. - port_write_str grows 1.5× past 256 MB instead of 2× throughout. At realloc time the transient peak is old + new; 2× at 8 GB → 16 GB transient needs 24 GB. 1.5× bounds peak at 2.5× and keeps multi-GB string-port workloads inside a 16 GB VM. Tests: 88/88 c-test, 4/4 regression-named-let-leak, 205/205 functional, zoe-favorites all tiers. Binary roundtrip with embedded nulls at 10/1000/100000 bytes passes byte-for-byte. End-to-end: foxhop ecdsa DIALOG_GCD secp256k1 emit lands a 4.7 GB binary at 322 MB peak RSS in 7:41 wall on a 16 GB QEMU guest.
This commit is contained in:
parent
7b6643d1fc
commit
1731304ed8
3 changed files with 138 additions and 5 deletions
79
c/builtins.c
79
c/builtins.c
|
|
@ -1147,6 +1147,31 @@ static Value bi_open_output_file(Value *a, int n, Env *e) {
|
|||
if (!f) lisp_error("open-output-file: cannot open: %s", AS_STRING(a[0])->data);
|
||||
return make_file_port(f, PORT_OUTPUT);
|
||||
}
|
||||
|
||||
/* open-binary-output-file: like open-output-file but opens in w+b mode
|
||||
* so emit-stream can stream gates directly to disk AND seek back to
|
||||
* rewrite the header once n_ops is known. Pairs with port-set-position!. */
|
||||
static Value bi_open_binary_output_file(Value *a, int n, Env *e) {
|
||||
(void)e; CHECK_ARITY("open-binary-output-file", 1); check_string(a[0]);
|
||||
FILE *f = fopen(AS_STRING(a[0])->data, "w+b");
|
||||
if (!f) lisp_error("open-binary-output-file: cannot open: %s", AS_STRING(a[0])->data);
|
||||
return make_file_port(f, PORT_OUTPUT);
|
||||
}
|
||||
|
||||
/* port-set-position!: fseek to absolute byte offset on a file port.
|
||||
* Errors on string ports. Returns VOID. */
|
||||
static Value bi_port_set_position(Value *a, int n, Env *e) {
|
||||
(void)e; CHECK_ARITY("port-set-position!", 2);
|
||||
ULPort *p = AS_PORT(a[0]);
|
||||
if (p->kind != PORT_FILE) {
|
||||
lisp_error("port-set-position!: port must be a file port");
|
||||
}
|
||||
long offset = (long)as_int(a[1]);
|
||||
if (fseek(p->fp, offset, SEEK_SET) != 0) {
|
||||
lisp_error("port-set-position!: fseek failed");
|
||||
}
|
||||
return VAL_VOID;
|
||||
}
|
||||
static Value bi_write_file(Value *a, int n, Env *e) {
|
||||
(void)e; CHECK_ARITY("write-file", 2); check_string(a[0]); check_string(a[1]);
|
||||
FILE *f = fopen(AS_STRING(a[0])->data, "w");
|
||||
|
|
@ -1367,6 +1392,38 @@ static Value bi_write_binary_file(Value *a, int n, Env *e) {
|
|||
return wrote == data->len ? VAL_VOID : VAL_FALSE;
|
||||
}
|
||||
|
||||
static Value bi_append_binary_file(Value *a, int n, Env *e) {
|
||||
(void)e; CHECK_ARITY("append-binary-file", 2);
|
||||
check_string(a[0]); check_string(a[1]);
|
||||
const char *path = AS_STRING(a[0])->data;
|
||||
ULString *data = AS_STRING(a[1]);
|
||||
FILE *f = fopen(path, "ab");
|
||||
if (!f) return VAL_FALSE;
|
||||
size_t wrote = fwrite(data->data, 1, data->len, f);
|
||||
fclose(f);
|
||||
return wrote == data->len ? VAL_VOID : VAL_FALSE;
|
||||
}
|
||||
|
||||
/* append-port-to-binary-file: stream a string-output port's buffer
|
||||
* directly to a file via fwrite, skipping the Scheme-side string
|
||||
* materialization (get-output-string + append-binary-file would copy
|
||||
* the buffer twice — at multi-GB body sizes this blows past any VM RAM
|
||||
* budget). The port stays usable after; caller closes it. */
|
||||
static Value bi_append_port_to_binary_file(Value *a, int n, Env *e) {
|
||||
(void)e; CHECK_ARITY("append-port-to-binary-file", 2);
|
||||
check_string(a[0]);
|
||||
const char *path = AS_STRING(a[0])->data;
|
||||
ULPort *p = AS_PORT(a[1]);
|
||||
if (p->kind != PORT_STRING || p->dir != PORT_OUTPUT) {
|
||||
lisp_error("append-port-to-binary-file: port must be a string output port");
|
||||
}
|
||||
FILE *f = fopen(path, "ab");
|
||||
if (!f) return VAL_FALSE;
|
||||
size_t wrote = fwrite(p->str_buf, 1, p->str_len, f);
|
||||
fclose(f);
|
||||
return wrote == p->str_len ? VAL_VOID : VAL_FALSE;
|
||||
}
|
||||
|
||||
static Value bi_read_binary_file(Value *a, int n, Env *e) {
|
||||
(void)e; CHECK_ARITY("read-binary-file", 1);
|
||||
check_string(a[0]);
|
||||
|
|
@ -2362,12 +2419,18 @@ static Value bi_procedure_name(Value *a, int n, Env *e) {
|
|||
|
||||
static Value bi_write_string(Value *a, int n, Env *e) {
|
||||
(void)e; CHECK_MIN_ARITY("write-string", 1); check_string(a[0]);
|
||||
FILE *out = n > 1 && IS_PORT(a[1]) ? AS_PORT(a[1])->fp : stdout;
|
||||
ULString *s = AS_STRING(a[0]);
|
||||
if (n > 1 && IS_PORT(a[1]) && AS_PORT(a[1])->kind == PORT_STRING) {
|
||||
ULString *s = AS_STRING(a[0]);
|
||||
port_write_str(AS_PORT(a[1]), s->data, s->len);
|
||||
} else if (n > 1 && IS_PORT(a[1]) && AS_PORT(a[1])->kind == PORT_FILE) {
|
||||
/* Use fwrite with known s->len — fputs strlen-truncates and
|
||||
* loses any 0x00 in the payload (breaks binary emit through
|
||||
* a file port). */
|
||||
FILE *out = AS_PORT(a[1])->fp;
|
||||
fwrite(s->data, 1, s->len, out);
|
||||
} else {
|
||||
fputs(AS_STRING(a[0])->data, out ? out : stdout);
|
||||
FILE *out = (n > 1 && IS_PORT(a[1])) ? AS_PORT(a[1])->fp : stdout;
|
||||
fwrite(s->data, 1, s->len, out ? out : stdout);
|
||||
fflush(out ? out : stdout);
|
||||
}
|
||||
return VAL_VOID;
|
||||
|
|
@ -2385,7 +2448,11 @@ static Value bi_write_char(Value *a, int n, Env *e) {
|
|||
}
|
||||
FILE *out = n > 1 && IS_PORT(a[1]) ? AS_PORT(a[1])->fp : stdout;
|
||||
fputc(AS_CHAR(a[0]), out ? out : stdout);
|
||||
fflush(out ? out : stdout);
|
||||
/* Per-byte fflush only when writing to stdout for interactive REPL
|
||||
* feedback. File ports buffer until close (or explicit flush-port)
|
||||
* — emit-stream calls write-char millions of times per second and
|
||||
* per-write fflush would tank throughput. */
|
||||
if (!(n > 1 && IS_PORT(a[1]))) fflush(stdout);
|
||||
return VAL_VOID;
|
||||
}
|
||||
|
||||
|
|
@ -2619,6 +2686,8 @@ Env *make_global_env(void) {
|
|||
DEF("read-line", bi_read_line); DEF("read-char", bi_read_char);
|
||||
DEF("open-input-file", bi_open_input_file);
|
||||
DEF("open-output-file", bi_open_output_file);
|
||||
DEF("open-binary-output-file", bi_open_binary_output_file);
|
||||
DEF("port-set-position!", bi_port_set_position);
|
||||
DEF("write-file", bi_write_file);
|
||||
DEF("file->string", bi_file_to_string);
|
||||
DEF("tcp-listen", bi_tcp_listen);
|
||||
|
|
@ -2630,6 +2699,8 @@ Env *make_global_env(void) {
|
|||
DEF("spawn-process-stdio", bi_spawn_process_stdio);
|
||||
DEF("flush-port", bi_flush_port);
|
||||
DEF("write-binary-file", bi_write_binary_file);
|
||||
DEF("append-binary-file", bi_append_binary_file);
|
||||
DEF("append-port-to-binary-file", bi_append_port_to_binary_file);
|
||||
DEF("read-binary-file", bi_read_binary_file);
|
||||
DEF("walk-circuit-ops", bi_walk_circuit_ops);
|
||||
DEF("op-specs->bytes", bi_op_specs_to_bytes);
|
||||
|
|
|
|||
10
c/types.c
10
c/types.c
|
|
@ -788,8 +788,16 @@ void port_write_str(ULPort *p, const char *s, size_t len) {
|
|||
if (p->kind == PORT_FILE) {
|
||||
fwrite(s, 1, len, p->fp);
|
||||
} else {
|
||||
/* Grow with 2x doubling on small buffers (fast amortization), shift
|
||||
* to 1.5x once we cross 256 MB. 2x at that scale creates a 3x peak
|
||||
* during realloc (old + new) — 8GB → 16GB needs 24GB transient,
|
||||
* OOMs any reasonable VM. 1.5x bounds peak at 2.5x. */
|
||||
while (p->str_len + len + 1 > p->str_cap) {
|
||||
p->str_cap *= 2;
|
||||
if (p->str_cap < (1ULL << 28)) {
|
||||
p->str_cap *= 2;
|
||||
} else {
|
||||
p->str_cap += p->str_cap / 2;
|
||||
}
|
||||
p->str_buf = (char *)ul_realloc(p->str_buf, p->str_cap);
|
||||
}
|
||||
memcpy(p->str_buf + p->str_len, s, len);
|
||||
|
|
|
|||
54
lumbda.py
54
lumbda.py
|
|
@ -2853,6 +2853,56 @@ def _write_binary_file(path, data):
|
|||
f.write(data.encode('latin-1') if isinstance(data, str) else bytes(data))
|
||||
return VOID
|
||||
|
||||
def _append_binary_file(path, data):
|
||||
"""Append Latin-1-encoded string to a binary file byte-for-byte.
|
||||
Pairs with write-binary-file so emit-stream can land header + body
|
||||
without materializing a string-append concat at production widths
|
||||
(multi-GB body would peak host RAM at 3x body size otherwise)."""
|
||||
with open(path, 'ab') as f:
|
||||
f.write(data.encode('latin-1') if isinstance(data, str) else bytes(data))
|
||||
return VOID
|
||||
|
||||
def _append_port_to_binary_file(path, port):
|
||||
"""Stream a string-output port's accumulated chunks directly to a
|
||||
file. Avoids materializing (get-output-string port) — at multi-GB
|
||||
body sizes that copy peaks host RAM unnecessarily."""
|
||||
if not isinstance(port, StringOutputPort):
|
||||
raise LispErr("append-port-to-binary-file: port must be a string output port")
|
||||
with open(path, 'ab') as f:
|
||||
for chunk in port._buf:
|
||||
f.write(chunk.encode('latin-1') if isinstance(chunk, str) else bytes(chunk))
|
||||
return VOID
|
||||
|
||||
class BinaryFilePort:
|
||||
"""Wraps a Python binary file so write-char / write-string can pass
|
||||
Scheme strings (str or MutableString) without an explicit encode.
|
||||
Mirrors C tier's open-binary-output-file + port-set-position! pair
|
||||
so emit-stream can write gates directly to disk without a string
|
||||
output buffer."""
|
||||
__slots__ = ('_f',)
|
||||
def __init__(self, path): self._f = open(path, 'w+b')
|
||||
def write(self, s):
|
||||
if isinstance(s, MutableString):
|
||||
s = ''.join(s._c)
|
||||
if isinstance(s, str):
|
||||
data = s.encode('latin-1')
|
||||
elif isinstance(s, (bytes, bytearray)):
|
||||
data = bytes(s)
|
||||
else:
|
||||
data = str(s).encode('latin-1')
|
||||
self._f.write(data)
|
||||
return len(data)
|
||||
def flush(self): self._f.flush()
|
||||
def close(self): self._f.close()
|
||||
def seek(self, offset): self._f.seek(offset)
|
||||
|
||||
def _open_binary_output_file(path):
|
||||
return BinaryFilePort(path)
|
||||
|
||||
def _port_set_position(port, offset):
|
||||
port.seek(int(offset))
|
||||
return VOID
|
||||
|
||||
def _read_binary_file(path):
|
||||
"""Read a binary file as a Latin-1 string (1:1 byte mapping)."""
|
||||
with open(path, 'rb') as f:
|
||||
|
|
@ -3757,6 +3807,8 @@ def make_global_env():
|
|||
d(S('read'), lambda a, _: _read_datum_port(a[0] if a else None))
|
||||
d(S('open-input-file'), lambda a, _: open(_str_val(a[0])))
|
||||
d(S('open-output-file'), lambda a, _: open(_str_val(a[0]), 'w'))
|
||||
d(S('open-binary-output-file'), lambda a, _: _open_binary_output_file(_str_val(a[0])))
|
||||
d(S('port-set-position!'), lambda a, _: _port_set_position(a[0], a[1]))
|
||||
d(S('write-file'), lambda a, _: _write_file(_str_val(a[0]), _str_val(a[1])))
|
||||
d(S('file->string'), lambda a, _: _read_file_to_string(_str_val(a[0])))
|
||||
d(S('tcp-listen'), lambda a, _: _tcp_listen(int(a[0])))
|
||||
|
|
@ -3776,6 +3828,8 @@ def make_global_env():
|
|||
lambda a, _: _spawn_process_stdio(_str_val(a[0]), _spawn_args(a)))
|
||||
d(S('flush-port'), lambda a, _: _flush_port(a[0]))
|
||||
d(S('write-binary-file'), lambda a, _: _write_binary_file(_str_val(a[0]), _str_val(a[1])))
|
||||
d(S('append-binary-file'), lambda a, _: _append_binary_file(_str_val(a[0]), _str_val(a[1])))
|
||||
d(S('append-port-to-binary-file'), lambda a, _: _append_port_to_binary_file(_str_val(a[0]), a[1]))
|
||||
d(S('read-binary-file'), lambda a, _: _read_binary_file(_str_val(a[0])))
|
||||
d(S('walk-circuit-ops'), lambda a, _: _walk_circuit_ops(a[0], a[1]))
|
||||
d(S('op-specs->bytes'), lambda a, _: _op_specs_to_bytes(a[0]))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue