From ea743565de26becfcd4a1d247292288513f54afa Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Tue, 26 May 2026 05:53:11 -0400 Subject: [PATCH] #000061: stream metadata files in tar, not f.read() into memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The extra_members handling in stream_packs did: with open(source_path, 'rb') as f: source_bytes = f.read() # loads entire file into RAM info = tarfile.TarInfo(...) info.size = len(source_bytes) state['tar'].addfile(info, io.BytesIO(source_bytes)) For a 4.3 GB edges.jsonl this is 4.3 GB of RAM per worker — exactly the failure mode the prior streaming-to-disk refactor was supposed to eliminate (57f8989). 4-way parallel would hit ~17 GB peak RAM, the same OOM-risk number we saw on the v1 corpus run. Fix: build the TarInfo by hand (size from .stat(), other fields zeroed), then pass the open file handle directly to tar.addfile. tarfile reads the body in chunks (~16 KB at a time) and feeds them into the zstd stream_writer. Peak RAM per worker stays at ~tens of KB regardless of metadata file size. Determinism preserved: mtime/uid/gid default to 0 (NOT taken from the filesystem via gettarinfo), so two writers' packs converge byte-for-byte regardless of when they ran or who owns the temp file. 24 cold-object + evict tests pass. --- arborist/cold_object.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/arborist/cold_object.py b/arborist/cold_object.py index f198499..a64a031 100644 --- a/arborist/cold_object.py +++ b/arborist/cold_object.py @@ -571,16 +571,18 @@ def stream_packs( state = _start_pack() try: # Pack format v2: ship the metadata table dumps as the FIRST tar - # members of the first pack. Add them up front so they're tarred - # before chunks; if the pack overflows the cap mid-chunk, the - # metadata is already safely in pack 1 and pack 2 onward holds - # only the chunk overflow. + # members of the first pack. Stream from disk — tarfile.addfile + # reads the file handle in chunks, never loading the whole thing + # into RAM. A 4 GB edges.jsonl thus uses ~tens of KB peak RAM + # instead of 4 GB per worker. TarInfo built by hand (NOT via + # gettarinfo) so mtime/uid/gid are zeroed — two writers' packs + # converge byte-for-byte regardless of filesystem timestamps. for member_name, source_path in pending_extras: - with open(source_path, "rb") as f: - source_bytes = f.read() info = tarfile.TarInfo(name=member_name) - info.size = len(source_bytes) - state["tar"].addfile(info, io.BytesIO(source_bytes)) + info.size = source_path.stat().st_size + # mtime, uid, gid, mode default to 0 / "" — deterministic. + with open(source_path, "rb") as f: + state["tar"].addfile(info, f) # Force the compressor to flush so size accounting reflects the # metadata write before we start streaming chunks. if pending_extras: