diff --git a/arborist/cold_object.py b/arborist/cold_object.py index f198499..a64a031 100644 --- a/arborist/cold_object.py +++ b/arborist/cold_object.py @@ -571,16 +571,18 @@ def stream_packs( state = _start_pack() try: # Pack format v2: ship the metadata table dumps as the FIRST tar - # members of the first pack. Add them up front so they're tarred - # before chunks; if the pack overflows the cap mid-chunk, the - # metadata is already safely in pack 1 and pack 2 onward holds - # only the chunk overflow. + # members of the first pack. Stream from disk — tarfile.addfile + # reads the file handle in chunks, never loading the whole thing + # into RAM. A 4 GB edges.jsonl thus uses ~tens of KB peak RAM + # instead of 4 GB per worker. TarInfo built by hand (NOT via + # gettarinfo) so mtime/uid/gid are zeroed — two writers' packs + # converge byte-for-byte regardless of filesystem timestamps. for member_name, source_path in pending_extras: - with open(source_path, "rb") as f: - source_bytes = f.read() info = tarfile.TarInfo(name=member_name) - info.size = len(source_bytes) - state["tar"].addfile(info, io.BytesIO(source_bytes)) + info.size = source_path.stat().st_size + # mtime, uid, gid, mode default to 0 / "" — deterministic. + with open(source_path, "rb") as f: + state["tar"].addfile(info, f) # Force the compressor to flush so size accounting reflects the # metadata write before we start streaming chunks. if pending_extras: