diff --git a/arborist/cold_pack_metadata.py b/arborist/cold_pack_metadata.py index e8017f6..3954a7a 100644 --- a/arborist/cold_pack_metadata.py +++ b/arborist/cold_pack_metadata.py @@ -470,7 +470,19 @@ def _restore_generic_table( table: str, in_path: Path, ) -> int: - """Read array-per-line JSONL → INSERT OR IGNORE batches into `table`.""" + """Read array-per-line JSONL → INSERT OR IGNORE batches into `table`. + + Chunks special case (#53): the producer dumps a synthetic + ``_content_size`` column carrying each chunk's content byte-length + (the actual ``content`` BLOB is dropped from the metadata pack to + keep it small). Consumer drops the synthetic column from the INSERT + column list — chunks rows land with ``content IS NULL`` per + ``pull_metadata_pack``'s documented contract; phase 2 + (``pull_chunk_pack``) fills the bytes via UPDATE WHERE leaf_hash=?. + The size hint is currently unused in the single-shard path; a + future zeroblob pre-allocation could read it back if page-split + cost becomes measurable. + """ BATCH = 5000 batch: list[tuple] = [] cols: list[str] | None = None @@ -478,7 +490,7 @@ def _restore_generic_table( insert_sql: str | None = None for row in read_columnar_jsonl(in_path): if cols is None: - cols = list(row.keys()) + cols = [c for c in row.keys() if c != "_content_size"] placeholders = ", ".join("?" for _ in cols) col_list = ", ".join(f'"{c}"' for c in cols) insert_sql = (