#000061: SPV pack split — metadata pack + N chunk packs (v3)

Bidirectional sync: producer always emits both kinds; consumer chooses
how much to pull.

  packs/<hash>.metadata.tar.zst       — one per shard (~0.6 GB compressed)
  packs/<hash>.metadata.manifest.ndjson
  packs/<hash>.chunks.tar.zst         — N per shard (each ≤ 4.4 GB cap)
  packs/<hash>.chunks.manifest.ndjson

Each artifact is independently content-addressed by its own manifest
hash. The metadata pack's manifest carries _chunk_pack_hashes — every
chunk pack covering this shard's content — so a "full" consumer can
iterate them. Chunks packs are anonymous from the consumer side
(reachable only via the metadata pack's reference list).

Consumer sync modes:

  arborist cold unpack <metadata_hash>            # default: just-enough
  arborist cold unpack <metadata_hash> --full     # also pulls chunks

just-enough: pull only the metadata pack. Schema fully restored; every
chunks row has content=NULL. Node is immediately queryable for
metadata operations (documents, edges, audit chain, Merkle); chunk-body
queries return null until a future JIT-fetch path fills them on cache
miss.

full: after the metadata pack lands, iterate _chunk_pack_hashes and pull
every chunk pack. Final state: full corpus offline-queryable.

This is fox's original SPV-wallet framing — was right from day one.

Key API changes:

  pack_key(hash, *, kind="chunks", manifest=False)  — kind in bucket path
  stream_packs(chunks, *, max_compressed_bytes, ...) -> Iterator[FilePack]
      now emits chunk-only packs (no extra_members)
  build_metadata_pack(table_files, *, snapshot_root, chunk_pack_hashes, ...)
      single FilePack with v3 metadata manifest
  pull_metadata_pack(conn, backend, hash) -> dict
  pull_chunk_pack(conn, backend, hash) -> dict
  hydrate_from_metadata_pack(conn, backend, metadata_hash, *, mode) -> dict

push_pack orchestrates: dump tables → stream chunk packs (collect hashes)
→ build metadata pack with chunk_pack_hashes → upload all. Returns
{metadata_pack_hash, chunk_pack_hashes, packs: [...]}.

Manifest format v3:
  metadata pack:
    {"_format_version": 3}
    {"_kind": "metadata"}
    {"_snapshot_root": "..."}
    {"_chunk_pack_hashes": [...]}
    {"table_file": "tables/X.jsonl", "hash": "...", "size": N}  per table
  chunks pack:
    {"_format_version": 3}
    {"_kind": "chunks"}
    {"leaf_hash": "...", "size": N}  per chunk

pack_hash for each = hash_leaf(manifest_bytes); content-addressed at
both layers. Two writers with the same shard state produce identical
metadata_pack_hash AND identical chunk_pack_hashes.

Cap-and-split applies only to chunk packs (chunks are bounded N).
Metadata pack is one file per shard; if a single table file is larger
than the cap, that's noted as future row-level split work.

cold list now surfaces kind per artifact (metadata vs chunks), plus
table_count + chunk_pack_hashes (for metadata packs) or chunk_count
(for chunks packs). Operators can quickly find the metadata pack hash
to feed `cold unpack`.

28 cold-object + evict + boto3 tests pass (3 new SPV-shape tests + 1
v3 manifest-shape test + tampered-metadata-pack test).
This commit is contained in:
russell@unturf.com 2026-05-26 09:27:02 -04:00
parent a193394ea8
commit de705c89a6
No known key found for this signature in database
4 changed files with 613 additions and 250 deletions

View file

@ -23,7 +23,9 @@ from arborist.cold_object import (
from arborist.compress import unpack_chunk
from arborist.document import Document
from arborist.evict import (
pull_pack,
hydrate_from_metadata_pack,
pull_chunk_pack,
pull_metadata_pack,
push_pack,
)
from arborist.ingest import ingest_source
@ -60,10 +62,13 @@ LONG = (
def test_pack_key_layout():
h = "b" * 64
assert pack_key(h).endswith(".tar.zst")
assert pack_key(h, manifest=True).endswith(".manifest.ndjson")
# Different scheme so a `list_keys("packs/")` can distinguish.
# Default kind="chunks". v3 SPV-split keys carry the kind in the path.
assert pack_key(h).endswith(".chunks.tar.zst")
assert pack_key(h, manifest=True).endswith(".chunks.manifest.ndjson")
assert pack_key(h, kind="metadata").endswith(".metadata.tar.zst")
assert pack_key(h, kind="metadata", manifest=True).endswith(".metadata.manifest.ndjson")
assert pack_key(h) != pack_key(h, manifest=True)
assert pack_key(h, kind="chunks") != pack_key(h, kind="metadata")
def test_build_pack_is_content_addressed():
@ -101,10 +106,9 @@ def test_parse_manifest_round_trip():
leaf = hash_leaf(body).hex()
pack = build_pack([(leaf, body)])
parsed = parse_manifest(pack.manifest_bytes)
# build_pack produces a chunks-only manifest (no tables) — that's the
# v1 shape because build_pack doesn't accept extra_members. Tests of
# the v2 shape live in test_push_pack_v2_hydrates_fresh_empty_db.
assert parsed.format_version == 1
# build_pack produces a chunks-only v3 manifest.
assert parsed.format_version == 3
assert parsed.kind == "chunks"
assert parsed.tables == ()
assert len(parsed.chunks) == 1
assert parsed.chunks[0].leaf_hash == leaf
@ -151,7 +155,9 @@ def test_memory_backend_identity_carries_no_credentials():
# ---------------------------------------------------------------------------
def test_push_pack_then_pull_pack_restores_chunks(tmp_path):
def test_push_pack_then_pull_full_restores_chunks(tmp_path):
"""Push produces 1 metadata pack + N chunk packs. Full hydration
(mode="full") pulls metadata then iterates chunk_pack_hashes."""
db = tmp_path / "pack.db"
conn = connect(db)
backend = MemoryBackend()
@ -161,41 +167,85 @@ def test_push_pack_then_pull_pack_restores_chunks(tmp_path):
"SELECT document_root FROM documents WHERE document_uri='html://p'"
).fetchone()["document_root"]
# Pack while content is still local. Tiny corpus → one pack.
push_result = push_pack(conn, backend, document_root=root)
assert push_result["status"] == "pushed"
assert push_result["pack_count"] == 1
# Pack is bound to a snapshot_root — packs are delayed point-in-time
# snapshots; the audit row pins which corpus state this pack covers
# so falsifications between repacks produce a different pack_hash.
assert len(push_result["snapshot_root"]) == 64
first = push_result["packs"][0]
assert first["chunk_count"] > 0
assert first["compressed_bytes"] <= first["uncompressed_bytes"] + 200
assert first["snapshot_root"] == push_result["snapshot_root"]
pack_hash = first["pack_hash"]
assert backend.head(pack_key(pack_hash))
assert backend.head(pack_key(pack_hash, manifest=True))
# SPV split: at least 1 metadata pack + at least 1 chunks pack.
assert push_result["metadata_pack_hash"] is not None
assert len(push_result["chunk_pack_hashes"]) >= 1
# Now NULL local content so pull_pack has something to do.
conn.execute("UPDATE chunks SET content = NULL, tier='cold'")
conn.execute("DELETE FROM chunks_fts")
conn.commit()
metadata_hash = push_result["metadata_pack_hash"]
# Both pack kinds are in the bucket at their kind-specific keys.
assert backend.head(pack_key(metadata_hash, kind="metadata"))
assert backend.head(pack_key(metadata_hash, kind="metadata", manifest=True))
for ch in push_result["chunk_pack_hashes"]:
assert backend.head(pack_key(ch, kind="chunks"))
pull_result = pull_pack(conn, backend, pack_hash)
assert pull_result["status"] == "pulled"
assert pull_result["chunks_restored"] == first["chunk_count"]
# NULL local content + delete documents so a full pull has to
# restore the whole schema, not just chunks.
with conn:
conn.execute("DELETE FROM chunks_fts")
conn.execute("DELETE FROM chunks")
conn.execute("DELETE FROM documents")
result = hydrate_from_metadata_pack(
conn, backend, metadata_hash, mode="full",
)
assert result["status"] == "hydrated"
assert result["mode"] == "full"
assert result["chunk_packs_pulled"] == len(push_result["chunk_pack_hashes"])
# Schema restored.
assert conn.execute("SELECT COUNT(*) FROM documents").fetchone()[0] == 1
# All chunks restored to hot tier with content.
cold = conn.execute(
"SELECT COUNT(*) FROM chunks WHERE tier='cold'"
"SELECT COUNT(*) FROM chunks WHERE tier='cold' OR content IS NULL"
).fetchone()[0]
assert cold == 0
finally:
conn.close()
def test_push_pack_then_just_enough_hydrate_leaves_chunks_null(tmp_path):
"""just-enough mode: pull only metadata pack. Schema fully restored
but chunks rows have content=NULL ready for JIT-fetch later."""
db = tmp_path / "je.db"
conn = connect(db)
backend = MemoryBackend()
try:
ingest_source(conn, FakeSource([_doc("html://je", LONG)]))
push_result = push_pack(conn, backend)
metadata_hash = push_result["metadata_pack_hash"]
expected_chunks = sum(p.get("chunk_count", 0) for p in push_result["packs"]
if p.get("kind") == "chunks")
finally:
conn.close()
rx_db = tmp_path / "rx.db"
conn_rx = connect(rx_db)
try:
result = hydrate_from_metadata_pack(
conn_rx, backend, metadata_hash, mode="just-enough",
)
assert result["status"] == "hydrated"
assert result["mode"] == "just-enough"
assert result["chunk_packs_pulled"] == 0
# Documents row exists.
assert conn_rx.execute("SELECT COUNT(*) FROM documents").fetchone()[0] == 1
# Chunks rows exist (metadata restored) but content is NULL.
n_chunks = conn_rx.execute("SELECT COUNT(*) FROM chunks").fetchone()[0]
n_null = conn_rx.execute(
"SELECT COUNT(*) FROM chunks WHERE content IS NULL"
).fetchone()[0]
assert n_chunks == expected_chunks
assert n_null == n_chunks
finally:
conn_rx.close()
def test_push_pack_idempotent(tmp_path):
"""Same chunk set → same pack_hash(es) → bucket overwrite is a no-op."""
"""Same shard state on two runs produces the same pack_hashes:
metadata_pack_hash + every chunk_pack_hash. Bucket overwrite is a
no-op."""
db = tmp_path / "pack-id.db"
conn = connect(db)
backend = MemoryBackend()
@ -203,9 +253,8 @@ def test_push_pack_idempotent(tmp_path):
ingest_source(conn, FakeSource([_doc("html://p", LONG)]))
a = push_pack(conn, backend)
b = push_pack(conn, backend)
assert [p["pack_hash"] for p in a["packs"]] == [
p["pack_hash"] for p in b["packs"]
]
assert a["metadata_pack_hash"] == b["metadata_pack_hash"]
assert a["chunk_pack_hashes"] == b["chunk_pack_hashes"]
finally:
conn.close()
@ -258,31 +307,38 @@ def test_push_pack_splits_to_fit_dvdr(tmp_path):
# Tar trailer (~1 KB padding) + zstd frame footer get emitted after
# the last in-loop size check, so overshoot is bounded by trailer
# size + one chunk's worth of compressed bytes.
# size + one chunk's worth of compressed bytes. Only the chunks
# packs are subject to the cap; the metadata pack is its own
# artifact and can be any size.
OVERSHOOT_SLACK = 4_096
for p in result["packs"]:
chunk_packs = [p for p in result["packs"] if p["kind"] == "chunks"]
assert len(chunk_packs) > 1, "small cap should force multi-chunks-pack split"
for p in chunk_packs:
assert p["compressed_bytes"] <= CAP + OVERSHOOT_SLACK, (
f"pack {p['pack_hash'][:8]} compressed={p['compressed_bytes']} "
f"exceeds cap {CAP}+slack {OVERSHOOT_SLACK}"
f"chunks pack {p['pack_hash'][:8]} compressed="
f"{p['compressed_bytes']} exceeds cap {CAP}+slack"
)
assert p["chunk_count"] >= 1
# Round-trip: NULL local content, pull every pack back.
# Round-trip: NULL local content, hydrate-full from metadata pack.
conn.execute("UPDATE chunks SET content = NULL, tier='cold'")
conn.execute("DELETE FROM chunks_fts")
conn.commit()
total_restored = 0
for p in result["packs"]:
r = pull_pack(conn, backend, p["pack_hash"])
total_restored += r["chunks_restored"]
hyd = hydrate_from_metadata_pack(
conn, backend, result["metadata_pack_hash"], mode="full",
)
assert hyd["status"] == "hydrated"
# Total chunks restored == total chunks across all chunk packs.
total_restored = sum(r["chunks_restored"] for r in hyd["chunk_pulls"])
assert total_restored == result["total_chunks"]
finally:
conn.close()
def test_push_pack_local_dir_writes_files(tmp_path):
"""--local-dir writes pack + manifest files for burning to physical media."""
"""--local-dir writes pack + manifest files for burning to physical media.
v3: both metadata pack and chunk packs land in local-dir, with kind
suffix in the filename."""
db = tmp_path / "burn.db"
conn = connect(db)
backend = MemoryBackend()
@ -293,13 +349,17 @@ def test_push_pack_local_dir_writes_files(tmp_path):
assert result["status"] == "pushed"
assert result["local_dir"] == str(burn_dir)
assert burn_dir.exists()
pack_files = sorted(burn_dir.glob("arborist-pack-*.tar.zst"))
manifest_files = sorted(burn_dir.glob("arborist-pack-*.manifest.ndjson"))
assert len(pack_files) == result["pack_count"]
assert len(manifest_files) == result["pack_count"]
# Local files are byte-identical to what we'd burn.
for pack_file, summary in zip(pack_files, result["packs"]):
assert pack_file.stat().st_size == summary["compressed_bytes"]
# v3 filenames: arborist-pack-<short>.{metadata,chunks}.tar.zst
metadata_files = sorted(burn_dir.glob("arborist-pack-*.metadata.tar.zst"))
chunks_files = sorted(burn_dir.glob("arborist-pack-*.chunks.tar.zst"))
assert len(metadata_files) == 1
assert len(chunks_files) == len(result["chunk_pack_hashes"])
# Manifest sidecars present too.
assert len(sorted(burn_dir.glob("arborist-pack-*.metadata.manifest.ndjson"))) == 1
assert (
len(sorted(burn_dir.glob("arborist-pack-*.chunks.manifest.ndjson")))
== len(result["chunk_pack_hashes"])
)
finally:
conn.close()
@ -373,44 +433,44 @@ def test_edges_fan_in_batches_huge_destinations(tmp_path):
conn.close()
def test_v2_pack_hash_binds_table_contents(tmp_path):
"""Two packs with the same chunk set but different table content
must NOT collide on pack_hash.
The v1 bug: pack_hash was hash_leaf(chunk-only-manifest), so
same-chunks + different-tables produced same pack_hash. Bucket
overwrites or mesh-peer disagreement silent table swap. v2
fixes this by including content_hash for each tables/<name>.jsonl
in the manifest before chunk entries.
This test forces the issue by ingesting two different document
sets that happen to share a chunk wait, that's impossible by
construction. Instead we check the manifest format directly: a
v2 pack's manifest has a `_format_version: 2` header and
`table_file` entries before `leaf_hash` entries.
"""
db = tmp_path / "v2bind.db"
def test_v3_metadata_pack_manifest_shape(tmp_path):
"""v3 metadata pack manifest carries: _format_version=3, _kind="metadata",
_snapshot_root, _chunk_pack_hashes list, and one table_file entry per
non-empty shipped table. Chunks pack manifest separately carries
_kind="chunks" and leaf_hash entries."""
db = tmp_path / "v3shape.db"
conn = connect(db)
backend = MemoryBackend()
try:
ingest_source(conn, FakeSource([_doc("html://a", LONG)]))
result = push_pack(conn, backend)
manifest = backend.get_pack_manifest(result["packs"][0]["pack_hash"])
parsed = parse_manifest(manifest)
metadata_hash = result["metadata_pack_hash"]
chunk_hashes = result["chunk_pack_hashes"]
assert metadata_hash is not None
assert len(chunk_hashes) >= 1
assert parsed.format_version == 2
# All shipped tables that had rows should appear in the manifest.
# documents + chunks + merkle_nodes + audit_events minimum.
table_names = {t.member_name for t in parsed.tables}
meta_manifest = backend.get_pack_manifest(metadata_hash, kind="metadata")
meta = parse_manifest(meta_manifest)
assert meta.format_version == 3
assert meta.kind == "metadata"
assert meta.snapshot_root and len(meta.snapshot_root) == 64
assert list(meta.chunk_pack_hashes) == chunk_hashes
assert meta.chunks == ()
table_names = {t.member_name for t in meta.tables}
assert "tables/documents.jsonl" in table_names
assert "tables/chunks.jsonl" in table_names
assert "tables/audit_events.jsonl" in table_names
# Every table_file ref carries a hash and size.
for t in parsed.tables:
assert len(t.content_hash) == 64 # sha256 hex
for t in meta.tables:
assert len(t.content_hash) == 64
assert t.size > 0
# Chunks are also there.
assert len(parsed.chunks) > 0
# First chunks pack: chunks only, no tables.
first_chunks_manifest = backend.get_pack_manifest(chunk_hashes[0], kind="chunks")
ch = parse_manifest(first_chunks_manifest)
assert ch.format_version == 3
assert ch.kind == "chunks"
assert ch.tables == ()
assert len(ch.chunks) > 0
finally:
conn.close()
@ -433,14 +493,14 @@ def test_pull_pack_rejects_tampered_table(tmp_path):
try:
ingest_source(conn, FakeSource([_doc("html://t", LONG)]))
result = push_pack(conn, backend)
pack_hash = result["packs"][0]["pack_hash"]
metadata_hash = result["metadata_pack_hash"]
finally:
conn.close()
# Decompress the pack, rewrite documents.jsonl with corrupted bytes,
# repack, replace the pack in the bucket. Manifest stays unchanged
# so the recorded table content_hash no longer matches.
body = backend.get_pack(pack_hash)
# Decompress the metadata pack, rewrite documents.jsonl with corrupted
# bytes, repack, replace in the bucket. Manifest stays unchanged so
# the recorded table content_hash no longer matches.
body = backend.get_pack(metadata_hash, kind="metadata")
raw_tar = _zstd.ZstdDecompressor().stream_reader(_io.BytesIO(body)).read()
out_buf = _io.BytesIO()
cctx = _zstd.ZstdCompressor(level=3)
@ -454,20 +514,20 @@ def test_pull_pack_rejects_tampered_table(tmp_path):
data = f.read()
if m.name == "tables/documents.jsonl":
data = data + b'\n{"injected": "tamper"}\n'
info = _tarfile.TarInfo(name=m.name)
info.size = len(data)
else:
info = _tarfile.TarInfo(name=m.name)
info.size = len(data)
info = _tarfile.TarInfo(name=m.name)
info.size = len(data)
out_tar.addfile(info, _io.BytesIO(data))
backend._tamper(f"packs/{pack_hash}.tar.zst", out_buf.getvalue())
backend._tamper(
f"packs/{metadata_hash}.metadata.tar.zst", out_buf.getvalue()
)
# Fresh DB; expect ValueError on pull.
# Fresh DB; expect ValueError on hydrate (metadata pack tamper fires
# the hash-check during pull_metadata_pack).
rx_db = tmp_path / "rx.db"
conn_rx = connect(rx_db)
try:
with pytest.raises(ValueError, match="table-hash mismatch"):
pull_pack(conn_rx, backend, pack_hash)
pull_metadata_pack(conn_rx, backend, metadata_hash)
finally:
conn_rx.close()
@ -507,7 +567,8 @@ def test_push_pack_v2_hydrates_fresh_empty_db(tmp_path):
result = push_pack(conn_src, backend)
assert result["status"] == "pushed"
pack_hash = result["packs"][0]["pack_hash"]
metadata_hash = result["metadata_pack_hash"]
assert metadata_hash is not None
finally:
conn_src.close()
@ -520,20 +581,19 @@ def test_push_pack_v2_hydrates_fresh_empty_db(tmp_path):
"SELECT COUNT(*) FROM documents"
).fetchone()[0] == 0
pull_result = pull_pack(conn_rx, backend, pack_hash)
assert pull_result["status"] == "pulled"
assert pull_result["pack_format"] == "v2"
# Every table that was non-empty in src should have rows now in rx.
assert pull_result["tables_restored"]["documents"] == src_docs
assert pull_result["tables_restored"]["chunks"] == src_chunks
assert pull_result["tables_restored"]["merkle_nodes"] == src_merkle
assert pull_result["tables_restored"]["audit_events"] == src_audit
# Chunk bodies restored.
hyd = hydrate_from_metadata_pack(
conn_rx, backend, metadata_hash, mode="full",
)
assert hyd["status"] == "hydrated"
assert hyd["mode"] == "full"
assert conn_rx.execute(
"SELECT COUNT(*) FROM documents"
).fetchone()[0] == src_docs
# Chunks rows + content restored.
rx_hot = conn_rx.execute(
"SELECT COUNT(*) FROM chunks WHERE tier='hot' AND content IS NOT NULL"
).fetchone()[0]
assert rx_hot == src_chunks
# FTS5 rows rebuilt.
rx_fts = conn_rx.execute("SELECT COUNT(*) FROM chunks_fts").fetchone()[0]
assert rx_fts == src_chunks
# All three documents reachable by URI.