#000061: SPV pack split — metadata pack + N chunk packs (v3)
Bidirectional sync: producer always emits both kinds; consumer chooses
how much to pull.
packs/<hash>.metadata.tar.zst — one per shard (~0.6 GB compressed)
packs/<hash>.metadata.manifest.ndjson
packs/<hash>.chunks.tar.zst — N per shard (each ≤ 4.4 GB cap)
packs/<hash>.chunks.manifest.ndjson
Each artifact is independently content-addressed by its own manifest
hash. The metadata pack's manifest carries _chunk_pack_hashes — every
chunk pack covering this shard's content — so a "full" consumer can
iterate them. Chunks packs are anonymous from the consumer side
(reachable only via the metadata pack's reference list).
Consumer sync modes:
arborist cold unpack <metadata_hash> # default: just-enough
arborist cold unpack <metadata_hash> --full # also pulls chunks
just-enough: pull only the metadata pack. Schema fully restored; every
chunks row has content=NULL. Node is immediately queryable for
metadata operations (documents, edges, audit chain, Merkle); chunk-body
queries return null until a future JIT-fetch path fills them on cache
miss.
full: after the metadata pack lands, iterate _chunk_pack_hashes and pull
every chunk pack. Final state: full corpus offline-queryable.
This is fox's original SPV-wallet framing — was right from day one.
Key API changes:
pack_key(hash, *, kind="chunks", manifest=False) — kind in bucket path
stream_packs(chunks, *, max_compressed_bytes, ...) -> Iterator[FilePack]
now emits chunk-only packs (no extra_members)
build_metadata_pack(table_files, *, snapshot_root, chunk_pack_hashes, ...)
single FilePack with v3 metadata manifest
pull_metadata_pack(conn, backend, hash) -> dict
pull_chunk_pack(conn, backend, hash) -> dict
hydrate_from_metadata_pack(conn, backend, metadata_hash, *, mode) -> dict
push_pack orchestrates: dump tables → stream chunk packs (collect hashes)
→ build metadata pack with chunk_pack_hashes → upload all. Returns
{metadata_pack_hash, chunk_pack_hashes, packs: [...]}.
Manifest format v3:
metadata pack:
{"_format_version": 3}
{"_kind": "metadata"}
{"_snapshot_root": "..."}
{"_chunk_pack_hashes": [...]}
{"table_file": "tables/X.jsonl", "hash": "...", "size": N} per table
chunks pack:
{"_format_version": 3}
{"_kind": "chunks"}
{"leaf_hash": "...", "size": N} per chunk
pack_hash for each = hash_leaf(manifest_bytes); content-addressed at
both layers. Two writers with the same shard state produce identical
metadata_pack_hash AND identical chunk_pack_hashes.
Cap-and-split applies only to chunk packs (chunks are bounded N).
Metadata pack is one file per shard; if a single table file is larger
than the cap, that's noted as future row-level split work.
cold list now surfaces kind per artifact (metadata vs chunks), plus
table_count + chunk_pack_hashes (for metadata packs) or chunk_count
(for chunks packs). Operators can quickly find the metadata pack hash
to feed `cold unpack`.
28 cold-object + evict + boto3 tests pass (3 new SPV-shape tests + 1
v3 manifest-shape test + tampered-metadata-pack test).
This commit is contained in:
parent
a193394ea8
commit
de705c89a6
4 changed files with 613 additions and 250 deletions
|
|
@ -23,7 +23,9 @@ from arborist.cold_object import (
|
|||
from arborist.compress import unpack_chunk
|
||||
from arborist.document import Document
|
||||
from arborist.evict import (
|
||||
pull_pack,
|
||||
hydrate_from_metadata_pack,
|
||||
pull_chunk_pack,
|
||||
pull_metadata_pack,
|
||||
push_pack,
|
||||
)
|
||||
from arborist.ingest import ingest_source
|
||||
|
|
@ -60,10 +62,13 @@ LONG = (
|
|||
|
||||
def test_pack_key_layout():
|
||||
h = "b" * 64
|
||||
assert pack_key(h).endswith(".tar.zst")
|
||||
assert pack_key(h, manifest=True).endswith(".manifest.ndjson")
|
||||
# Different scheme so a `list_keys("packs/")` can distinguish.
|
||||
# Default kind="chunks". v3 SPV-split keys carry the kind in the path.
|
||||
assert pack_key(h).endswith(".chunks.tar.zst")
|
||||
assert pack_key(h, manifest=True).endswith(".chunks.manifest.ndjson")
|
||||
assert pack_key(h, kind="metadata").endswith(".metadata.tar.zst")
|
||||
assert pack_key(h, kind="metadata", manifest=True).endswith(".metadata.manifest.ndjson")
|
||||
assert pack_key(h) != pack_key(h, manifest=True)
|
||||
assert pack_key(h, kind="chunks") != pack_key(h, kind="metadata")
|
||||
|
||||
|
||||
def test_build_pack_is_content_addressed():
|
||||
|
|
@ -101,10 +106,9 @@ def test_parse_manifest_round_trip():
|
|||
leaf = hash_leaf(body).hex()
|
||||
pack = build_pack([(leaf, body)])
|
||||
parsed = parse_manifest(pack.manifest_bytes)
|
||||
# build_pack produces a chunks-only manifest (no tables) — that's the
|
||||
# v1 shape because build_pack doesn't accept extra_members. Tests of
|
||||
# the v2 shape live in test_push_pack_v2_hydrates_fresh_empty_db.
|
||||
assert parsed.format_version == 1
|
||||
# build_pack produces a chunks-only v3 manifest.
|
||||
assert parsed.format_version == 3
|
||||
assert parsed.kind == "chunks"
|
||||
assert parsed.tables == ()
|
||||
assert len(parsed.chunks) == 1
|
||||
assert parsed.chunks[0].leaf_hash == leaf
|
||||
|
|
@ -151,7 +155,9 @@ def test_memory_backend_identity_carries_no_credentials():
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_push_pack_then_pull_pack_restores_chunks(tmp_path):
|
||||
def test_push_pack_then_pull_full_restores_chunks(tmp_path):
|
||||
"""Push produces 1 metadata pack + N chunk packs. Full hydration
|
||||
(mode="full") pulls metadata then iterates chunk_pack_hashes."""
|
||||
db = tmp_path / "pack.db"
|
||||
conn = connect(db)
|
||||
backend = MemoryBackend()
|
||||
|
|
@ -161,41 +167,85 @@ def test_push_pack_then_pull_pack_restores_chunks(tmp_path):
|
|||
"SELECT document_root FROM documents WHERE document_uri='html://p'"
|
||||
).fetchone()["document_root"]
|
||||
|
||||
# Pack while content is still local. Tiny corpus → one pack.
|
||||
push_result = push_pack(conn, backend, document_root=root)
|
||||
assert push_result["status"] == "pushed"
|
||||
assert push_result["pack_count"] == 1
|
||||
# Pack is bound to a snapshot_root — packs are delayed point-in-time
|
||||
# snapshots; the audit row pins which corpus state this pack covers
|
||||
# so falsifications between repacks produce a different pack_hash.
|
||||
assert len(push_result["snapshot_root"]) == 64
|
||||
first = push_result["packs"][0]
|
||||
assert first["chunk_count"] > 0
|
||||
assert first["compressed_bytes"] <= first["uncompressed_bytes"] + 200
|
||||
assert first["snapshot_root"] == push_result["snapshot_root"]
|
||||
pack_hash = first["pack_hash"]
|
||||
assert backend.head(pack_key(pack_hash))
|
||||
assert backend.head(pack_key(pack_hash, manifest=True))
|
||||
# SPV split: at least 1 metadata pack + at least 1 chunks pack.
|
||||
assert push_result["metadata_pack_hash"] is not None
|
||||
assert len(push_result["chunk_pack_hashes"]) >= 1
|
||||
|
||||
# Now NULL local content so pull_pack has something to do.
|
||||
conn.execute("UPDATE chunks SET content = NULL, tier='cold'")
|
||||
conn.execute("DELETE FROM chunks_fts")
|
||||
conn.commit()
|
||||
metadata_hash = push_result["metadata_pack_hash"]
|
||||
# Both pack kinds are in the bucket at their kind-specific keys.
|
||||
assert backend.head(pack_key(metadata_hash, kind="metadata"))
|
||||
assert backend.head(pack_key(metadata_hash, kind="metadata", manifest=True))
|
||||
for ch in push_result["chunk_pack_hashes"]:
|
||||
assert backend.head(pack_key(ch, kind="chunks"))
|
||||
|
||||
pull_result = pull_pack(conn, backend, pack_hash)
|
||||
assert pull_result["status"] == "pulled"
|
||||
assert pull_result["chunks_restored"] == first["chunk_count"]
|
||||
# NULL local content + delete documents so a full pull has to
|
||||
# restore the whole schema, not just chunks.
|
||||
with conn:
|
||||
conn.execute("DELETE FROM chunks_fts")
|
||||
conn.execute("DELETE FROM chunks")
|
||||
conn.execute("DELETE FROM documents")
|
||||
|
||||
result = hydrate_from_metadata_pack(
|
||||
conn, backend, metadata_hash, mode="full",
|
||||
)
|
||||
assert result["status"] == "hydrated"
|
||||
assert result["mode"] == "full"
|
||||
assert result["chunk_packs_pulled"] == len(push_result["chunk_pack_hashes"])
|
||||
|
||||
# Schema restored.
|
||||
assert conn.execute("SELECT COUNT(*) FROM documents").fetchone()[0] == 1
|
||||
# All chunks restored to hot tier with content.
|
||||
cold = conn.execute(
|
||||
"SELECT COUNT(*) FROM chunks WHERE tier='cold'"
|
||||
"SELECT COUNT(*) FROM chunks WHERE tier='cold' OR content IS NULL"
|
||||
).fetchone()[0]
|
||||
assert cold == 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_push_pack_then_just_enough_hydrate_leaves_chunks_null(tmp_path):
|
||||
"""just-enough mode: pull only metadata pack. Schema fully restored
|
||||
but chunks rows have content=NULL — ready for JIT-fetch later."""
|
||||
db = tmp_path / "je.db"
|
||||
conn = connect(db)
|
||||
backend = MemoryBackend()
|
||||
try:
|
||||
ingest_source(conn, FakeSource([_doc("html://je", LONG)]))
|
||||
push_result = push_pack(conn, backend)
|
||||
metadata_hash = push_result["metadata_pack_hash"]
|
||||
expected_chunks = sum(p.get("chunk_count", 0) for p in push_result["packs"]
|
||||
if p.get("kind") == "chunks")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
rx_db = tmp_path / "rx.db"
|
||||
conn_rx = connect(rx_db)
|
||||
try:
|
||||
result = hydrate_from_metadata_pack(
|
||||
conn_rx, backend, metadata_hash, mode="just-enough",
|
||||
)
|
||||
assert result["status"] == "hydrated"
|
||||
assert result["mode"] == "just-enough"
|
||||
assert result["chunk_packs_pulled"] == 0
|
||||
# Documents row exists.
|
||||
assert conn_rx.execute("SELECT COUNT(*) FROM documents").fetchone()[0] == 1
|
||||
# Chunks rows exist (metadata restored) but content is NULL.
|
||||
n_chunks = conn_rx.execute("SELECT COUNT(*) FROM chunks").fetchone()[0]
|
||||
n_null = conn_rx.execute(
|
||||
"SELECT COUNT(*) FROM chunks WHERE content IS NULL"
|
||||
).fetchone()[0]
|
||||
assert n_chunks == expected_chunks
|
||||
assert n_null == n_chunks
|
||||
finally:
|
||||
conn_rx.close()
|
||||
|
||||
|
||||
def test_push_pack_idempotent(tmp_path):
|
||||
"""Same chunk set → same pack_hash(es) → bucket overwrite is a no-op."""
|
||||
"""Same shard state on two runs produces the same pack_hashes:
|
||||
metadata_pack_hash + every chunk_pack_hash. Bucket overwrite is a
|
||||
no-op."""
|
||||
db = tmp_path / "pack-id.db"
|
||||
conn = connect(db)
|
||||
backend = MemoryBackend()
|
||||
|
|
@ -203,9 +253,8 @@ def test_push_pack_idempotent(tmp_path):
|
|||
ingest_source(conn, FakeSource([_doc("html://p", LONG)]))
|
||||
a = push_pack(conn, backend)
|
||||
b = push_pack(conn, backend)
|
||||
assert [p["pack_hash"] for p in a["packs"]] == [
|
||||
p["pack_hash"] for p in b["packs"]
|
||||
]
|
||||
assert a["metadata_pack_hash"] == b["metadata_pack_hash"]
|
||||
assert a["chunk_pack_hashes"] == b["chunk_pack_hashes"]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
|
@ -258,31 +307,38 @@ def test_push_pack_splits_to_fit_dvdr(tmp_path):
|
|||
|
||||
# Tar trailer (~1 KB padding) + zstd frame footer get emitted after
|
||||
# the last in-loop size check, so overshoot is bounded by trailer
|
||||
# size + one chunk's worth of compressed bytes.
|
||||
# size + one chunk's worth of compressed bytes. Only the chunks
|
||||
# packs are subject to the cap; the metadata pack is its own
|
||||
# artifact and can be any size.
|
||||
OVERSHOOT_SLACK = 4_096
|
||||
for p in result["packs"]:
|
||||
chunk_packs = [p for p in result["packs"] if p["kind"] == "chunks"]
|
||||
assert len(chunk_packs) > 1, "small cap should force multi-chunks-pack split"
|
||||
for p in chunk_packs:
|
||||
assert p["compressed_bytes"] <= CAP + OVERSHOOT_SLACK, (
|
||||
f"pack {p['pack_hash'][:8]} compressed={p['compressed_bytes']} "
|
||||
f"exceeds cap {CAP}+slack {OVERSHOOT_SLACK}"
|
||||
f"chunks pack {p['pack_hash'][:8]} compressed="
|
||||
f"{p['compressed_bytes']} exceeds cap {CAP}+slack"
|
||||
)
|
||||
assert p["chunk_count"] >= 1
|
||||
|
||||
# Round-trip: NULL local content, pull every pack back.
|
||||
# Round-trip: NULL local content, hydrate-full from metadata pack.
|
||||
conn.execute("UPDATE chunks SET content = NULL, tier='cold'")
|
||||
conn.execute("DELETE FROM chunks_fts")
|
||||
conn.commit()
|
||||
|
||||
total_restored = 0
|
||||
for p in result["packs"]:
|
||||
r = pull_pack(conn, backend, p["pack_hash"])
|
||||
total_restored += r["chunks_restored"]
|
||||
hyd = hydrate_from_metadata_pack(
|
||||
conn, backend, result["metadata_pack_hash"], mode="full",
|
||||
)
|
||||
assert hyd["status"] == "hydrated"
|
||||
# Total chunks restored == total chunks across all chunk packs.
|
||||
total_restored = sum(r["chunks_restored"] for r in hyd["chunk_pulls"])
|
||||
assert total_restored == result["total_chunks"]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_push_pack_local_dir_writes_files(tmp_path):
|
||||
"""--local-dir writes pack + manifest files for burning to physical media."""
|
||||
"""--local-dir writes pack + manifest files for burning to physical media.
|
||||
v3: both metadata pack and chunk packs land in local-dir, with kind
|
||||
suffix in the filename."""
|
||||
db = tmp_path / "burn.db"
|
||||
conn = connect(db)
|
||||
backend = MemoryBackend()
|
||||
|
|
@ -293,13 +349,17 @@ def test_push_pack_local_dir_writes_files(tmp_path):
|
|||
assert result["status"] == "pushed"
|
||||
assert result["local_dir"] == str(burn_dir)
|
||||
assert burn_dir.exists()
|
||||
pack_files = sorted(burn_dir.glob("arborist-pack-*.tar.zst"))
|
||||
manifest_files = sorted(burn_dir.glob("arborist-pack-*.manifest.ndjson"))
|
||||
assert len(pack_files) == result["pack_count"]
|
||||
assert len(manifest_files) == result["pack_count"]
|
||||
# Local files are byte-identical to what we'd burn.
|
||||
for pack_file, summary in zip(pack_files, result["packs"]):
|
||||
assert pack_file.stat().st_size == summary["compressed_bytes"]
|
||||
# v3 filenames: arborist-pack-<short>.{metadata,chunks}.tar.zst
|
||||
metadata_files = sorted(burn_dir.glob("arborist-pack-*.metadata.tar.zst"))
|
||||
chunks_files = sorted(burn_dir.glob("arborist-pack-*.chunks.tar.zst"))
|
||||
assert len(metadata_files) == 1
|
||||
assert len(chunks_files) == len(result["chunk_pack_hashes"])
|
||||
# Manifest sidecars present too.
|
||||
assert len(sorted(burn_dir.glob("arborist-pack-*.metadata.manifest.ndjson"))) == 1
|
||||
assert (
|
||||
len(sorted(burn_dir.glob("arborist-pack-*.chunks.manifest.ndjson")))
|
||||
== len(result["chunk_pack_hashes"])
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
|
@ -373,44 +433,44 @@ def test_edges_fan_in_batches_huge_destinations(tmp_path):
|
|||
conn.close()
|
||||
|
||||
|
||||
def test_v2_pack_hash_binds_table_contents(tmp_path):
|
||||
"""Two packs with the same chunk set but different table content
|
||||
must NOT collide on pack_hash.
|
||||
|
||||
The v1 bug: pack_hash was hash_leaf(chunk-only-manifest), so
|
||||
same-chunks + different-tables produced same pack_hash. Bucket
|
||||
overwrites or mesh-peer disagreement → silent table swap. v2
|
||||
fixes this by including content_hash for each tables/<name>.jsonl
|
||||
in the manifest before chunk entries.
|
||||
|
||||
This test forces the issue by ingesting two different document
|
||||
sets that happen to share a chunk — wait, that's impossible by
|
||||
construction. Instead we check the manifest format directly: a
|
||||
v2 pack's manifest has a `_format_version: 2` header and
|
||||
`table_file` entries before `leaf_hash` entries.
|
||||
"""
|
||||
db = tmp_path / "v2bind.db"
|
||||
def test_v3_metadata_pack_manifest_shape(tmp_path):
|
||||
"""v3 metadata pack manifest carries: _format_version=3, _kind="metadata",
|
||||
_snapshot_root, _chunk_pack_hashes list, and one table_file entry per
|
||||
non-empty shipped table. Chunks pack manifest separately carries
|
||||
_kind="chunks" and leaf_hash entries."""
|
||||
db = tmp_path / "v3shape.db"
|
||||
conn = connect(db)
|
||||
backend = MemoryBackend()
|
||||
try:
|
||||
ingest_source(conn, FakeSource([_doc("html://a", LONG)]))
|
||||
result = push_pack(conn, backend)
|
||||
manifest = backend.get_pack_manifest(result["packs"][0]["pack_hash"])
|
||||
parsed = parse_manifest(manifest)
|
||||
metadata_hash = result["metadata_pack_hash"]
|
||||
chunk_hashes = result["chunk_pack_hashes"]
|
||||
assert metadata_hash is not None
|
||||
assert len(chunk_hashes) >= 1
|
||||
|
||||
assert parsed.format_version == 2
|
||||
# All shipped tables that had rows should appear in the manifest.
|
||||
# documents + chunks + merkle_nodes + audit_events minimum.
|
||||
table_names = {t.member_name for t in parsed.tables}
|
||||
meta_manifest = backend.get_pack_manifest(metadata_hash, kind="metadata")
|
||||
meta = parse_manifest(meta_manifest)
|
||||
assert meta.format_version == 3
|
||||
assert meta.kind == "metadata"
|
||||
assert meta.snapshot_root and len(meta.snapshot_root) == 64
|
||||
assert list(meta.chunk_pack_hashes) == chunk_hashes
|
||||
assert meta.chunks == ()
|
||||
table_names = {t.member_name for t in meta.tables}
|
||||
assert "tables/documents.jsonl" in table_names
|
||||
assert "tables/chunks.jsonl" in table_names
|
||||
assert "tables/audit_events.jsonl" in table_names
|
||||
# Every table_file ref carries a hash and size.
|
||||
for t in parsed.tables:
|
||||
assert len(t.content_hash) == 64 # sha256 hex
|
||||
for t in meta.tables:
|
||||
assert len(t.content_hash) == 64
|
||||
assert t.size > 0
|
||||
# Chunks are also there.
|
||||
assert len(parsed.chunks) > 0
|
||||
|
||||
# First chunks pack: chunks only, no tables.
|
||||
first_chunks_manifest = backend.get_pack_manifest(chunk_hashes[0], kind="chunks")
|
||||
ch = parse_manifest(first_chunks_manifest)
|
||||
assert ch.format_version == 3
|
||||
assert ch.kind == "chunks"
|
||||
assert ch.tables == ()
|
||||
assert len(ch.chunks) > 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
|
@ -433,14 +493,14 @@ def test_pull_pack_rejects_tampered_table(tmp_path):
|
|||
try:
|
||||
ingest_source(conn, FakeSource([_doc("html://t", LONG)]))
|
||||
result = push_pack(conn, backend)
|
||||
pack_hash = result["packs"][0]["pack_hash"]
|
||||
metadata_hash = result["metadata_pack_hash"]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# Decompress the pack, rewrite documents.jsonl with corrupted bytes,
|
||||
# repack, replace the pack in the bucket. Manifest stays unchanged
|
||||
# so the recorded table content_hash no longer matches.
|
||||
body = backend.get_pack(pack_hash)
|
||||
# Decompress the metadata pack, rewrite documents.jsonl with corrupted
|
||||
# bytes, repack, replace in the bucket. Manifest stays unchanged so
|
||||
# the recorded table content_hash no longer matches.
|
||||
body = backend.get_pack(metadata_hash, kind="metadata")
|
||||
raw_tar = _zstd.ZstdDecompressor().stream_reader(_io.BytesIO(body)).read()
|
||||
out_buf = _io.BytesIO()
|
||||
cctx = _zstd.ZstdCompressor(level=3)
|
||||
|
|
@ -454,20 +514,20 @@ def test_pull_pack_rejects_tampered_table(tmp_path):
|
|||
data = f.read()
|
||||
if m.name == "tables/documents.jsonl":
|
||||
data = data + b'\n{"injected": "tamper"}\n'
|
||||
info = _tarfile.TarInfo(name=m.name)
|
||||
info.size = len(data)
|
||||
else:
|
||||
info = _tarfile.TarInfo(name=m.name)
|
||||
info.size = len(data)
|
||||
info = _tarfile.TarInfo(name=m.name)
|
||||
info.size = len(data)
|
||||
out_tar.addfile(info, _io.BytesIO(data))
|
||||
backend._tamper(f"packs/{pack_hash}.tar.zst", out_buf.getvalue())
|
||||
backend._tamper(
|
||||
f"packs/{metadata_hash}.metadata.tar.zst", out_buf.getvalue()
|
||||
)
|
||||
|
||||
# Fresh DB; expect ValueError on pull.
|
||||
# Fresh DB; expect ValueError on hydrate (metadata pack tamper fires
|
||||
# the hash-check during pull_metadata_pack).
|
||||
rx_db = tmp_path / "rx.db"
|
||||
conn_rx = connect(rx_db)
|
||||
try:
|
||||
with pytest.raises(ValueError, match="table-hash mismatch"):
|
||||
pull_pack(conn_rx, backend, pack_hash)
|
||||
pull_metadata_pack(conn_rx, backend, metadata_hash)
|
||||
finally:
|
||||
conn_rx.close()
|
||||
|
||||
|
|
@ -507,7 +567,8 @@ def test_push_pack_v2_hydrates_fresh_empty_db(tmp_path):
|
|||
|
||||
result = push_pack(conn_src, backend)
|
||||
assert result["status"] == "pushed"
|
||||
pack_hash = result["packs"][0]["pack_hash"]
|
||||
metadata_hash = result["metadata_pack_hash"]
|
||||
assert metadata_hash is not None
|
||||
finally:
|
||||
conn_src.close()
|
||||
|
||||
|
|
@ -520,20 +581,19 @@ def test_push_pack_v2_hydrates_fresh_empty_db(tmp_path):
|
|||
"SELECT COUNT(*) FROM documents"
|
||||
).fetchone()[0] == 0
|
||||
|
||||
pull_result = pull_pack(conn_rx, backend, pack_hash)
|
||||
assert pull_result["status"] == "pulled"
|
||||
assert pull_result["pack_format"] == "v2"
|
||||
# Every table that was non-empty in src should have rows now in rx.
|
||||
assert pull_result["tables_restored"]["documents"] == src_docs
|
||||
assert pull_result["tables_restored"]["chunks"] == src_chunks
|
||||
assert pull_result["tables_restored"]["merkle_nodes"] == src_merkle
|
||||
assert pull_result["tables_restored"]["audit_events"] == src_audit
|
||||
# Chunk bodies restored.
|
||||
hyd = hydrate_from_metadata_pack(
|
||||
conn_rx, backend, metadata_hash, mode="full",
|
||||
)
|
||||
assert hyd["status"] == "hydrated"
|
||||
assert hyd["mode"] == "full"
|
||||
assert conn_rx.execute(
|
||||
"SELECT COUNT(*) FROM documents"
|
||||
).fetchone()[0] == src_docs
|
||||
# Chunks rows + content restored.
|
||||
rx_hot = conn_rx.execute(
|
||||
"SELECT COUNT(*) FROM chunks WHERE tier='hot' AND content IS NOT NULL"
|
||||
).fetchone()[0]
|
||||
assert rx_hot == src_chunks
|
||||
# FTS5 rows rebuilt.
|
||||
rx_fts = conn_rx.execute("SELECT COUNT(*) FROM chunks_fts").fetchone()[0]
|
||||
assert rx_fts == src_chunks
|
||||
# All three documents reachable by URI.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue