#000061: bind pack_hash to full pack content (tables + chunks)
Caught a real defect on the live v2 corpus run: pack_hash was computed
from the chunk-only manifest, so two packs with the same chunk set but
different table content collided on pack_hash. Observed live: v1 packs
and v2 packs for the same shard produced the SAME pack_hash:
v1 (chunks-only) b3c427baaf9b40e3c33ace8438b4254e3abdc7b38bdcb7b1a8012588cb32848a
v2 (tables + chunks) b3c427baaf9b40e3c33ace8438b4254e3abdc7b38bdcb7b1a8012588cb32848a
↑ identical, different bytes
Consequences if shipped: bucket overwrites swap whose tables you get,
mesh peers A and B disagree on what "pack X" is while both claim to
have it, content-addressing story is broken for the metadata half.
Fix: manifest format v2 prepends a `{"_format_version": 2}` header,
then one row per shipped table:
{"table_file": "tables/audit_events.jsonl", "hash": "...", "size": N}
{"table_file": "tables/chunks.jsonl", "hash": "...", "size": N}
...
{"leaf_hash": "...", "size": N} # chunks come after
{"leaf_hash": "...", "size": N}
pack_hash = hash_leaf(manifest_bytes) now binds the full pack content.
Two writers with the same shard state produce the same pack_hash; same
chunks + different tables produce different pack_hashes.
Implementation:
- hash_file_leaf(path) — streaming sha256 with leaf 0x00 prefix, 1 MB
reads, used to fingerprint multi-GB tables/<name>.jsonl files without
loading them into RAM.
- stream_packs' extra_members now takes (member_name, source_path,
content_hash) tuples. push_pack computes the hash with hash_file_leaf
after the dump finishes.
- _finalize_pack writes the format header + table refs (sorted by name
for determinism) before chunk entries.
- parse_manifest returns a ParsedManifest (format_version, tables[],
chunks[]) instead of just list[PackEntry]. Backward-compatible with
v1 manifests (no header → format_version=1, tables=()).
- pull_pack pulls the manifest aside during tar walk, then after
extraction verifies each tables/<name>.jsonl file via hash_file_leaf
against the manifest's content_hash. Mismatch → ValueError, no rows
reach the live schema.
New tests:
- test_v2_pack_hash_binds_table_contents — asserts manifest format +
table refs + per-table hash & size.
- test_pull_pack_rejects_tampered_table — corrupts one table's bytes
in a real pack, confirms pull_pack raises on hash mismatch.
27 cold-object + evict tests pass.
This commit is contained in:
parent
bc7efe4434
commit
a193394ea8
4 changed files with 271 additions and 24 deletions
|
|
@ -100,10 +100,15 @@ def test_parse_manifest_round_trip():
|
|||
body = b"abc" * 100
|
||||
leaf = hash_leaf(body).hex()
|
||||
pack = build_pack([(leaf, body)])
|
||||
entries = parse_manifest(pack.manifest_bytes)
|
||||
assert len(entries) == 1
|
||||
assert entries[0].leaf_hash == leaf
|
||||
assert entries[0].size == len(body)
|
||||
parsed = parse_manifest(pack.manifest_bytes)
|
||||
# build_pack produces a chunks-only manifest (no tables) — that's the
|
||||
# v1 shape because build_pack doesn't accept extra_members. Tests of
|
||||
# the v2 shape live in test_push_pack_v2_hydrates_fresh_empty_db.
|
||||
assert parsed.format_version == 1
|
||||
assert parsed.tables == ()
|
||||
assert len(parsed.chunks) == 1
|
||||
assert parsed.chunks[0].leaf_hash == leaf
|
||||
assert parsed.chunks[0].size == len(body)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -368,6 +373,105 @@ def test_edges_fan_in_batches_huge_destinations(tmp_path):
|
|||
conn.close()
|
||||
|
||||
|
||||
def test_v2_pack_hash_binds_table_contents(tmp_path):
|
||||
"""Two packs with the same chunk set but different table content
|
||||
must NOT collide on pack_hash.
|
||||
|
||||
The v1 bug: pack_hash was hash_leaf(chunk-only-manifest), so
|
||||
same-chunks + different-tables produced same pack_hash. Bucket
|
||||
overwrites or mesh-peer disagreement → silent table swap. v2
|
||||
fixes this by including content_hash for each tables/<name>.jsonl
|
||||
in the manifest before chunk entries.
|
||||
|
||||
This test forces the issue by ingesting two different document
|
||||
sets that happen to share a chunk — wait, that's impossible by
|
||||
construction. Instead we check the manifest format directly: a
|
||||
v2 pack's manifest has a `_format_version: 2` header and
|
||||
`table_file` entries before `leaf_hash` entries.
|
||||
"""
|
||||
db = tmp_path / "v2bind.db"
|
||||
conn = connect(db)
|
||||
backend = MemoryBackend()
|
||||
try:
|
||||
ingest_source(conn, FakeSource([_doc("html://a", LONG)]))
|
||||
result = push_pack(conn, backend)
|
||||
manifest = backend.get_pack_manifest(result["packs"][0]["pack_hash"])
|
||||
parsed = parse_manifest(manifest)
|
||||
|
||||
assert parsed.format_version == 2
|
||||
# All shipped tables that had rows should appear in the manifest.
|
||||
# documents + chunks + merkle_nodes + audit_events minimum.
|
||||
table_names = {t.member_name for t in parsed.tables}
|
||||
assert "tables/documents.jsonl" in table_names
|
||||
assert "tables/chunks.jsonl" in table_names
|
||||
assert "tables/audit_events.jsonl" in table_names
|
||||
# Every table_file ref carries a hash and size.
|
||||
for t in parsed.tables:
|
||||
assert len(t.content_hash) == 64 # sha256 hex
|
||||
assert t.size > 0
|
||||
# Chunks are also there.
|
||||
assert len(parsed.chunks) > 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_pull_pack_rejects_tampered_table(tmp_path):
|
||||
"""If the manifest says tables/X.jsonl should hash to H but the tar
|
||||
member has different bytes, pull_pack refuses to restore.
|
||||
|
||||
Tampering test: build a real v2 pack, then construct a corrupted
|
||||
copy that swaps one table's bytes but keeps the original manifest.
|
||||
pull_pack must raise on hash mismatch.
|
||||
"""
|
||||
import io as _io
|
||||
import tarfile as _tarfile
|
||||
import zstandard as _zstd
|
||||
|
||||
db = tmp_path / "src.db"
|
||||
conn = connect(db)
|
||||
backend = MemoryBackend()
|
||||
try:
|
||||
ingest_source(conn, FakeSource([_doc("html://t", LONG)]))
|
||||
result = push_pack(conn, backend)
|
||||
pack_hash = result["packs"][0]["pack_hash"]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# Decompress the pack, rewrite documents.jsonl with corrupted bytes,
|
||||
# repack, replace the pack in the bucket. Manifest stays unchanged
|
||||
# so the recorded table content_hash no longer matches.
|
||||
body = backend.get_pack(pack_hash)
|
||||
raw_tar = _zstd.ZstdDecompressor().stream_reader(_io.BytesIO(body)).read()
|
||||
out_buf = _io.BytesIO()
|
||||
cctx = _zstd.ZstdCompressor(level=3)
|
||||
with cctx.stream_writer(out_buf, closefd=False) as writer:
|
||||
with _tarfile.open(fileobj=writer, mode="w|") as out_tar:
|
||||
with _tarfile.open(fileobj=_io.BytesIO(raw_tar), mode="r") as in_tar:
|
||||
for m in in_tar:
|
||||
f = in_tar.extractfile(m)
|
||||
if f is None:
|
||||
continue
|
||||
data = f.read()
|
||||
if m.name == "tables/documents.jsonl":
|
||||
data = data + b'\n{"injected": "tamper"}\n'
|
||||
info = _tarfile.TarInfo(name=m.name)
|
||||
info.size = len(data)
|
||||
else:
|
||||
info = _tarfile.TarInfo(name=m.name)
|
||||
info.size = len(data)
|
||||
out_tar.addfile(info, _io.BytesIO(data))
|
||||
backend._tamper(f"packs/{pack_hash}.tar.zst", out_buf.getvalue())
|
||||
|
||||
# Fresh DB; expect ValueError on pull.
|
||||
rx_db = tmp_path / "rx.db"
|
||||
conn_rx = connect(rx_db)
|
||||
try:
|
||||
with pytest.raises(ValueError, match="table-hash mismatch"):
|
||||
pull_pack(conn_rx, backend, pack_hash)
|
||||
finally:
|
||||
conn_rx.close()
|
||||
|
||||
|
||||
def test_push_pack_v2_hydrates_fresh_empty_db(tmp_path):
|
||||
"""Pack format v2 self-sufficiency test: build a pack from a populated
|
||||
DB, then unpack it into a FRESH empty DB and verify every table is
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue