#000065 fix: cross-shard FK refs blow up the migration writer

The 2026-05-26 cutover crashed mid-build with sqlite3.IntegrityError
"FOREIGN KEY constraint failed" inside _route_per_doc_table on the
derivations table.

Root cause: derivations.src_root carries a FK to
documents.document_root, but under content-hash routing a derivation
row's src_root can legitimately reference a surface doc that hashes
to a DIFFERENT target shard than the derivation's core_root. The FK
is a single-shard-era guard; it must stay live for the runtime
write path (to catch typo'd inserts into the wrong shard) but must
be OFF for the migration writer which legitimately produces
cross-shard refs.

Fix: arborist/migrate.py _connect_target now applies
`PRAGMA foreign_keys = OFF` after SCHEMA_SQL executescript runs.
Schema's own `PRAGMA foreign_keys = ON` still applies to the schema
DDL pass (and runtime connect() / connect_query() still get FK=ON
since they don't touch this helper). Only the migration writer is
relaxed. Documented inline.

Regression test
(TestCrossShardForeignKeys.test_cross_shard_derivation_succeeds)
synthesizes a derivation row whose core_root and src_root hash to
different M=4 target shards, runs the migration, asserts the row
lands on core_root's target with src_root pointing cross-shard. Pre-
fix this raised IntegrityError; post-fix it passes.

Originals untouched on the production host — the executor crashed
BEFORE the atomic-promote step, so .db files are intact;
~/.arborist/shards/00X.db.new files from the failed run will be
cleared by the next attempt's "stale .new before opening" cleanup
hook (already in _execute_all_at_once).
This commit is contained in:
russell@unturf.com 2026-05-26 14:12:50 -04:00
parent f4397a9217
commit 04edff7905
No known key found for this signature in database
2 changed files with 130 additions and 1 deletions

View file

@ -406,6 +406,121 @@ class TestInPlacePromotion:
assert not list(target.glob("00[0-9].db"))
class TestCrossShardForeignKeys:
"""Regression test: derivations.src_root can legitimately reference
a surface doc that hash-routes to a different target shard. FK
enforcement on the migration writer connection must be OFF or the
INSERT fails (the 2026-05-26 cutover-crash reproducer)."""
def test_cross_shard_derivation_succeeds(
self, tmp_path: Path
):
"""Synthesize a derivation row whose core_root and src_root
hash to different target shards, run migration, verify it
lands without FK violation."""
# Build a source shard with two docs A and B that we KNOW hash
# to different target shards under M=4. Then INSERT a
# derivation linking them, run migration, expect success.
src_dir = tmp_path / "src"
src_dir.mkdir()
src_db = src_dir / "000.db"
conn = connect(src_db)
try:
# Two docs with content chosen so their roots split across
# M=4 targets. Find them by trial.
roots: dict[int, str] = {}
for i in range(200):
content = f"crosshard fixture seed {i} body text " * 30
ds = _FakeSource([
Document(
uri=f"https://test/cs-{i}",
content=content,
source_type="html",
title=f"cs-{i}",
)
])
ingest_source(conn, ds)
row = conn.execute(
"SELECT document_root FROM documents WHERE document_uri = ?",
(f"https://test/cs-{i}",),
).fetchone()
root = row[0]
idx = shard_for_document(root, 4)
roots.setdefault(idx, root)
if len(roots) >= 2:
break
assert len(roots) >= 2, "couldn't find two docs hashing to different shards"
two_idxs = sorted(roots.keys())[:2]
core_root = roots[two_idxs[0]]
src_root = roots[two_idxs[1]]
# Synthesize a derivation linking core_root (target A) to
# src_root (target B != A).
conn.execute(
"INSERT INTO derivations "
"(core_root, src_root, proof_blob, process_id, distilled_at) "
"VALUES (?, ?, ?, ?, ?)",
(core_root, src_root, "{}", "test_cross_shard", 0),
)
conn.commit()
finally:
conn.close()
# Extract audit events.
ndjson = tmp_path / "a.ndjson"
with ndjson.open("w") as f:
c = sqlite3.connect(f"file:{src_db}?mode=ro", uri=True)
try:
cur = c.execute(
"SELECT seq, event_hash, prev_event_hash, event_type, "
"subject_root, body, ts FROM audit_events ORDER BY seq"
)
for seq, eh, peh, et, subj, body, ts in cur:
f.write(json.dumps({
"src_shard": 0, "src_seq": seq, "src_event_hash": eh,
"prev_event_hash": peh, "event_type": et,
"subject_root": subj, "body": body, "ts": ts,
}, separators=(",", ":")) + "\n")
finally:
c.close()
target = tmp_path / "tgt"
plan = HydrationPlanner().plan(
source_dir=src_dir, target_dir=target, target_M=4,
free_bytes=200 * 1024 ** 3,
)
# The reproducer: pre-fix this raised sqlite3.IntegrityError
# ("FOREIGN KEY constraint failed") inside _route_per_doc_table.
from arborist.migrate import execute_plan as run
run(plan, audit_events_ndjson=ndjson)
# Verify: derivation row lands on the core_root's target shard
# (where the row is routed by `core_root`); src_root reference
# points cross-shard.
tdb = target / f"{shard_for_document(core_root, 4):03d}.db"
c = sqlite3.connect(f"file:{tdb}?mode=ro", uri=True)
try:
row = c.execute(
"SELECT core_root, src_root FROM derivations "
"WHERE process_id = 'test_cross_shard'"
).fetchone()
finally:
c.close()
assert row is not None
assert row[0] == core_root
assert row[1] == src_root
# src_root is NOT in this same target shard — that's the
# cross-shard semantic this test exists to protect.
c = sqlite3.connect(f"file:{tdb}?mode=ro", uri=True)
try:
row = c.execute(
"SELECT document_root FROM documents WHERE document_root = ?",
(src_root,),
).fetchone()
finally:
c.close()
assert row is None, "src_root unexpectedly local; fixture didn't cross-shard"
class TestExecutorErrorCases:
def test_missing_ndjson_raises(
self, source_shards: Path, tmp_path: Path