#000065 fix: cross-shard FK refs blow up the migration writer
The 2026-05-26 cutover crashed mid-build with sqlite3.IntegrityError "FOREIGN KEY constraint failed" inside _route_per_doc_table on the derivations table. Root cause: derivations.src_root carries a FK to documents.document_root, but under content-hash routing a derivation row's src_root can legitimately reference a surface doc that hashes to a DIFFERENT target shard than the derivation's core_root. The FK is a single-shard-era guard; it must stay live for the runtime write path (to catch typo'd inserts into the wrong shard) but must be OFF for the migration writer which legitimately produces cross-shard refs. Fix: arborist/migrate.py _connect_target now applies `PRAGMA foreign_keys = OFF` after SCHEMA_SQL executescript runs. Schema's own `PRAGMA foreign_keys = ON` still applies to the schema DDL pass (and runtime connect() / connect_query() still get FK=ON since they don't touch this helper). Only the migration writer is relaxed. Documented inline. Regression test (TestCrossShardForeignKeys.test_cross_shard_derivation_succeeds) synthesizes a derivation row whose core_root and src_root hash to different M=4 target shards, runs the migration, asserts the row lands on core_root's target with src_root pointing cross-shard. Pre- fix this raised IntegrityError; post-fix it passes. Originals untouched on the production host — the executor crashed BEFORE the atomic-promote step, so .db files are intact; ~/.arborist/shards/00X.db.new files from the failed run will be cleared by the next attempt's "stale .new before opening" cleanup hook (already in _execute_all_at_once).
This commit is contained in:
parent
f4397a9217
commit
04edff7905
2 changed files with 130 additions and 1 deletions
|
|
@ -408,7 +408,17 @@ class ExecutionResult:
|
|||
|
||||
|
||||
def _connect_target(path: Path) -> sqlite3.Connection:
|
||||
"""Open or create a target shard with the full v9.8 schema."""
|
||||
"""Open or create a target shard with the full v9.8 schema.
|
||||
|
||||
FK enforcement is **disabled** on this connection. Reason:
|
||||
``derivations.src_root`` carries a foreign key to
|
||||
``documents.document_root``, but under content-hash routing a
|
||||
derivation can legitimately reference a source surface doc that
|
||||
hashes to a different target shard. The FK was a single-shard-era
|
||||
guard and would falsely reject a cross-shard reference during
|
||||
migration. Runtime ``connect()`` / ``connect_query()`` paths keep
|
||||
FK=ON; only the migration writer connection turns it off.
|
||||
"""
|
||||
from arborist.store import SCHEMA_SQL
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
|
@ -416,6 +426,10 @@ def _connect_target(path: Path) -> sqlite3.Connection:
|
|||
conn.row_factory = sqlite3.Row
|
||||
conn.executescript("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;")
|
||||
conn.executescript(SCHEMA_SQL)
|
||||
# The schema executescript above flips FK=ON. Disable for migration
|
||||
# writes (post-applied because schema's own PRAGMA wins inside the
|
||||
# script's scope).
|
||||
conn.execute("PRAGMA foreign_keys = OFF")
|
||||
return conn
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -406,6 +406,121 @@ class TestInPlacePromotion:
|
|||
assert not list(target.glob("00[0-9].db"))
|
||||
|
||||
|
||||
class TestCrossShardForeignKeys:
|
||||
"""Regression test: derivations.src_root can legitimately reference
|
||||
a surface doc that hash-routes to a different target shard. FK
|
||||
enforcement on the migration writer connection must be OFF or the
|
||||
INSERT fails (the 2026-05-26 cutover-crash reproducer)."""
|
||||
|
||||
def test_cross_shard_derivation_succeeds(
|
||||
self, tmp_path: Path
|
||||
):
|
||||
"""Synthesize a derivation row whose core_root and src_root
|
||||
hash to different target shards, run migration, verify it
|
||||
lands without FK violation."""
|
||||
# Build a source shard with two docs A and B that we KNOW hash
|
||||
# to different target shards under M=4. Then INSERT a
|
||||
# derivation linking them, run migration, expect success.
|
||||
src_dir = tmp_path / "src"
|
||||
src_dir.mkdir()
|
||||
src_db = src_dir / "000.db"
|
||||
conn = connect(src_db)
|
||||
try:
|
||||
# Two docs with content chosen so their roots split across
|
||||
# M=4 targets. Find them by trial.
|
||||
roots: dict[int, str] = {}
|
||||
for i in range(200):
|
||||
content = f"crosshard fixture seed {i} body text " * 30
|
||||
ds = _FakeSource([
|
||||
Document(
|
||||
uri=f"https://test/cs-{i}",
|
||||
content=content,
|
||||
source_type="html",
|
||||
title=f"cs-{i}",
|
||||
)
|
||||
])
|
||||
ingest_source(conn, ds)
|
||||
row = conn.execute(
|
||||
"SELECT document_root FROM documents WHERE document_uri = ?",
|
||||
(f"https://test/cs-{i}",),
|
||||
).fetchone()
|
||||
root = row[0]
|
||||
idx = shard_for_document(root, 4)
|
||||
roots.setdefault(idx, root)
|
||||
if len(roots) >= 2:
|
||||
break
|
||||
assert len(roots) >= 2, "couldn't find two docs hashing to different shards"
|
||||
two_idxs = sorted(roots.keys())[:2]
|
||||
core_root = roots[two_idxs[0]]
|
||||
src_root = roots[two_idxs[1]]
|
||||
# Synthesize a derivation linking core_root (target A) to
|
||||
# src_root (target B != A).
|
||||
conn.execute(
|
||||
"INSERT INTO derivations "
|
||||
"(core_root, src_root, proof_blob, process_id, distilled_at) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
(core_root, src_root, "{}", "test_cross_shard", 0),
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# Extract audit events.
|
||||
ndjson = tmp_path / "a.ndjson"
|
||||
with ndjson.open("w") as f:
|
||||
c = sqlite3.connect(f"file:{src_db}?mode=ro", uri=True)
|
||||
try:
|
||||
cur = c.execute(
|
||||
"SELECT seq, event_hash, prev_event_hash, event_type, "
|
||||
"subject_root, body, ts FROM audit_events ORDER BY seq"
|
||||
)
|
||||
for seq, eh, peh, et, subj, body, ts in cur:
|
||||
f.write(json.dumps({
|
||||
"src_shard": 0, "src_seq": seq, "src_event_hash": eh,
|
||||
"prev_event_hash": peh, "event_type": et,
|
||||
"subject_root": subj, "body": body, "ts": ts,
|
||||
}, separators=(",", ":")) + "\n")
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
target = tmp_path / "tgt"
|
||||
plan = HydrationPlanner().plan(
|
||||
source_dir=src_dir, target_dir=target, target_M=4,
|
||||
free_bytes=200 * 1024 ** 3,
|
||||
)
|
||||
# The reproducer: pre-fix this raised sqlite3.IntegrityError
|
||||
# ("FOREIGN KEY constraint failed") inside _route_per_doc_table.
|
||||
from arborist.migrate import execute_plan as run
|
||||
run(plan, audit_events_ndjson=ndjson)
|
||||
|
||||
# Verify: derivation row lands on the core_root's target shard
|
||||
# (where the row is routed by `core_root`); src_root reference
|
||||
# points cross-shard.
|
||||
tdb = target / f"{shard_for_document(core_root, 4):03d}.db"
|
||||
c = sqlite3.connect(f"file:{tdb}?mode=ro", uri=True)
|
||||
try:
|
||||
row = c.execute(
|
||||
"SELECT core_root, src_root FROM derivations "
|
||||
"WHERE process_id = 'test_cross_shard'"
|
||||
).fetchone()
|
||||
finally:
|
||||
c.close()
|
||||
assert row is not None
|
||||
assert row[0] == core_root
|
||||
assert row[1] == src_root
|
||||
# src_root is NOT in this same target shard — that's the
|
||||
# cross-shard semantic this test exists to protect.
|
||||
c = sqlite3.connect(f"file:{tdb}?mode=ro", uri=True)
|
||||
try:
|
||||
row = c.execute(
|
||||
"SELECT document_root FROM documents WHERE document_root = ?",
|
||||
(src_root,),
|
||||
).fetchone()
|
||||
finally:
|
||||
c.close()
|
||||
assert row is None, "src_root unexpectedly local; fixture didn't cross-shard"
|
||||
|
||||
|
||||
class TestExecutorErrorCases:
|
||||
def test_missing_ndjson_raises(
|
||||
self, source_shards: Path, tmp_path: Path
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue