diff --git a/arborist/migrate.py b/arborist/migrate.py index ae9226c..03d766f 100644 --- a/arborist/migrate.py +++ b/arborist/migrate.py @@ -408,7 +408,17 @@ class ExecutionResult: def _connect_target(path: Path) -> sqlite3.Connection: - """Open or create a target shard with the full v9.8 schema.""" + """Open or create a target shard with the full v9.8 schema. + + FK enforcement is **disabled** on this connection. Reason: + ``derivations.src_root`` carries a foreign key to + ``documents.document_root``, but under content-hash routing a + derivation can legitimately reference a source surface doc that + hashes to a different target shard. The FK was a single-shard-era + guard and would falsely reject a cross-shard reference during + migration. Runtime ``connect()`` / ``connect_query()`` paths keep + FK=ON; only the migration writer connection turns it off. + """ from arborist.store import SCHEMA_SQL path.parent.mkdir(parents=True, exist_ok=True) @@ -416,6 +426,10 @@ def _connect_target(path: Path) -> sqlite3.Connection: conn.row_factory = sqlite3.Row conn.executescript("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;") conn.executescript(SCHEMA_SQL) + # The schema executescript above flips FK=ON. Disable for migration + # writes (post-applied because schema's own PRAGMA wins inside the + # script's scope). + conn.execute("PRAGMA foreign_keys = OFF") return conn diff --git a/tests/test_migrate_executor.py b/tests/test_migrate_executor.py index ff7d640..00b0967 100644 --- a/tests/test_migrate_executor.py +++ b/tests/test_migrate_executor.py @@ -406,6 +406,121 @@ class TestInPlacePromotion: assert not list(target.glob("00[0-9].db")) +class TestCrossShardForeignKeys: + """Regression test: derivations.src_root can legitimately reference + a surface doc that hash-routes to a different target shard. FK + enforcement on the migration writer connection must be OFF or the + INSERT fails (the 2026-05-26 cutover-crash reproducer).""" + + def test_cross_shard_derivation_succeeds( + self, tmp_path: Path + ): + """Synthesize a derivation row whose core_root and src_root + hash to different target shards, run migration, verify it + lands without FK violation.""" + # Build a source shard with two docs A and B that we KNOW hash + # to different target shards under M=4. Then INSERT a + # derivation linking them, run migration, expect success. + src_dir = tmp_path / "src" + src_dir.mkdir() + src_db = src_dir / "000.db" + conn = connect(src_db) + try: + # Two docs with content chosen so their roots split across + # M=4 targets. Find them by trial. + roots: dict[int, str] = {} + for i in range(200): + content = f"crosshard fixture seed {i} body text " * 30 + ds = _FakeSource([ + Document( + uri=f"https://test/cs-{i}", + content=content, + source_type="html", + title=f"cs-{i}", + ) + ]) + ingest_source(conn, ds) + row = conn.execute( + "SELECT document_root FROM documents WHERE document_uri = ?", + (f"https://test/cs-{i}",), + ).fetchone() + root = row[0] + idx = shard_for_document(root, 4) + roots.setdefault(idx, root) + if len(roots) >= 2: + break + assert len(roots) >= 2, "couldn't find two docs hashing to different shards" + two_idxs = sorted(roots.keys())[:2] + core_root = roots[two_idxs[0]] + src_root = roots[two_idxs[1]] + # Synthesize a derivation linking core_root (target A) to + # src_root (target B != A). + conn.execute( + "INSERT INTO derivations " + "(core_root, src_root, proof_blob, process_id, distilled_at) " + "VALUES (?, ?, ?, ?, ?)", + (core_root, src_root, "{}", "test_cross_shard", 0), + ) + conn.commit() + finally: + conn.close() + + # Extract audit events. + ndjson = tmp_path / "a.ndjson" + with ndjson.open("w") as f: + c = sqlite3.connect(f"file:{src_db}?mode=ro", uri=True) + try: + cur = c.execute( + "SELECT seq, event_hash, prev_event_hash, event_type, " + "subject_root, body, ts FROM audit_events ORDER BY seq" + ) + for seq, eh, peh, et, subj, body, ts in cur: + f.write(json.dumps({ + "src_shard": 0, "src_seq": seq, "src_event_hash": eh, + "prev_event_hash": peh, "event_type": et, + "subject_root": subj, "body": body, "ts": ts, + }, separators=(",", ":")) + "\n") + finally: + c.close() + + target = tmp_path / "tgt" + plan = HydrationPlanner().plan( + source_dir=src_dir, target_dir=target, target_M=4, + free_bytes=200 * 1024 ** 3, + ) + # The reproducer: pre-fix this raised sqlite3.IntegrityError + # ("FOREIGN KEY constraint failed") inside _route_per_doc_table. + from arborist.migrate import execute_plan as run + run(plan, audit_events_ndjson=ndjson) + + # Verify: derivation row lands on the core_root's target shard + # (where the row is routed by `core_root`); src_root reference + # points cross-shard. + tdb = target / f"{shard_for_document(core_root, 4):03d}.db" + c = sqlite3.connect(f"file:{tdb}?mode=ro", uri=True) + try: + row = c.execute( + "SELECT core_root, src_root FROM derivations " + "WHERE process_id = 'test_cross_shard'" + ).fetchone() + finally: + c.close() + assert row is not None + assert row[0] == core_root + assert row[1] == src_root + # src_root is NOT in this same target shard — that's the + # cross-shard semantic this test exists to protect. + c = sqlite3.connect(f"file:{tdb}?mode=ro", uri=True) + try: + row = c.execute( + "SELECT document_root FROM documents WHERE document_root = ?", + (src_root,), + ).fetchone() + finally: + c.close() + assert row is None, "src_root unexpectedly local; fixture didn't cross-shard" + + class TestExecutorErrorCases: def test_missing_ndjson_raises( self, source_shards: Path, tmp_path: Path