#000065 step 3: strategy A executor + CLI + plan-only preview

Build the 'all_at_once' executor and wire it into the CLI as
`arborist corpus reshard`. Plan-only preview against fox's host
confirms the planner picks all_at_once at 95.5 GB free, 64.2 GB
peak draw, 31.3 GB free at peak (well above the 4 GB safety).

Executor (arborist/migrate.py):
  * ROUTED_BY_DOCUMENT_ROOT — documents, document_http_meta, chunks,
    merkle_nodes, edges (by src_root), derivations (by core_root).
    Each row hashed to shard_for_document(root, M) and INSERT'd into
    the chosen target.
  * CONSOLIDATED_TABLES — snapshots, concept_relations,
    concept_token_idf, citation_aliases, term_aliases,
    providence_cache, falsifications all land on canonical
    target shard 000.
  * REBUILT_ON_TARGET — chunks_fts + documents_fts rebuilt from the
    materialized data after content moves. chunks.content is
    zstd-packed at rest so the rebuilder decompresses via
    arborist.compress.unpack_chunk before inserting plaintext into
    the FTS5 index.
  * Audit chain consolidation (Option A) — all rows from
    /tmp/audit-events.ndjson are sorted by (ts, src_shard, src_seq),
    re-chained with fresh event_hash values, and INSERT'd into
    target shard 000. Bodies preserved unchanged for forensic
    fidelity.
  * One 'reshard' audit event appended at the tail, carrying the
    plan + result as the body. An operator months later can answer
    "where did this corpus topology come from" from this one row.
  * corpus_shard_count meta stamped on every target shard.
  * VACUUM each target at end to reclaim INSERT-pattern fragmentation.

CLI: `arborist corpus reshard --to M --source-dir DIR --target-dir DIR
                              [--plan-only | --force-strategy X
                               | --allow-in-place | --dry-run]`

11 integration tests build a tiny 2-shard corpus, run the migration,
verify per-doc routing, chunk-follows-doc, audit chain integrity,
reshard event at tail, corpus_shard_count meta on every target,
chunks_fts searchability, doc count preservation, dry-run no-op,
error handling.

Also: get_meta() now indexes by position so callers without
row_factory=sqlite3.Row don't trip. No behavior change for callers
that DO use Row factory.

Strategy B (per_source_shard) and C (streaming_row) are stubbed in
the planner (peak-draw estimators wired) but execute_plan raises
NotImplementedError for them. Not needed at fox's current disk
(88+ GB free); skipping the build until that's ever the constrained
path.

tests/test_migrate_executor.py

@@ -0,0 +1,343 @@
+"""Integration tests for the strategy A 'all_at_once' executor (#000065).
+
+Builds a tiny 2-shard test corpus, extracts audit events, plans, and
+executes a reshard into a 4-shard target layout. Verifies:
+
+  * every document survives
+  * every document lands in the shard its document_root hashes to
+  * the FTS5 indexes serve search queries on the target
+  * the consolidated audit chain re-verifies on the target
+  * the reshard's own audit event appears at the tail of that chain
+  * every target shard has corpus_shard_count meta = M
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import sqlite3
+from pathlib import Path
+from typing import Iterator
+
+import pytest
+
+from arborist.document import Document, shard_for_document
+from arborist.ingest import ingest_source
+from arborist.migrate import (
+    HydrationPlanner,
+    execute_plan,
+    _table_exists,
+)
+from arborist.source import Source
+from arborist.store import (
+    connect,
+    get_corpus_shard_count,
+)
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+class _FakeSource(Source):
+    source_type = "html"
+
+    def __init__(self, docs: list[Document]):
+        self.docs = docs
+
+    def iter_documents(self) -> Iterator[Document]:
+        yield from self.docs
+
+
+def _make_doc(uri: str, seed: str) -> Document:
+    # Long enough to chunk into 2+ chunks. ``seed`` varies content so
+    # document_root is distinct per doc (the Merkle root keys on
+    # CONTENT, not URI — identical content dedups in the cache).
+    content = (
+        f"This is the document at {uri} with content seed {seed}. " * 10
+        + f"Body text {seed} with enough tokens for proper chunking. " * 30
+    )
+    return Document(
+        uri=uri, content=content, source_type="html", title=uri.split("/")[-1]
+    )
+
+
+@pytest.fixture
+def source_shards(tmp_path: Path) -> Path:
+    """Two source shards, ~15 docs each."""
+    src = tmp_path / "src"
+    src.mkdir()
+    docs_per_shard = 15
+    for shard_idx in range(2):
+        db = src / f"{shard_idx:03d}.db"
+        conn = connect(db)
+        try:
+            docs = [
+                _make_doc(
+                    f"https://example.com/shard{shard_idx}/doc-{i}",
+                    seed=f"s{shard_idx}d{i}",
+                )
+                for i in range(docs_per_shard)
+            ]
+            ingest_source(conn, _FakeSource(docs))
+        finally:
+            conn.close()
+    return src
+
+
+@pytest.fixture
+def audit_ndjson(source_shards: Path, tmp_path: Path) -> Path:
+    """Extract the source-shards' audit chain to ndjson (what
+    bench/extract_audit_events.py would produce on the real host)."""
+    out = tmp_path / "audit-events.ndjson"
+    with out.open("w") as f:
+        for shard in sorted(source_shards.glob("00[0-9].db")):
+            idx = int(shard.stem)
+            c = sqlite3.connect(f"file:{shard}?mode=ro", uri=True)
+            try:
+                cur = c.execute(
+                    "SELECT seq, event_hash, prev_event_hash, event_type, "
+                    "subject_root, body, ts FROM audit_events ORDER BY seq ASC"
+                )
+                for seq, eh, peh, etype, subj, body, ts in cur:
+                    f.write(json.dumps({
+                        "src_shard": idx,
+                        "src_seq": seq,
+                        "src_event_hash": eh,
+                        "prev_event_hash": peh,
+                        "event_type": etype,
+                        "subject_root": subj,
+                        "body": body,
+                        "ts": ts,
+                    }, separators=(",", ":")))
+                    f.write("\n")
+            finally:
+                c.close()
+    return out
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+class TestMigrationAllAtOnce:
+    @pytest.fixture
+    def migrated(self, source_shards: Path, audit_ndjson: Path, tmp_path: Path):
+        target = tmp_path / "tgt"
+        plan = HydrationPlanner().plan(
+            source_dir=source_shards,
+            target_dir=target,
+            target_M=4,
+            free_bytes=200 * 1024 ** 3,  # plenty
+        )
+        assert plan.strategy == "all_at_once"
+        result = execute_plan(
+            plan, audit_events_ndjson=audit_ndjson,
+        )
+        return target, plan, result
+
+    def test_target_dir_has_M_shards(self, migrated):
+        target, plan, _ = migrated
+        assert sorted(p.name for p in target.glob("*.db")) == [
+            "000.db", "001.db", "002.db", "003.db"
+        ]
+
+    def test_every_source_doc_lands_in_correct_shard(
+        self, source_shards: Path, migrated
+    ):
+        target, plan, _ = migrated
+        # Build the expected mapping from source shards.
+        expected: dict[str, int] = {}
+        for shard in sorted(source_shards.glob("*.db")):
+            c = sqlite3.connect(f"file:{shard}?mode=ro", uri=True)
+            try:
+                cur = c.execute("SELECT document_root FROM documents")
+                for (root,) in cur:
+                    expected[root] = shard_for_document(root, plan.target_M)
+            finally:
+                c.close()
+
+        # Verify each target shard holds exactly the docs hashed to it.
+        for tidx in range(plan.target_M):
+            tdb = target / f"{tidx:03d}.db"
+            c = sqlite3.connect(f"file:{tdb}?mode=ro", uri=True)
+            try:
+                actual = {
+                    r[0] for r in c.execute("SELECT document_root FROM documents")
+                }
+            finally:
+                c.close()
+            for root, expect_idx in expected.items():
+                if expect_idx == tidx:
+                    assert root in actual, (
+                        f"doc {root[:8]}... expected on shard {tidx} but missing"
+                    )
+                else:
+                    assert root not in actual, (
+                        f"doc {root[:8]}... on shard {tidx} but hashes to {expect_idx}"
+                    )
+
+    def test_every_chunk_followed_its_document(self, migrated):
+        target, plan, _ = migrated
+        # For each target shard, every chunk's document_root must route here.
+        for tidx in range(plan.target_M):
+            tdb = target / f"{tidx:03d}.db"
+            c = sqlite3.connect(f"file:{tdb}?mode=ro", uri=True)
+            try:
+                cur = c.execute(
+                    "SELECT DISTINCT document_root FROM chunks"
+                )
+                for (root,) in cur:
+                    assert shard_for_document(root, plan.target_M) == tidx, (
+                        f"chunk doc {root[:8]}... on shard {tidx} but routes "
+                        f"to {shard_for_document(root, plan.target_M)}"
+                    )
+            finally:
+                c.close()
+
+    def test_audit_chain_valid_on_target(self, migrated):
+        target, plan, result = migrated
+        tdb = target / "000.db"
+        c = sqlite3.connect(f"file:{tdb}?mode=ro", uri=True)
+        try:
+            cur = c.execute(
+                "SELECT seq, event_hash, prev_event_hash, body "
+                "FROM audit_events ORDER BY seq ASC"
+            )
+            prev = None
+            for seq, event_hash, peh, body in cur:
+                assert peh == prev, f"chain break at seq {seq}"
+                h = hashlib.sha256()
+                if prev is not None:
+                    h.update(bytes.fromhex(prev))
+                h.update(body.encode("utf-8", errors="surrogatepass"))
+                assert h.hexdigest() == event_hash, (
+                    f"hash mismatch at seq {seq}"
+                )
+                prev = event_hash
+        finally:
+            c.close()
+
+    def test_reshard_event_at_tail(self, migrated):
+        target, plan, result = migrated
+        tdb = target / "000.db"
+        c = sqlite3.connect(f"file:{tdb}?mode=ro", uri=True)
+        try:
+            row = c.execute(
+                "SELECT event_type, body FROM audit_events "
+                "ORDER BY seq DESC LIMIT 1"
+            ).fetchone()
+        finally:
+            c.close()
+        assert row[0] == "reshard"
+        body = json.loads(row[1])
+        assert body["kind"] == "reshard"
+        assert body["plan"]["strategy"] == "all_at_once"
+        assert body["plan"]["target_M"] == 4
+        assert body["result"]["rows_moved_total"] > 0
+
+    def test_corpus_shard_count_set_on_every_target(self, migrated):
+        target, plan, _ = migrated
+        for tidx in range(plan.target_M):
+            tdb = target / f"{tidx:03d}.db"
+            c = sqlite3.connect(f"file:{tdb}?mode=ro", uri=True)
+            try:
+                assert get_corpus_shard_count(c) == plan.target_M, (
+                    f"shard {tidx} missing corpus_shard_count meta"
+                )
+            finally:
+                c.close()
+
+    def test_chunks_fts_searchable_on_target(self, migrated):
+        target, plan, _ = migrated
+        # Verify FTS5 index serves a hit on every target shard that has
+        # documents — the test content uses the word "document" liberally.
+        hit_count = 0
+        for tidx in range(plan.target_M):
+            tdb = target / f"{tidx:03d}.db"
+            c = sqlite3.connect(f"file:{tdb}?mode=ro", uri=True)
+            try:
+                row = c.execute(
+                    "SELECT COUNT(*) FROM chunks_fts WHERE chunks_fts MATCH ?",
+                    ("document",),
+                ).fetchone()
+                hit_count += int(row[0])
+            finally:
+                c.close()
+        assert hit_count > 0, "no chunks_fts hits across any target shard"
+
+    def test_total_doc_count_preserved(
+        self, source_shards: Path, migrated
+    ):
+        target, _, _ = migrated
+        # 2 source × 15 docs = 30 unique docs.
+        src_total = 0
+        for shard in source_shards.glob("*.db"):
+            c = sqlite3.connect(f"file:{shard}?mode=ro", uri=True)
+            try:
+                src_total += c.execute(
+                    "SELECT COUNT(*) FROM documents"
+                ).fetchone()[0]
+            finally:
+                c.close()
+        tgt_total = 0
+        for shard in target.glob("*.db"):
+            c = sqlite3.connect(f"file:{shard}?mode=ro", uri=True)
+            try:
+                tgt_total += c.execute(
+                    "SELECT COUNT(*) FROM documents"
+                ).fetchone()[0]
+            finally:
+                c.close()
+        assert tgt_total == src_total, (
+            f"doc count drift: src={src_total} tgt={tgt_total}"
+        )
+
+    def test_dry_run_creates_no_targets(
+        self, source_shards: Path, audit_ndjson: Path, tmp_path: Path
+    ):
+        target = tmp_path / "tgt-dry"
+        plan = HydrationPlanner().plan(
+            source_dir=source_shards,
+            target_dir=target,
+            target_M=4,
+            free_bytes=200 * 1024 ** 3,
+        )
+        result = execute_plan(
+            plan, audit_events_ndjson=audit_ndjson, dry_run=True
+        )
+        assert "DRY RUN" in " ".join(result.notes)
+        # target dir exists but no DBs.
+        assert list(target.glob("*.db")) == []
+
+
+class TestExecutorErrorCases:
+    def test_missing_ndjson_raises(
+        self, source_shards: Path, tmp_path: Path
+    ):
+        target = tmp_path / "tgt"
+        plan = HydrationPlanner().plan(
+            source_dir=source_shards,
+            target_dir=target,
+            target_M=4,
+            free_bytes=200 * 1024 ** 3,
+        )
+        with pytest.raises(FileNotFoundError, match="audit-event ndjson"):
+            execute_plan(
+                plan, audit_events_ndjson=tmp_path / "does-not-exist.ndjson"
+            )
+
+    def test_per_source_shard_not_implemented_yet(
+        self, source_shards: Path, audit_ndjson: Path, tmp_path: Path
+    ):
+        target = tmp_path / "tgt"
+        plan = HydrationPlanner().plan(
+            source_dir=source_shards,
+            target_dir=target,
+            target_M=4,
+            free_bytes=200 * 1024 ** 3,
+            force_strategy="per_source_shard",
+        )
+        with pytest.raises(NotImplementedError, match="per_source_shard"):
+            execute_plan(plan, audit_events_ndjson=audit_ndjson)