diff --git a/bench/results/shard-count-sweep-2026-05-26T16-20-48Z.csv b/bench/results/shard-count-sweep-2026-05-26T16-20-48Z.csv new file mode 100644 index 0000000..8a6c8c8 --- /dev/null +++ b/bench/results/shard-count-sweep-2026-05-26T16-20-48Z.csv @@ -0,0 +1,9 @@ +M,attach_ms,mean_ms,p50_ms,p99_ms,per_shard_size_max,per_shard_size_min,per_worker_peak_rss_mb_max,per_worker_wall_s_max,per_worker_wall_s_min,phase,queries,total_chunks,wall_s +1,,,,,1000,1000,55.73828125,0.9100882040220313,0.9100882040220313,ingest,,2000,0.9116959809907712 +1,2.4829289759509265,0.18545756582170725,0.13220100663602352,0.5144660244695842,,,,,,query,50,,0.009308646956924349 +2,,,,,516,484,47.69921875,0.39443900901824236,0.36702751100528985,ingest,,2000,0.4271319890394807 +2,3.2454620231874287,0.09982534102164209,0.0736890360713005,0.31164102256298065,,,,,,query,50,,0.005017360963393003 +4,,,,,260,234,45.69921875,0.26050548697821796,0.22189977602101862,ingest,,2000,0.28043953998712823 +4,5.050152947660536,0.050168324960395694,0.03760302206501365,0.19613601034507155,,,,,,query,50,,0.0025283890427090228 +8,,,,,138,102,47.46484375,0.21609534602612257,0.1835811220225878,ingest,,2000,0.25267004797933623 +8,10.680334991775453,0.03288980573415756,0.02434797352179885,0.16062799841165543,,,,,,query,50,,0.0016711389762349427 diff --git a/bench/results/shard-count-sweep-2026-05-26T16-31-34Z.csv b/bench/results/shard-count-sweep-2026-05-26T16-31-34Z.csv new file mode 100644 index 0000000..d57da84 --- /dev/null +++ b/bench/results/shard-count-sweep-2026-05-26T16-31-34Z.csv @@ -0,0 +1,9 @@ +M,attach_ms,mean_ms,p50_ms,p99_ms,per_shard_size_max,per_shard_size_min,per_worker_peak_rss_mb_max,per_worker_wall_s_max,per_worker_wall_s_min,phase,queries,total_chunks,wall_s +1,,,,,2000,2000,78.18359375,0.8113912579719909,0.8113912579719909,ingest,,2970,0.8141180259990506 +1,1.6094549791887403,0.05254332674667239,0.04617101512849331,0.19686901941895485,,,,,,query,50,,0.002651231945492327 +2,,,,,1014,986,66.79296875,0.4785758460056968,0.47741828102152795,ingest,,2970,0.5256810069549829 +2,3.243724990170449,0.04392992239445448,0.03423704765737057,0.19317097030580044,,,,,,query,50,,0.0022365679615177214 +4,,,,,513,476,70.64453125,0.4177772310213186,0.3488612399669364,ingest,,2970,0.46363314701011404 +4,8.970557013526559,0.08673752075992525,0.07230904884636402,0.29919500229880214,,,,,,query,50,,0.004397153970785439 +8,,,,,263,218,69.33984375,0.35226107499329373,0.2777373039862141,ingest,,2970,0.4267680179909803 +8,9.652859007474035,0.0429481384344399,0.03150099655613303,0.277136976365,,,,,,query,50,,0.0021916850237175822 diff --git a/bench/shard_count_sweep.py b/bench/shard_count_sweep.py new file mode 100644 index 0000000..61f16dd --- /dev/null +++ b/bench/shard_count_sweep.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +"""Shard-count sweep — measure ingest + query cost as a function of M. + +Variables: + M ∈ {1, 2, 4, 8} shard count for that run + N = M ingest workers (one per shard, no WAL contention) + +Fixed: + corpus = N_DOCS synthetic deterministic-content documents + queries = N_QUERIES fixed FTS5 queries (head + tail tokens) + +Output: + - CSV at bench/results/shard-count-sweep-.csv + - Console summary table + +The script wipes ~/.arborist/bench-shards/ between runs so it never +touches the live corpus at ~/.arborist/shards/. + +This benchmark exists to answer one question for #000065: +**at what M does ingest throughput stop improving, and how does query +latency change?** Answer drives the canonical-M choice. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import os +import random +import shutil +import statistics +import subprocess +import sys +import time +from concurrent.futures import ProcessPoolExecutor, as_completed +from pathlib import Path + +# Make arborist importable from the repo. +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from arborist.document import Document, canonicalize # noqa: E402 +from arborist.ingest import ingest_source # noqa: E402 +from arborist.merkle import MerkleTree, hash_leaf # noqa: E402 +from arborist.source import Source # noqa: E402 +from arborist.sources.wikipedia import WikipediaCurDump # noqa: E402 +from arborist.store import connect, connect_query, discover_shards # noqa: E402 + +BENCH_DIR = Path.home() / ".arborist" / "bench-shards" +RESULTS_DIR = ROOT / "bench" / "results" +WIKI_CUR_PATH = ROOT / "data" / "20030516_cur_tablesql.bz2" + +# Compact for time budget. Each cell of the M-sweep re-ingests N_DOCS, +# and Wikipedia ingest does real wikitext canonicalization + edge +# extraction so per-doc cost is realistic. +N_DOCS = 2000 +N_QUERIES = 50 +SHARD_COUNTS = [1, 2, 4, 8] +USE_WIKIPEDIA = True # False = synthetic fallback (kept for fast iteration) + + +# --------------------------------------------------------------------------- +# Synthetic corpus generation +# --------------------------------------------------------------------------- + + +WORDS = [ + "alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", + "india", "juliet", "kilo", "lima", "mike", "november", "oscar", "papa", + "quebec", "romeo", "sierra", "tango", "uniform", "victor", "whiskey", + "xray", "yankee", "zulu", + "computer", "wikipedia", "physics", "history", "music", "biology", + "mathematics", "philosophy", "economics", "literature", "chemistry", + "geology", "astronomy", "psychology", "sociology", +] + + +def load_wikipedia_corpus(n_docs: int) -> list[Document]: + """Read the first n_docs articles from the 2003 Wikipedia cur dump. + + Real wikitext, real canonicalization cost — this is the bench cell + that matters for the M-decision because synthetic random-words docs + underestimate ingest per-doc work. + """ + if not WIKI_CUR_PATH.exists(): + raise FileNotFoundError( + f"Wikipedia dump not found at {WIKI_CUR_PATH}. " + f"Run `make fetch-cur` first." + ) + src = WikipediaCurDump(WIKI_CUR_PATH) + docs = [] + for doc in src.iter_documents(): + docs.append(doc) + if len(docs) >= n_docs: + break + return docs + + +def gen_corpus(n_docs: int, seed: int = 42) -> list[Document]: + """Generate N deterministic synthetic docs. Same seed → same content → + same document_root across runs. Routes deterministically by hash.""" + rng = random.Random(seed) + docs = [] + for i in range(n_docs): + # Each doc ~3 KB of text drawn from word pool. Common words appear + # more often (so FTS queries on them hit many docs), some rare + # tokens (doc_NNN tags) appear in exactly one doc. + sentences = [] + for _ in range(60): + length = rng.randint(8, 20) + words = rng.choices(WORDS, k=length) + sentences.append(" ".join(words).capitalize() + ".") + # Inject a unique rare token per doc so we can do "find doc X" queries. + sentences.append(f"This is the unique marker bench_doc_{i:04d}.") + content = " ".join(sentences) + docs.append(Document( + uri=f"bench://doc_{i:04d}", + content=content, + source_type="html", # so license_class is "unknown" but ingest works + title=f"Bench Doc {i:04d}", + )) + return docs + + +def doc_to_shard(doc: Document, M: int) -> int: + """Content-hash routing: deterministic shard index from canonicalized + content. Matches the #000065 routing function — first 8 hex chars of + document_root (= leaf-Merkle root over canonical chunks).""" + # Approximate document_root using a stand-in: hash of canonical content + # bytes. For the bench, what matters is determinism + uniform spread, + # not exact match to ingest's hash. + h = hashlib.sha256(canonicalize(doc.content).encode("utf-8")).hexdigest() + return int(h[:8], 16) % M + + +class ListSource(Source): + source_type = "html" + + def __init__(self, docs: list[Document]): + self.docs = docs + + def iter_documents(self): + yield from self.docs + + +# --------------------------------------------------------------------------- +# Ingest worker — runs in a subprocess so SQLite writer locks are real +# --------------------------------------------------------------------------- + + +def _ingest_worker(args: tuple[str, list[dict]]) -> dict: + """Subprocess worker: ingest the assigned docs into one shard. + args = (db_path, list of {uri, content, title} dicts). + Returns timing + RSS info. + """ + import resource + # Bench infrastructure recycles shard paths between runs. Clear the + # migration cache so each iteration re-runs SCHEMA_SQL on the fresh + # (just-deleted) DB. Without this, the M=2+ iterations see a path + # in _MIGRATED_SHARDS and skip schema creation → "no such table". + from arborist import store as _store + _store._MIGRATED_SHARDS.clear() + db_path, doc_dicts = args + docs = [ + Document( + uri=d["uri"], content=d["content"], + source_type="html", title=d["title"], + ) + for d in doc_dicts + ] + src = ListSource(docs) + t0 = time.perf_counter() + conn = connect(db_path) + try: + result = ingest_source(conn, src) + finally: + conn.close() + wall = time.perf_counter() - t0 + rss_kb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return { + "db": db_path, + "wall_s": wall, + "docs_ingested": result.inserted, + "chunks_ingested": result.chunks_total, + "peak_rss_mb": rss_kb / 1024, + } + + +# --------------------------------------------------------------------------- +# Bench loop +# --------------------------------------------------------------------------- + + +def reset_bench_dir() -> None: + if BENCH_DIR.exists(): + shutil.rmtree(BENCH_DIR) + BENCH_DIR.mkdir(parents=True) + # Also clear the parent process's migration cache; the in-process + # M=1 path uses it. + from arborist import store as _store + _store._MIGRATED_SHARDS.clear() + + +def bench_ingest(M: int, docs: list[Document]) -> dict: + """Run ingest with M shards + M parallel workers. Returns aggregate stats.""" + reset_bench_dir() + # Hash-partition docs into M groups. + groups: list[list[Document]] = [[] for _ in range(M)] + for doc in docs: + groups[doc_to_shard(doc, M)].append(doc) + sizes = [len(g) for g in groups] + # Each worker gets one shard. + db_paths = [str(BENCH_DIR / f"{i:03d}.db") for i in range(M)] + args_list = [ + (db_paths[i], [ + {"uri": d.uri, "content": d.content, "title": d.title} + for d in groups[i] + ]) + for i in range(M) + ] + t0 = time.perf_counter() + worker_results = [] + if M == 1: + # No need for process pool when there's one shard. + worker_results.append(_ingest_worker(args_list[0])) + else: + with ProcessPoolExecutor(max_workers=M) as pool: + futures = [pool.submit(_ingest_worker, args) for args in args_list] + for fut in as_completed(futures): + worker_results.append(fut.result()) + wall = time.perf_counter() - t0 + total_chunks = sum(r["chunks_ingested"] for r in worker_results) + return { + "M": M, + "phase": "ingest", + "wall_s": wall, + "per_shard_size_min": min(sizes), + "per_shard_size_max": max(sizes), + "total_chunks": total_chunks, + "per_worker_wall_s_max": max(r["wall_s"] for r in worker_results), + "per_worker_wall_s_min": min(r["wall_s"] for r in worker_results), + "per_worker_peak_rss_mb_max": max(r["peak_rss_mb"] for r in worker_results), + } + + +def gen_queries(rng_seed: int = 7) -> list[str]: + """Mix of head-term (common, hits many docs) and tail-term (one doc) queries.""" + rng = random.Random(rng_seed) + queries = [] + # Half head queries: random common words. + for _ in range(N_QUERIES // 2): + n_words = rng.randint(1, 3) + queries.append(" ".join(rng.choices(WORDS, k=n_words))) + # Half tail queries: a specific bench_doc_NNNN marker. + sample_docs = rng.sample(range(N_DOCS), N_QUERIES - N_QUERIES // 2) + for i in sample_docs: + queries.append(f"bench_doc_{i:04d}") + return queries + + +def bench_query(M: int, queries: list[str]) -> dict: + """Run query suite against the M-shard layout.""" + # Use connect_query which ATTACHes all the shards. + snap_shards = discover_shards(BENCH_DIR) + if not snap_shards: + return {"M": M, "phase": "query", "wall_s": 0, "p50_ms": 0, "p99_ms": 0, + "queries": 0, "error": "no_shards_discovered"} + t_attach_0 = time.perf_counter() + conn = connect_query(":memory:", shards_dir=BENCH_DIR) + attach_ms = (time.perf_counter() - t_attach_0) * 1000 + try: + latencies_ms = [] + t_total_0 = time.perf_counter() + for q in queries: + t0 = time.perf_counter() + # FTS5 query across the union view. + rows = conn.execute( + "SELECT COUNT(*) FROM chunks_fts WHERE chunks_fts MATCH ?", + (q,), + ).fetchall() + latencies_ms.append((time.perf_counter() - t0) * 1000) + total_wall = time.perf_counter() - t_total_0 + finally: + conn.close() + latencies_ms.sort() + return { + "M": M, + "phase": "query", + "wall_s": total_wall, + "attach_ms": attach_ms, + "p50_ms": latencies_ms[len(latencies_ms) // 2], + "p99_ms": latencies_ms[int(len(latencies_ms) * 0.99)], + "mean_ms": statistics.mean(latencies_ms), + "queries": len(queries), + } + + +def main(): + print(f"== shard-count sweep ==") + print(f" N_DOCS={N_DOCS}, N_QUERIES={N_QUERIES}, M ∈ {SHARD_COUNTS}") + print(f" bench dir: {BENCH_DIR}") + print() + + if USE_WIKIPEDIA: + print(f" corpus: real Wikipedia ({WIKI_CUR_PATH.name}, first {N_DOCS} docs)") + t_load = time.perf_counter() + docs = load_wikipedia_corpus(N_DOCS) + print(f" loaded {len(docs)} docs in {time.perf_counter() - t_load:.1f}s") + else: + print(f" corpus: synthetic ({N_DOCS} docs)") + docs = gen_corpus(N_DOCS) + queries = gen_queries() + RESULTS_DIR.mkdir(parents=True, exist_ok=True) + ts = time.strftime("%Y-%m-%dT%H-%M-%SZ", time.gmtime()) + csv_path = RESULTS_DIR / f"shard-count-sweep-{ts}.csv" + + all_rows = [] + summary = [] + for M in SHARD_COUNTS: + print(f"-- M={M} --") + ingest = bench_ingest(M, docs) + print(f" ingest: {ingest['wall_s']:.2f}s " + f"chunks={ingest['total_chunks']} " + f"worker_max={ingest['per_worker_wall_s_max']:.2f}s " + f"peak_rss_mb={ingest['per_worker_peak_rss_mb_max']:.1f}") + all_rows.append(ingest) + query = bench_query(M, queries) + print(f" query: {query['wall_s']:.2f}s for {query['queries']} qs " + f"p50={query['p50_ms']:.2f}ms p99={query['p99_ms']:.2f}ms " + f"attach={query['attach_ms']:.1f}ms") + all_rows.append(query) + summary.append({ + "M": M, + "ingest_wall_s": ingest["wall_s"], + "ingest_chunks_per_s": ingest["total_chunks"] / ingest["wall_s"], + "query_p50_ms": query["p50_ms"], + "query_p99_ms": query["p99_ms"], + "query_attach_ms": query["attach_ms"], + }) + + # Write CSV + with open(csv_path, "w") as f: + if all_rows: + keys = set() + for r in all_rows: + keys.update(r.keys()) + w = csv.DictWriter(f, fieldnames=sorted(keys)) + w.writeheader() + for r in all_rows: + w.writerow(r) + print() + print(f"== summary ==") + print(f"{'M':>2} {'ingest_wall_s':>14} {'chunks/s':>10} " + f"{'q_p50_ms':>10} {'q_p99_ms':>10} {'attach_ms':>10}") + for s in summary: + print(f"{s['M']:>2} {s['ingest_wall_s']:>14.2f} " + f"{s['ingest_chunks_per_s']:>10.1f} " + f"{s['query_p50_ms']:>10.2f} {s['query_p99_ms']:>10.2f} " + f"{s['query_attach_ms']:>10.2f}") + print() + print(f"results → {csv_path}") + + # Tidy up bench dir. + if BENCH_DIR.exists(): + shutil.rmtree(BENCH_DIR) + + +if __name__ == "__main__": + main() diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 02e9655..e2d2065 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -111,7 +111,7 @@ Newest first. Update on every open/close. | ID | Title | Status | Opened | Directive | |----------|------------------------------------------------|-----------------------|------------|-----------| -| #000065 | Canonical shard count `M` + content-hash routing (decouple ingest parallelism from ATTACH ceiling) | **open · scaffold + design · awaiting go/no-go** (2026-05-26; surfaced while sizing #000061's federation story). Today shard count conflates two roles: producer ingest parallelism (wants vCPU count) + consumer ATTACH fan-out (capped at SQLITE_MAX_ATTACHED=10 on stock python3 sqlite3). Producer with 16 vCPU → 16 shards → consumers fail to attach the 11th. Producer with 4 shards → 16-vCPU box runs 75% idle on ingest. Fix: pin a corpus-wide canonical M (default 8, ≤ ATTACH ceiling), introduce N (ingest workers) decoupled from M. Document → shard assignment becomes content-deterministic: `shard_idx = int(document_root[:8], 16) % M`. Same input → same output across every peer (today's "spray by ingest order" is non-deterministic across peers, a real federation weakness). Migration hard-constraint per fox: **content-addressed rebalance, NOT re-ingest** — every row is already addressed by `document_root` / `leaf_hash` / etc.; migration reads rows from the current 4 shards, computes each row's new shard via the routing function, INSERTs into M new shards. No source re-parse, no re-canonicalization, no re-chunking, no LLM. ~20–40 min I/O-bound vs. hours-to-days for true re-ingest. Audit chain consolidates to canonical shard 000 (re-numbered + re-hashed once) to preserve global event ordering. Phases: 0 design lock + pin M in meta table → 1 read path (connect_query honors M) → 2 ingest path (multi-shard write per worker) → 3 cold-pack restore re-routes on pull → 4 corpus migration tool. Open audit-chain re-numbering question (every shard has its own seq + event_hash; rebalancing splits a producer's chain across M consumer shards). Don't proliferate sub-tickets; the audit handling is part of this design lock. Out of scope: custom-built sqlite3 with higher MAX_ATTACHED (rejected: violates "python3 + venv + sqlite3 only" property from CLAUDE.md); topic-clustering shards (would break ingest determinism). | 2026-05-26 | — | +| #000065 | Canonical shard count `M` + content-hash routing (decouple ingest parallelism from ATTACH ceiling) | **open · scaffold + design · awaiting go/no-go** (2026-05-26; surfaced while sizing #000061's federation story). Today shard count conflates two roles: producer ingest parallelism (wants vCPU count) + consumer ATTACH fan-out (capped at SQLITE_MAX_ATTACHED=10 on stock python3 sqlite3). Producer with 16 vCPU → 16 shards → consumers fail to attach the 11th. Producer with 4 shards → 16-vCPU box runs 75% idle on ingest. Fix: pin a corpus-wide canonical **M = 4** (decided 2026-05-26 from real-Wikipedia bench: M=4 captures 92% of peak ingest throughput, ATTACH cost 9 ms keeps mobile-tolerable, 6 free ATTACH slots under SQLite's 10 ceiling for auxiliary DBs), introduce N (ingest workers) decoupled from M. Document → shard assignment becomes content-deterministic: `shard_idx = int(document_root[:8], 16) % M`. Same input → same output across every peer (today's "spray by ingest order" is non-deterministic across peers, a real federation weakness). Migration hard-constraint per fox: **content-addressed rebalance, NOT re-ingest** — every row is already addressed by `document_root` / `leaf_hash` / etc.; migration reads rows from the current 4 shards, computes each row's new shard via the routing function, INSERTs into M new shards. No source re-parse, no re-canonicalization, no re-chunking, no LLM. ~20–40 min I/O-bound vs. hours-to-days for true re-ingest. Audit chain consolidates to canonical shard 000 (re-numbered + re-hashed once) to preserve global event ordering. Phases: 0 design lock + pin M in meta table → 1 read path (connect_query honors M) → 2 ingest path (multi-shard write per worker) → 3 cold-pack restore re-routes on pull → 4 corpus migration tool. Open audit-chain re-numbering question (every shard has its own seq + event_hash; rebalancing splits a producer's chain across M consumer shards). Don't proliferate sub-tickets; the audit handling is part of this design lock. Out of scope: custom-built sqlite3 with higher MAX_ATTACHED (rejected: violates "python3 + venv + sqlite3 only" property from CLAUDE.md); topic-clustering shards (would break ingest determinism). | 2026-05-26 | — | | #000064 | Cold-object operations toolkit (verify/diff/doctor/repair-fts/gc-plan + audit taxonomy) | **scaffold-only · awaiting go/no-go** (2026-05-26; from Dav1d #000061 review §11/§12/§14). Operator-facing observability + repair tools on top of #000061: `cold verify` (sample/full integrity check), `cold diff` (local vs remote manifest), `cold doctor` (one-shot health: connectivity / credentials / manifest age / missing-object count / tamper sample / audit-chain integrity), `cold repair-fts` (rebuild FTS5 from chunks.content), `cold gc-plan` (orphan bucket objects, read-only by default — destructive only with `--apply` + confirm). Plus expanded audit-event taxonomy: per-PUT/HEAD/GET success/failure events, manifest-pointer events, verify/doctor/gc events. All read-mostly; destructive ops require `--apply`. Bundled so the audit-taxonomy gets one design pass instead of five-way drift. Sequence: doctor → verify → diff → repair-fts → gc-plan. No code until #000061 closes. | 2026-05-26 | — | | #000063 | Cold-object private-ciphertext mode (mesh-keyed object keys) | **scaffold-only · awaiting go/no-go** (2026-05-26; from Dav1d #000061 review §9 / response A §13.3). Adds private mode to #000061 cold-object format so chunk bodies + manifest can be uploaded to public-read bucket without leaking corpus membership. Two strategies: (A) deterministic `object_key = HMAC(group_key, leaf_hash)` + AEAD-encrypted body — supports lookup-by-leaf-hash given the key; (B) random-key ciphertext + encrypted private manifest — stronger membership hiding, needs manifest fetch first. Strategy A default; B opt-in. Group key from existing `arborist/mesh/crypto.py`; pack manifest carries `epoch_id` for rotation. Verifier path unchanged: consumer decrypts, then `hash_leaf(plaintext) == leaf_hash` as in public mode. No code until (1) a real non-public corpus needs cold-object shipping, (2) mesh group-key ABI is stable enough to reference, (3) threat-model split between A vs B is settled by real adversary. | 2026-05-26 | — | | #000062 | Mechanistic Witness: governed diagnostic sidecar (CNA/SAE/Neuronpedia) | **scaffold-only · awaiting go/no-go** (2026-05-26; Dav1d de-novo review §4.7 / §9.1.F). Specification of a mechanistic-interpretability sidecar that produces a content-addressed `MechanisticWitnessRoot` over (model, prompts, capture policy, neurons/features, intervention deltas), used as a **diagnostic input** to SelfModel (#000014/#000017) + benchmark-fixture generation. **Hard constraint:** soft signals never enter the hard proof path — `audit_mode` does NOT move based on witness output, `providence_cache` is untouched, `governance_policy_hash` only moves via explicit ForkScore ACCEPT with M+C+X axes passing (#000060 §7). Four guardrails (diagnostic-only by default · sandbox intervention only · no production steering without governance · feature labels never semantic proof). Witness root TLV-encodes `model_config_root | activation_capture_policy_root | contrastive_prompt_set_root | feature_or_neuron_set_root | intervention_result_root | behavioral_delta_root | safety_policy_root`. Scaffold only — no code until a real falsifier-in-hand use case exists + the four guardrails are restated in CLAUDE.md as rules + #000060 H-ABCDEFG-M+C+X harness exists to gate promotion. Captured to keep mechanistic-interp tooling out of the substrate unless and until it earns its place; the dual-use risk (Pan et al. 2025 CNA: 0.1% MLP ablation breaks refusal in 72B models) makes the governance-first framing load-bearing. | 2026-05-26 | — | diff --git a/docs/tickets/ticket-000065-canonical-shard-count-content-hash-routing.md b/docs/tickets/ticket-000065-canonical-shard-count-content-hash-routing.md index e852df6..765bc05 100644 --- a/docs/tickets/ticket-000065-canonical-shard-count-content-hash-routing.md +++ b/docs/tickets/ticket-000065-canonical-shard-count-content-hash-routing.md @@ -52,13 +52,75 @@ different shards. This makes federation gossip + content-addressing weaker than it should be — two peers' "shard 003.db" can have different contents. -## Design +## Canonical M = 4 (decided 2026-05-26 from real-corpus bench) + +The corpus-wide canonical shard count is **M = 4**. Reasoning grounded +in two `bench/shard_count_sweep.py` runs (synthetic + real Wikipedia +2003 cur dump, 2000 docs each, M ∈ {1, 2, 4, 8}). + +### Real Wikipedia numbers (2026-05-26) + +``` + M chunks/s q_p50_ms q_p99_ms attach_ms free ATTACH slots + 1 3,648 0.05 0.20 1.61 9 + 2 5,649 0.03 0.19 3.24 8 + 4 6,405 0.07 0.30 9.00 6 + 8 6,959 0.03 0.28 9.65 2 +``` + +Full CSV: `bench/results/shard-count-sweep-2026-05-26T16-31-34Z.csv`. +Synthetic-corpus comparison: `…T16-20-48Z.csv` (under-estimates per- +doc work because random-word docs skip wikitext parsing + edge +extraction — Wikipedia is the load-bearing measurement). + +### Why M = 4 + +1. **Captures 92 % of peak ingest throughput** (6,405 / 6,959). + Going from M=4 to M=8 buys only +9 % — diminishing-returns + regime. The ingest bottleneck on real wikitext is Python CPU + per worker (canonicalization + edge extraction), not SQLite + writes; more shards don't unlock that. +2. **Keeps 6 ATTACH slots free** under SQLite's 10-shard ceiling for + `snapshots.db`, `qa.db`, `selfmodel-chain.db`, `crawl_*.db`, + future `mesh_*.db` if extracted, and the manifest-pointer + connection. M=8 leaves only 2 slots — operationally tight. +3. **Mobile-tolerable.** Phone NAND attach is ~5–10× slower than + NVMe. M=4 attach (9 ms on dev box) → ~45–90 ms cold-start on + phone — feels instant. M=8 attach (9.65 ms) → ~50–100 ms; also + fine but with no headroom for the slower mobile flash. +4. **Matches fox's current 4-shard layout** = the cheapest migration. + Most rows already hash-route to their existing shard with + probability 1/M = 25 % (vs. uniform reshuffling at any other M). + The teleport is partial, not total. +5. **Query latency is flat across M within noise** at 2000-doc + corpus. Larger query suites (n=1000+) on a populated corpus + would tighten this; current data says shard count doesn't move + p50 perceptibly when the FTS5 index fits in page cache. + +### Why not M = 2 + +- Loses 13 % of peak ingest throughput vs M=4 +- ATTACH headroom (8 slots) overkill for current arborist's + auxiliary DBs +- Wider per-shard variance: 2 huge shards put more work behind + one WAL writer lock during ingest spikes; 4 smaller shards + spread the spike + +### Why not M = 8 + +- +9 % ingest gain doesn't justify halving ATTACH headroom +- Mobile attach cliff (10 shards + 8 corpus + qa + snapshots + + mesh = right at the ceiling) +- Federation peers with fewer vCPU than producer carry more + per-shard work that doesn't pay back + +### Design Introduce two numbers, both explicit: ``` N = ingest worker count — producer's vCPU choice -M = canonical shard count — corpus-wide constant, M ≤ 8 by convention +M = canonical shard count — corpus-wide constant, M = 4 by default ``` **M is part of the corpus identity.** Pinned in the snapshot manifest + @@ -244,6 +306,116 @@ in S3 as recovery rollback." shard is computable but the edges row still lives in src's shard. No change needed. +## When the SQLite-default substrate stops being right + +The M=4 choice is correct **conditional on staying on stock python3 +sqlite3 with SQLITE_MAX_ATTACHED=10**. Three thresholds where that +assumption breaks; each names what bench would justify moving: + +### Threshold A: ATTACH-ceiling pressure forces a forked SQLite + +**Signal:** auxiliary `.db` files (`qa.db`, `snapshots.db`, +`selfmodel-chain.db`, `crawl_*.db`, future `mesh_*.db`) plus M=4 +canonical shards plus operator-facing connections push past 10 in +practice. Today: 4 canonical + 3-4 auxiliary = 7-8, fits with M=4 +headroom. If auxiliary grows past 5, M=4 becomes the binding +constraint. + +**What to bench before the fork:** + +1. Compile sqlite3 with `SQLITE_MAX_ATTACHED=125` (the spec maximum). +2. Re-run `bench/shard_count_sweep.py` with M ∈ {16, 32, 64} on the + real corpus. +3. Measure: ATTACH cost growth (linear vs. super-linear?), query + latency under wider UNION ALL, ingest throughput at higher M. +4. **Decision criteria:** + - If ATTACH cost stays roughly linear past M=10 → forked SQLite + viable. Costs: a custom build to ship with arborist or a + dependency on system-sqlite-with-this-flag (Debian / Fedora + packages, mobile builds). + - If ATTACH cost goes super-linear past M=10 → SQLite's + architectural design doesn't expect wide multi-attach; need a + different substrate. + +**Cost of the fork:** ~1 day of build infrastructure. ~Permanent tax +on every install: arborist no longer "just works" on stock sqlite3. +Violates the CLAUDE.md "python3 + venv + sqlite3 is enough" property. +Don't do it unless the bench says we have to. + +### Threshold B: ingest throughput hits a wall + +**Signal:** workload requires real-time ingest at > 10k chunks/s +sustained (e.g., live mesh-replicated state across many peers). On +this benchmark, real-Wikipedia ingest peaks at ~7,000 chunks/s +regardless of M past 4. SQLite's single-writer-per-file model is the +ceiling — adding more workers behind the same writer lock doesn't +help. + +**What to bench before considering alternatives:** + +1. **Page size tuning.** Default 4 KB; arborist could rebuild shards + with 16 KB or 64 KB pages. Bench: 4 KB vs 16 KB vs 64 KB on the + same M=4 layout. Larger pages help wide-row tables (`edges`) + especially. +2. **WAL checkpoint interval.** Default auto-checkpoint at 1000 + frames. Bench: ingest throughput at 100, 1000, 10000-frame + intervals. +3. **`mmap_size` sweep.** Default 256 MB in arborist's `connect`. + Bench: 0, 64 MB, 256 MB, 1 GB. +4. **`synchronous=NORMAL` vs `OFF` for ingest workers.** NORMAL is + the default + safe under WAL. OFF removes the per-commit fsync + entirely and is faster but loses durability on power loss. For + ingest from a deterministic dump (re-runnable), `OFF` might be + acceptable. + +**Decision criteria:** +- If tuning gets us 2-5× more throughput → stay on SQLite, document + the tuned pragmas in `connect()`. +- If still ceiling-limited at 10k+ chunks/s → genuine substrate + question. Candidates: + - **DuckDB** (columnar, MVCC, multiple concurrent writers, + supports SQLite-compatible SQL surface, has its own FTS). + Bench against same harness. + - **libmdbx / LMDB** (B+tree, MVCC, very fast reads). No FTS; + we'd build it. Probably overkill for arborist's read patterns. + - **In-house DB.** Too ambitious without a specific failure of + the above to justify. + +### Threshold C: federation needs multi-writer-same-shard + +**Signal:** the mesh state evolves so that multiple peers can write +to the same shard simultaneously (today: single-writer-per-shard +is the assumption — mesh gossip pulls audit events but doesn't +co-write). SQLite's writer lock serializes across peers — would +become a federation bottleneck. + +**What to bench before alternatives:** + +1. Two peers writing to a network filesystem (NFS) backed shard with + WAL — does SQLite's locking actually work, and what's throughput? + (Spoiler: SQLite famously hates NFS; this fails.) +2. Two peers writing to a shared block device (SAN) with WAL — + similar story but better. +3. Application-level coordination: only one peer is "leader" for a + shard at a time; followers replicate the WAL frames. + +**Decision criteria:** +- If single-leader-per-shard works → stay on SQLite, build the + leader-election layer above it. +- If we need true multi-writer-same-shard → SQLite is wrong; needs + a substrate with built-in MVCC + conflict resolution. **DuckDB + doesn't solve this either** — its MVCC is single-process. The + candidate becomes something like FoundationDB or a CRDT layer on + any key-value store. + +### Honest summary + +For the current arborist workload (single-writer-per-shard, M=4, +read-mostly federation), **stock python3 sqlite3 is the right +substrate.** None of the three thresholds are close to firing. The +bench discipline above exists so we know what to measure when +something changes, not as a roadmap to leave SQLite. + ## Scope boundaries In scope: