diff --git a/arborist/cli.py b/arborist/cli.py index 258fa8b..a055bf8 100644 --- a/arborist/cli.py +++ b/arborist/cli.py @@ -211,6 +211,7 @@ def _cmd_ingest(args: argparse.Namespace) -> int: ) conn = connect(target_db) + embedded = None try: result = ingest_source( conn, @@ -224,9 +225,20 @@ def _cmd_ingest(args: argparse.Namespace) -> int: loss_report_excerpts=not args.no_loss_excerpts, loss_report_max_excerpt_bytes=args.loss_excerpt_bytes, ) + if getattr(args, "embed", False): + # Eager opt-in: after the chunk+Merkle-commit pass, embed the + # chunks this run added (incremental — only chunk_ids not + # already in chunk_vecs). Default ingest does NOT embed; the + # lazy out-of-band pass (`arborist embed`, or a cron, or the + # Prometheus-Σ unconscious sweep) is the usual path. See #000039. + from arborist.search.vec import embed_documents + embedded = embed_documents(conn, incremental=True) finally: conn.close() - print(json.dumps(result.__dict__, indent=2, ensure_ascii=False)) + out = dict(result.__dict__) + if embedded is not None: + out["chunks_embedded"] = embedded + print(json.dumps(out, indent=2, ensure_ascii=False)) return 0 @@ -315,6 +327,8 @@ def _cmd_embed(args: argparse.Namespace) -> int: conn, limit=args.limit, batch_size=args.batch_size, + incremental=not getattr(args, "rebuild", False), + rebuild=getattr(args, "rebuild", False), progress=_progress, ) finally: @@ -323,6 +337,7 @@ def _cmd_embed(args: argparse.Namespace) -> int: print(json.dumps({ "db": str(args.db), "backend_version": VEC_BACKEND_VERSION, + "mode": "rebuild" if getattr(args, "rebuild", False) else "incremental", "chunks_embedded": n, "elapsed_s": round(elapsed, 2), "rate_per_s": round(n / max(elapsed, 1e-6), 1), @@ -4514,6 +4529,14 @@ def build_parser() -> argparse.ArgumentParser: ) sub = p.add_subparsers(dest="cmd", required=True) + # Optional sqlite-vec backend (#000039): only surface --embed / + # `arborist embed` / `search --backend vec` when the [vec] extra + # is installed. + try: + from arborist.search import VEC_AVAILABLE as _vec_ok + except Exception: # pragma: no cover + _vec_ok = False + ingest = sub.add_parser("ingest", help="ingest documents from a source") ingest.add_argument( "--source", @@ -4676,19 +4699,23 @@ def build_parser() -> argparse.ArgumentParser: default=200, help="max excerpt byte length (default 200)", ) + if _vec_ok: + ingest.add_argument( + "--embed", action="store_true", + help=( + "after ingest, embed the chunks this run added (incremental " + "chunk_vecs population — #000039; default ingest does NOT " + "embed, the lazy `arborist embed` pass / cron / Prometheus-Σ " + "sweep is the usual path)" + ), + ) ingest.set_defaults(func=_cmd_ingest) search = sub.add_parser("search", help="keyword/semantic search (UNGROUNDED audit mode)") search.add_argument("query", help="query string") search.add_argument("--limit", type=int, default=20) search.add_argument("--json", action="store_true", help="output JSON") - _search_backends = ["fts5"] - try: - from arborist.search import VEC_AVAILABLE as _vec_ok - if _vec_ok: - _search_backends.append("vec") - except Exception: # pragma: no cover - pass + _search_backends = ["fts5"] + (["vec"] if _vec_ok else []) search.add_argument( "--backend", choices=_search_backends, default="fts5", help=( @@ -4710,6 +4737,14 @@ def build_parser() -> argparse.ArgumentParser: help="cap the number of chunks embedded (smoke-test on a real shard)", ) embed_cmd.add_argument("--batch-size", type=int, default=256) + embed_cmd.add_argument( + "--rebuild", action="store_true", + help=( + "DROP + recreate chunk_vecs, then full re-embed — the clean " + "VEC_BACKEND_VERSION-bump path (default is incremental: embed " + "only chunks not already in chunk_vecs)" + ), + ) embed_cmd.set_defaults(func=_cmd_embed) verify = sub.add_parser( diff --git a/arborist/search/vec.py b/arborist/search/vec.py index a1d3ff5..aebe346 100644 --- a/arborist/search/vec.py +++ b/arborist/search/vec.py @@ -173,23 +173,50 @@ def embed_documents( embedder: Embedder | None = None, limit: int | None = None, batch_size: int = 256, + incremental: bool = True, + rebuild: bool = False, progress: Callable[[int, int], None] | None = None, ) -> int: - """Populate ``chunk_vecs`` for every ``chunks`` row with content. + """Populate ``chunk_vecs`` for ``chunks`` rows with content. - Cold-evicted chunks (``content IS NULL``) are skipped — there is - nothing to embed and rehydration would re-trigger this. ``limit`` - caps the number of chunks processed (useful for a quick smoke - test on a real shard). Returns the count embedded. + Idempotency model (see #000039 — chunk content is immutable per + ``chunk_id``: a chunk, once embedded, never needs re-embedding + unless the *embedder* changes): - Idempotent at the row level: ``INSERT OR REPLACE`` keyed on - ``chunk_id``, so re-running after a partial pass resumes cleanly. + - ``incremental=True`` (default): embed only chunks **not already + in chunk_vecs** (``chunk_id NOT IN (SELECT chunk_id FROM chunk_vecs)``). + This is the after-ingest / cron / Prometheus-Σ-sweep path — it + picks up exactly the newly-ingested chunks; re-running is cheap + (no-op once everything's embedded). + - ``incremental=False``: re-embed every chunk with content + (delete-then-insert all). The embedder-changed case. + - ``rebuild=True``: DROP + recreate ``chunk_vecs`` first, then a + full pass — the clean ``VEC_BACKEND_VERSION`` bump (so a search + mid-rebuild never mixes old- and new-model embeddings: it's all + new-model from the recreated table, growing as the pass runs). + Implies a full (non-incremental) pass. + + Cold-evicted chunks (``content IS NULL``) are skipped — the vec + row, if it exists, stays valid (content is the same on rehydrate). + ``limit`` caps chunks processed (smoke-test knob). Returns the + count newly embedded this call. """ embedder = embedder or default_embedder() + if rebuild: + try: + load_vec_extension(conn) + conn.execute("DROP TABLE IF EXISTS chunk_vecs") + except sqlite3.OperationalError: + pass + incremental = False # recreated table is empty → full pass anyway ensure_chunk_vecs_table(conn) + where = "content IS NOT NULL" + if incremental: + where += " AND chunk_id NOT IN (SELECT chunk_id FROM chunk_vecs)" + total_known = conn.execute( - "SELECT COUNT(*) FROM chunks WHERE content IS NOT NULL" + f"SELECT COUNT(*) FROM chunks WHERE {where}" ).fetchone()[0] if limit is not None: total_known = min(total_known, limit) @@ -207,7 +234,8 @@ def embed_documents( try: # vec0 virtual tables don't honor INSERT OR REPLACE — re-inserting # an existing chunk_id is a hard UNIQUE error. Delete-then-insert - # gives idempotent re-runs and a content-changed → re-embed path. + # is the upsert; harmless no-op on rows that aren't present yet + # (the incremental path) and the re-embed path otherwise. conn.executemany( "DELETE FROM chunk_vecs WHERE chunk_id = ?", [(cid,) for cid in batch_ids], @@ -227,8 +255,7 @@ def embed_documents( batch_texts.clear() rows = conn.execute( - "SELECT chunk_id, content FROM chunks WHERE content IS NOT NULL " - "ORDER BY chunk_id" + f"SELECT chunk_id, content FROM chunks WHERE {where} ORDER BY chunk_id" + (f" LIMIT {int(limit)}" if limit is not None else "") ) for row in rows: diff --git a/docs/tickets/ticket-000039-sqlite-vec-optional-backend.md b/docs/tickets/ticket-000039-sqlite-vec-optional-backend.md index 9880354..1f47ddf 100644 --- a/docs/tickets/ticket-000039-sqlite-vec-optional-backend.md +++ b/docs/tickets/ticket-000039-sqlite-vec-optional-backend.md @@ -1,6 +1,6 @@ # Ticket #000039 — Optional `sqlite-vec` retrieval backend (A/B vs FTS5, hybrid not replacement) -**Status:** in progress · Phase 0 (doc) + **Phase 1 landed 2026-05-11**. Phase 1: `arborist/search/vec.py` — `VecBackend(SearchBackend)` (UNGROUNDED hits, never in proof path), `chunk_vecs` vec0 virtual table + `vec_meta` (sibling tables — don't touch chunks/documents/audit chain), `embed_documents()` ingest (delete-then-insert idempotent; vec0 doesn't honor INSERT-OR-REPLACE), pluggable `Embedder` callable with a fastembed `bge-small-en-v1.5` default. CLI: `arborist embed [--limit] [--batch-size]` + `arborist search --backend vec`. `[vec]` optional extra (sqlite-vec + fastembed). **Obvious v1 tuning** (`VEC_BACKEND_VERSION = vec-v1-bge-small-en-v1.5-384float32-cosine-flat`): embedder `BAAI/bge-small-en-v1.5`, dim 384, quant float32 (int8/binary = the production storage knob per §3.1, not wired in v1), metric cosine (bge outputs L2-normalized so cosine ≡ L2 ranking), ANN flat (vec0 default), top_k 20. 7 tests (`tests/test_search_vec.py`, stub embedder — plumbing only; semantic quality demonstrated on a real shard). **Demonstrated on `crawl_appliedcombinatorics_org.db`** (168 chunks embedded in ~37s incl. model load; semantic queries return topically-correct hits — "how many ways to choose k things from n" → top hit "AC Combinations"; chain-check on that shard reports 0 after embedding). The 5 hyperparams fold into `governance_policy_hash` in a later phase (§6 — not wired yet). **Phase 2** (RRF hybrid fusion in `query.py`) gated on a ≥5pp recall-lift measurement on bench fixtures with no STRICT-rate regression (§8). +**Status:** in progress · Phase 0 (doc) + **Phase 1 landed 2026-05-11**. Phase 1: `arborist/search/vec.py` — `VecBackend(SearchBackend)` (UNGROUNDED hits, never in proof path), `chunk_vecs` vec0 virtual table + `vec_meta` (sibling tables — don't touch chunks/documents/audit chain), `embed_documents()` ingest (delete-then-insert idempotent; vec0 doesn't honor INSERT-OR-REPLACE), pluggable `Embedder` callable with a fastembed `bge-small-en-v1.5` default. CLI: `arborist embed [--limit] [--batch-size]` + `arborist search --backend vec`. `[vec]` optional extra (sqlite-vec + fastembed). **Obvious v1 tuning** (`VEC_BACKEND_VERSION = vec-v1-bge-small-en-v1.5-384float32-cosine-flat`): embedder `BAAI/bge-small-en-v1.5`, dim 384, quant float32 (int8/binary = the production storage knob per §3.1, not wired in v1), metric cosine (bge outputs L2-normalized so cosine ≡ L2 ranking), ANN flat (vec0 default), top_k 20. 7 tests (`tests/test_search_vec.py`, stub embedder — plumbing only; semantic quality demonstrated on a real shard). **Demonstrated on `crawl_appliedcombinatorics_org.db`** (168 chunks embedded in ~37s incl. model load; semantic queries return topically-correct hits — "how many ways to choose k things from n" → top hit "AC Combinations"; chain-check on that shard reports 0 after embedding). The 5 hyperparams fold into `governance_policy_hash` in a later phase (§6 — not wired yet). **Ingest integration landed 2026-05-11** (see §14): `embed_documents()` is now **incremental by default** (embeds only `chunk_id`s not already in `chunk_vecs` — re-runs are cheap no-ops); `arborist embed --rebuild` does the DROP+recreate+full-re-embed for a `VEC_BACKEND_VERSION` bump; `arborist ingest --embed` is the **eager opt-in** (embed this run's new chunks after the chunk+Merkle-commit pass; default ingest does NOT embed — the lazy `arborist embed` pass / cron / Prometheus-Σ unconscious sweep is the usual path). 9 vec tests. **Phase 2** (RRF hybrid fusion in `query.py`) gated on a ≥5pp recall-lift measurement on bench fixtures with no STRICT-rate regression (§8). **Opened:** 2026-05-09 **Scope:** Spec an optional `sqlite-vec` backend that runs **alongside** the existing FTS5 retrieval pipeline (never replacing it), with phased gates @@ -815,3 +815,76 @@ If fox rejects the dep footprint → fall back to the lighter `onnxruntime` path with a `bge-small` ONNX model (still ~33 MB weights but smaller runtime). Adds complexity but keeps the no-server constraint. + +--- + +## 14. Ingest integration & idempotency (landed 2026-05-11) + +The load-bearing fact: **a `chunk_id`'s content is immutable in +arborist.** A chunk is content-addressed-ish (`leaf_hash`) — same +content → same `chunk_id`; different content → a *new* `chunk_id` +(re-ingest with changed content makes a new `document_root` + new +chunk_ids, linked by `supersedes`; a chunker-version bump re-chunks +→ new chunk_ids). So **a chunk, once embedded, never needs +re-embedding** — the only re-embed trigger is the *embedder* +changing (`VEC_BACKEND_VERSION` bump). + +### 14.1 Idempotency table + +| Event | `chunk_vecs` effect | +|---|---| +| Re-ingest same content | doc_root unchanged, chunks no-op-insert → already embedded → no-op | +| Re-ingest changed content (new doc_root, new chunk_ids) | old chunks + old vecs stay (lossless history); new chunk_ids embedded on next pass | +| Chunker bump (`tok-512-v1 → v2`) | re-chunk → new chunk_ids → embedded next pass; old chunk_ids' vecs are orphan-ish but harmless (old chunks still exist) | +| Cold eviction (`content = NULL`) | vec row persists — still valid; on rehydrate the same `chunk_id` returns with the same content | +| Embedder bump (`VEC_BACKEND_VERSION` changes) | the *only* full-re-embed case — `arborist embed --rebuild` (DROP+recreate+full pass); the `vec_meta.backend_version` row records the populating version | +| Superseded docs | old chunks stay embedded → searchable via vec — consistent with `chunks_fts` keeping superseded docs; the retrieval `supersedes`-filter applies equally | + +### 14.2 Two integration models + +- **Lazy / out-of-band (default).** `arborist ingest` does *not* + embed. `arborist embed` (incremental — embeds only `chunk_id`s + not in `chunk_vecs`) populates on demand; the natural ops home is + a cron after the nightly ingest, or — better — a **Prometheus-Σ + unconscious-sweep task** (#000037 §3.1 already walks "ingested- + but-not-yet-probed" documents; embed-the-new-chunks is exactly + that shape, so vec needn't carry its own cron). Ingest stays fast; + embedding is a separable, resumable, optional layer; `chunk_vecs` + trails ingest by ≤ one pass. +- **Eager opt-in.** `arborist ingest --embed` — after the + chunk+Merkle-commit pass, embed this run's new chunks (incremental, + so it's just the delta). `chunk_vecs` stays in sync. Pays + ~10-20 ms/chunk CPU per ingest — fine for incremental ingests, + not for a bulk wiki backfill. + +(#000039 §5.1's "embed at ingest, never at query" is about *when not* +to embed the query corpus — it doesn't mandate embedding inside the +ingest call.) + +### 14.3 Commands + +| Command | Behavior | +|---|---| +| `arborist --db D embed` | incremental — embed `chunk_id`s not yet in `chunk_vecs` (cheap no-op once everything's embedded) | +| `arborist --db D embed --rebuild` | DROP + recreate `chunk_vecs`, then full re-embed — the clean `VEC_BACKEND_VERSION`-bump path (a search mid-rebuild never mixes old/new-model embeddings: the recreated table starts empty and grows new-model) | +| `arborist --db D embed --limit N` | cap chunks processed — smoke-test on a real shard | +| `arborist ... ingest --source ... --embed` | eager: after ingest, incremental-embed this run's new chunks | +| `arborist --db D search Q --backend vec` | semantic ANN; errors with an install/embed hint if `[vec]` missing or `chunk_vecs` empty | + +### 14.4 Concurrency + +WAL serializes writers → `arborist embed` and `arborist ingest` on +the *same* shard take turns. In the `--shards-dir` layout each shard +is its own file → embed-shard-0 and ingest-shard-1 don't contend. +Embed-during-ingest is safe under the lazy model: embed sees a +consistent per-transaction snapshot; ingest's new chunks get picked +up on the next incremental embed pass. + +### 14.5 Versioning + +`vec_meta.backend_version` (written by `ensure_chunk_vecs_table`) is +the per-shard discriminator. The 5 vec hyperparams fold into +`governance_policy_hash` in a later phase (§6 — not wired yet); the +8-dim `cache_key` invariant is untouched (vec config goes into the +existing `governance_policy_hash` dim, not a new one). Same +versioned-default discipline as `tok-512-v1` / `norm-v1` etc. diff --git a/tests/test_search_vec.py b/tests/test_search_vec.py index 6e98b13..0ae1267 100644 --- a/tests/test_search_vec.py +++ b/tests/test_search_vec.py @@ -154,6 +154,52 @@ def test_embed_limit_caps_count(db): def test_embed_is_idempotent(db): conn, _ = db embed_documents(conn, embedder=_stub_embedder) - embed_documents(conn, embedder=_stub_embedder) # re-run; INSERT OR REPLACE + embed_documents(conn, embedder=_stub_embedder) # re-run cnt = conn.execute("SELECT COUNT(*) FROM chunk_vecs").fetchone()[0] assert cnt == 3 + + +def test_embed_incremental_only_embeds_new_chunks(db): + """Default (incremental=True): a second embed pass after ingesting a + new doc embeds only the new chunk — re-running is cheap (#000039).""" + conn, _ = db + first = embed_documents(conn, embedder=_stub_embedder) + assert first == 3 + # Simulate a follow-up ingest: one new document + chunk. + new_root = "root_d" * 8 + conn.execute( + "INSERT INTO documents(" + " document_root, document_uri, source_type, kind, title," + " chunking_version, canonicalization_version, schema_version, ingest_ts" + ") VALUES (?, 'uri://d', 'test', 'surface', 'Delta doc'," + " 'tok-512-v1', 'norm-v1', 'v9.8.0', 0)", + (new_root,), + ) + conn.execute( + "INSERT INTO chunks(document_root, idx, leaf_hash, content, tier) " + "VALUES (?, 0, ?, 'delta content about derangements', 'hot')", + (new_root, hashlib.sha256(b"delta").hexdigest()), + ) + second = embed_documents(conn, embedder=_stub_embedder) # incremental + assert second == 1, "only the new chunk should embed on the second pass" + assert conn.execute("SELECT COUNT(*) FROM chunk_vecs").fetchone()[0] == 4 + # A third pass with nothing new is a no-op. + third = embed_documents(conn, embedder=_stub_embedder) + assert third == 0 + + +def test_embed_rebuild_re_embeds_all(db): + """rebuild=True: DROP + recreate + full pass — every chunk re-embedded + (the clean VEC_BACKEND_VERSION-bump path).""" + conn, _ = db + embed_documents(conn, embedder=_stub_embedder) + assert conn.execute("SELECT COUNT(*) FROM chunk_vecs").fetchone()[0] == 3 + n = embed_documents(conn, embedder=_stub_embedder, rebuild=True) + assert n == 3, "rebuild re-embeds all chunks" + assert conn.execute("SELECT COUNT(*) FROM chunk_vecs").fetchone()[0] == 3 + # vec_meta still records the backend version after the recreate. + from arborist.search.vec import VEC_BACKEND_VERSION + row = conn.execute( + "SELECT value FROM vec_meta WHERE key='backend_version'" + ).fetchone() + assert row[0] == VEC_BACKEND_VERSION