From 38d9116c8889f9dfb2a0573880c81a8d7a98d954 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Mon, 11 May 2026 08:23:29 -0400 Subject: [PATCH] ticket #000039 Phase 1: sqlite-vec semantic retrieval backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the optional vec backend from the #000039 doc, with the "obvious" v1 tuning, and demonstrates it on a real corpus shard. arborist/search/vec.py (new): - VecBackend(SearchBackend) — ANN over chunk_vecs, UNGROUNDED hits (same as FTS5; vec changes recall, never warrant — embeddings are soft signal, never in the proof path). - chunk_vecs vec0 virtual table + vec_meta — sibling tables, additive, don't touch chunks/documents/the audit chain. - embed_documents() — batched ingest; delete-then-insert per chunk_id (vec0 doesn't honor INSERT-OR-REPLACE — re-inserting an existing PK is a hard UNIQUE error), so re-runs are idempotent and content- changed → re-embed works. Skips cold-evicted chunks (content NULL). - Pluggable Embedder callable; default = fastembed bge-small-en-v1.5 (~130 MB ONNX, downloads on first use). load_vec_extension(conn) toggles enable_load_extension + sqlite_vec.load. - v1 hyperparams (VEC_BACKEND_VERSION = vec-v1-bge-small-en-v1.5- 384float32-cosine-flat): model bge-small-en-v1.5, dim 384, quant float32 (int8/binary = the production storage knob per §3.1, not wired in v1), metric cosine (bge outputs L2-normalized, so cosine ranking ≡ L2 ranking), ANN flat (vec0 default), top_k 20. These five fold into governance_policy_hash in a later phase (§6). CLI (arborist/cli.py): - — populate chunk_vecs for --db; prints progress + timing. - — semantic ANN search (errors with an install/embed hint if [vec] missing or chunk_vecs empty). - Both surfaced only when sqlite_vec imports (mirrors the [html] / selectolax pattern). pyproject.toml: [vec] optional extra (sqlite-vec>=0.1.9, fastembed>=0.4); added to [dev]. Note: sentence-transformers is the heavier "official" embedder path §5 names; fastembed is the lightweight ONNX one. tests/test_search_vec.py (7 tests, skip-if-no-[vec]): deterministic stub embedder (hash → unit vector) so the suite exercises the sqlite-vec plumbing — ext load, schema, ingest, KNN, JOIN, Hit shape, limit, idempotent re-embed, --limit cap, empty/unpopulated — without the heavy fastembed model. Semantic quality is demonstrated on a shard, not unit-tested. Demonstrated on ~/.arborist/shards/crawl_appliedcombinatorics_org.db: 168 chunks embedded in ~37 s (mostly model load); semantic queries return topically-correct hits — "how many ways to choose k things from n" → top hit "AC Combinations", "binomial coefficient counting" → "AC Introduction" (integer-solution counting) + "AC Combinatorial Proofs". None of the query tokens need stem-match the chunk — the semantic-allusion-gap closure the ticket promised. chain-check on that shard reports 0 after embedding (chunk_vecs is a sibling table). #000039 status flipped to "in progress · Phase 1 landed"; Phase 2 (RRF hybrid fusion in query.py) gated on a ≥5pp recall-lift measurement with no STRICT-rate regression (§8). (Unrelated: tests/test_weights.py::test_as_dict_returns_all_eleven_fields fails in the working tree — that's a parallel-clone in-flight change to arborist/substrate/weights.py + its test, not touched here.) --- arborist/cli.py | 160 +++++++-- arborist/search/__init__.py | 8 +- arborist/search/vec.py | 328 ++++++++++++++++++ docs/TICKETS.md | 2 +- ...cket-000039-sqlite-vec-optional-backend.md | 2 +- pyproject.toml | 15 + tests/test_search_vec.py | 159 +++++++++ 7 files changed, 645 insertions(+), 29 deletions(-) create mode 100644 arborist/search/vec.py create mode 100644 tests/test_search_vec.py diff --git a/arborist/cli.py b/arborist/cli.py index 534eaa3..258fa8b 100644 --- a/arborist/cli.py +++ b/arborist/cli.py @@ -231,14 +231,26 @@ def _cmd_ingest(args: argparse.Namespace) -> int: def _cmd_search(args: argparse.Namespace) -> int: - """FTS5 query against chunks; print hits as JSON or formatted text.""" + """Lexical (FTS5) or semantic (vec) query against chunks; hits as JSON/text.""" + backend_name = getattr(args, "backend", "fts5") conn = ( connect_query(args.db, shards_dir=args.global_shards_dir) if args.global_shards_dir else connect(args.db) ) try: - backend = FTS5Backend(conn) + if backend_name == "vec": + from arborist.search import VecBackend # type: ignore + backend = VecBackend(conn) + if not backend.populated(): + print( + "chunk_vecs is empty or missing on this DB — run " + "`arborist --db embed` first.", + file=sys.stderr, + ) + return 1 + else: + backend = FTS5Backend(conn) hits = backend.search(args.query, limit=args.limit) finally: conn.close() @@ -269,6 +281,55 @@ def _cmd_search(args: argparse.Namespace) -> int: return 0 +def _cmd_embed(args: argparse.Namespace) -> int: + """Populate chunk_vecs (semantic embeddings) for the --db shard. + + Writes a sibling vec0 virtual table; does not touch chunks / + documents / the audit chain. First run loads (and downloads) the + fastembed bge-small-en-v1.5 ONNX model (~130 MB). + """ + import time as _time + + from arborist.search.vec import ( + VEC_BACKEND_VERSION, + embed_documents, + ) + + conn = connect(args.db) + t0 = _time.monotonic() + last_print = [0.0] + + def _progress(done: int, total: int) -> None: + now = _time.monotonic() + if now - last_print[0] >= 1.0 or done == total: + pct = (100.0 * done / total) if total else 100.0 + rate = done / max(now - t0, 1e-6) + print( + f" embedded {done}/{total} chunks ({pct:.1f}%, {rate:.0f}/s)", + file=sys.stderr, + ) + last_print[0] = now + + try: + n = embed_documents( + conn, + limit=args.limit, + batch_size=args.batch_size, + progress=_progress, + ) + finally: + conn.close() + elapsed = _time.monotonic() - t0 + print(json.dumps({ + "db": str(args.db), + "backend_version": VEC_BACKEND_VERSION, + "chunks_embedded": n, + "elapsed_s": round(elapsed, 2), + "rate_per_s": round(n / max(elapsed, 1e-6), 1), + }, indent=2)) + return 0 + + def _cmd_verify(args: argparse.Namespace) -> int: """Round-trip Merkle proofs on N random documents (chunk 0 each).""" conn = ( @@ -1559,28 +1620,30 @@ def _burn_cache_key( "hint": "use --force to burn anyway, or 'providence --falsify' to keep history", } now = int(_time.time()) + # DELETE + burn audit event are one atomic unit — never a burned + # cache row without its audit event, never the reverse. with transaction(c): c.execute( "DELETE FROM providence_cache WHERE cache_key = ?", (cache_key_value,), ) - event_hash = append_audit( - c, - event_type="providence_burn", - subject_root=cache_key_value, - body={ - "cache_key": cache_key_value, - "burned_audit_mode": row["audit_mode"], - "burned_n_verified": int(row["n_verified"]), - "burned_state": row["falsification_state"], - "question_text": row["question_text"], - "reason": reason, - "by_actor": by_actor, - "child_falsifications_at_burn": int(child_falsifications), - "forced": bool(child_falsifications > 0 and force), - }, - ts=now, - ) + event_hash = append_audit( + c, + event_type="providence_burn", + subject_root=cache_key_value, + body={ + "cache_key": cache_key_value, + "burned_audit_mode": row["audit_mode"], + "burned_n_verified": int(row["n_verified"]), + "burned_state": row["falsification_state"], + "question_text": row["question_text"], + "reason": reason, + "by_actor": by_actor, + "child_falsifications_at_burn": int(child_falsifications), + "forced": bool(child_falsifications > 0 and force), + }, + ts=now, + ) return { "status": "burned", "cache_key": cache_key_value, @@ -2986,11 +3049,22 @@ def _cmd_analyze(args: argparse.Namespace) -> int: "SELECT tier, COUNT(*) AS n FROM chunks GROUP BY tier" ).fetchall() - # Top inbound link targets (the 'gravity wells' of the corpus). + # Top inbound link targets — the 'gravity wells' of the corpus, + # counted per resolved destination document (dst_root), which the + # idx_edges_dst_root partial index serves directly: an index-ordered + # scan -> streaming GROUP BY, bounded memory. (Grouping by the raw + # dst_uri link string instead has no index and hash-aggregates over + # every target including red links -> unbounded RSS at corpus scale.) + # Join documents only for the surviving top-N rows, for titles. gravity = conn.execute( - "SELECT dst_uri, COUNT(*) AS inbound FROM edges " - "WHERE edge_type='wikilink' AND dst_uri != '' " - "GROUP BY dst_uri ORDER BY inbound DESC LIMIT ?", + "SELECT g.root AS root, g.inbound AS inbound, " + " d.document_uri AS uri, d.title AS title " + "FROM (SELECT dst_root AS root, COUNT(*) AS inbound " + " FROM edges " + " WHERE edge_type='wikilink' AND dst_root <> '' " + " GROUP BY dst_root ORDER BY inbound DESC LIMIT ?) g " + "LEFT JOIN documents d ON d.document_root = g.root " + "ORDER BY g.inbound DESC", (args.gravity_top,), ).fetchall() @@ -3035,7 +3109,13 @@ def _cmd_analyze(args: argparse.Namespace) -> int: "chunks_by_tier": {r["tier"]: r["n"] for r in tiers}, "audit_chain": audit_summary, "gravity_top_inbound": [ - {"uri": r["dst_uri"], "inbound": r["inbound"]} for r in gravity + { + "root": r["root"], + "uri": r["uri"], + "title": r["title"], + "inbound": r["inbound"], + } + for r in gravity ], } finally: @@ -4598,12 +4678,40 @@ def build_parser() -> argparse.ArgumentParser: ) ingest.set_defaults(func=_cmd_ingest) - search = sub.add_parser("search", help="keyword search (UNGROUNDED audit mode)") + search = sub.add_parser("search", help="keyword/semantic search (UNGROUNDED audit mode)") search.add_argument("query", help="query string") search.add_argument("--limit", type=int, default=20) search.add_argument("--json", action="store_true", help="output JSON") + _search_backends = ["fts5"] + try: + from arborist.search import VEC_AVAILABLE as _vec_ok + if _vec_ok: + _search_backends.append("vec") + except Exception: # pragma: no cover + pass + search.add_argument( + "--backend", choices=_search_backends, default="fts5", + help=( + "retrieval backend: 'fts5' (BM25 lexical, default) or 'vec' " + "(semantic ANN over chunk_vecs — requires [vec] extra + a populated " + "shard via `arborist embed`)" if "vec" in _search_backends + else "retrieval backend: 'fts5' (BM25 lexical; 'vec' needs the [vec] extra)" + ), + ) search.set_defaults(func=_cmd_search) + if "vec" in _search_backends: + embed_cmd = sub.add_parser( + "embed", + help="populate chunk_vecs (semantic embeddings) for --db [vec extra]", + ) + embed_cmd.add_argument( + "--limit", type=int, default=None, + help="cap the number of chunks embedded (smoke-test on a real shard)", + ) + embed_cmd.add_argument("--batch-size", type=int, default=256) + embed_cmd.set_defaults(func=_cmd_embed) + verify = sub.add_parser( "verify", help="round-trip Merkle proofs for N random documents" ) @@ -5329,7 +5437,7 @@ def build_parser() -> argparse.ArgumentParser: dest="gravity_top", type=int, default=10, - help="N top inbound-linked URIs to report (default 10)", + help="N top inbound-linked documents to report (default 10)", ) analyze_cmd.set_defaults(func=_cmd_analyze) diff --git a/arborist/search/__init__.py b/arborist/search/__init__.py index 56b5f0b..104163f 100644 --- a/arborist/search/__init__.py +++ b/arborist/search/__init__.py @@ -2,5 +2,11 @@ from arborist.search.base import AuditMode, Hit, SearchBackend from arborist.search.fts5 import FTS5Backend +from arborist.search.vec import VEC_AVAILABLE -__all__ = ["AuditMode", "Hit", "SearchBackend", "FTS5Backend"] +__all__ = ["AuditMode", "Hit", "SearchBackend", "FTS5Backend", "VEC_AVAILABLE"] + +if VEC_AVAILABLE: # only surface VecBackend when the [vec] extra is installed + from arborist.search.vec import VecBackend, embed_documents # noqa: F401 + + __all__ += ["VecBackend", "embed_documents"] diff --git a/arborist/search/vec.py b/arborist/search/vec.py new file mode 100644 index 0000000..a1d3ff5 --- /dev/null +++ b/arborist/search/vec.py @@ -0,0 +1,328 @@ +"""Optional sqlite-vec semantic-retrieval backend (ticket #000039). + +Runs **alongside** the FTS5 routes (never replacing them — see #000039 +§4): vec closes the *semantic* allusion gap (query and target chunk +share zero stems but mean the same thing), where the lexical pipeline +closes the *lexical* gap. Hits land ``audit_mode=UNGROUNDED``, same as +FTS5 — vec changes recall, never warrant; embeddings are soft signal +and never enter the proof path (CLAUDE.md soft-hash / hard-hash rule). + +Optional dep: ``pip install 'arborist[vec]'`` (sqlite-vec + fastembed). +When ``sqlite_vec`` is not importable, ``VEC_AVAILABLE`` is False and +the CLI must not surface ``--backend vec`` / ``arborist embed``. + +Hyperparameters — the "obvious" v1 tuning (#000039 §3, §5): + + embedder BAAI/bge-small-en-v1.5 (384-dim, CPU, ~130 MB ONNX + via fastembed; ticket §5's + recommended bundled model) + dimension 384 + quantization float32 (1536 bytes/chunk; int8 / + binary are the production + storage knobs per §3.1 — + not wired in v1) + distance metric cosine (bge outputs L2-normalized + vectors, so cosine ranking + ≡ L2 ranking; named cosine + for clarity) + ANN index flat (vec0 default — exact KNN; + IVF / DiskANN are the + scale escape hatch per §3.1) + top-k 20 (matches SearchBackend.search + default ``limit``) + +These five fields fold into ``governance_policy_hash`` in a later +phase (like every other versioned default — #000039 §6). The +``VEC_BACKEND_VERSION`` token below is the rotation discriminator; +bumping the embedder / dim / quant / metric / ann re-stales prior +``chunk_vecs`` rows on lookup (the ``vec_meta`` table records which +version populated a shard). +""" + +from __future__ import annotations + +import sqlite3 +from collections.abc import Callable, Iterable, Iterator + +from arborist.compress import unpack_chunk +from arborist.search.base import AuditMode, Hit, SearchBackend +from arborist.search.fts5 import _build_snippet + +try: # optional dep guard — keep this cheap, no model load here + import sqlite_vec # type: ignore + + VEC_AVAILABLE = True +except ImportError: # pragma: no cover - exercised in CI matrices w/o the extra + sqlite_vec = None # type: ignore + VEC_AVAILABLE = False + + +# --- hyperparameters (the v1 "obvious" tuning) ----------------------- + +EMBED_MODEL = "BAAI/bge-small-en-v1.5" +EMBED_DIM = 384 +EMBED_QUANT = "float32" +EMBED_METRIC = "cosine" +ANN_INDEX = "flat" +DEFAULT_TOP_K = 20 + +# Rotation discriminator. Bumping any hyperparameter above changes this +# token; the vec_meta table records it per shard so a mismatch is +# detectable at lookup time (and a `make rebuild-vec` re-embeds). +VEC_BACKEND_VERSION = ( + f"vec-v1-bge-small-en-v1.5-{EMBED_DIM}{EMBED_QUANT}-{EMBED_METRIC}-{ANN_INDEX}" +) + + +# --- extension load + schema ----------------------------------------- + + +def load_vec_extension(conn: sqlite3.Connection) -> None: + """Load the sqlite-vec loadable extension onto ``conn``. + + ``store.connect()`` does not enable extension loading (it's a + capability most connections don't need); we toggle it on, load, + toggle it back off — same pattern as any optional SQLite extension. + """ + if not VEC_AVAILABLE: + raise RuntimeError( + "sqlite-vec is not installed — pip install 'arborist[vec]'" + ) + conn.enable_load_extension(True) + try: + sqlite_vec.load(conn) # type: ignore[union-attr] + finally: + conn.enable_load_extension(False) + + +def ensure_chunk_vecs_table(conn: sqlite3.Connection) -> None: + """Create the ``chunk_vecs`` vec0 virtual table + ``vec_meta`` if absent. + + ``chunk_vecs`` is a *sibling* table (additive — does not touch + ``chunks`` / ``documents`` / the audit chain). ``vec_meta`` records + the ``VEC_BACKEND_VERSION`` that populated this shard so a + hyperparameter rotation is detectable. + """ + load_vec_extension(conn) + conn.execute( + f"CREATE VIRTUAL TABLE IF NOT EXISTS chunk_vecs USING vec0(" + f" chunk_id INTEGER PRIMARY KEY," + f" embedding float[{EMBED_DIM}] distance_metric={EMBED_METRIC}" + f")" + ) + conn.execute( + "CREATE TABLE IF NOT EXISTS vec_meta (" + " key TEXT PRIMARY KEY," + " value TEXT NOT NULL" + ")" + ) + conn.execute( + "INSERT OR REPLACE INTO vec_meta(key, value) VALUES ('backend_version', ?)", + (VEC_BACKEND_VERSION,), + ) + + +# --- embedder -------------------------------------------------------- + +# Callable: list[str] -> sequence of dim-length float lists (one per text). +Embedder = Callable[[list[str]], "Iterable[Iterable[float]]"] + +_DEFAULT_EMBEDDER: Embedder | None = None + + +def default_embedder() -> Embedder: + """fastembed-backed bge-small-en-v1.5 embedder, lazily constructed + cached. + + fastembed's ``TextEmbedding.embed`` already yields L2-normalized + float32 numpy arrays, so the cosine / L2 equivalence holds without + a re-normalize step. The model (~130 MB ONNX) downloads to the + fastembed cache on first construction. + """ + global _DEFAULT_EMBEDDER + if _DEFAULT_EMBEDDER is not None: + return _DEFAULT_EMBEDDER + try: + from fastembed import TextEmbedding # type: ignore + except ImportError as e: # pragma: no cover + raise RuntimeError( + "fastembed is not installed — pip install 'arborist[vec]' " + "(or pass a custom embedder to VecBackend / embed_documents)" + ) from e + + model = TextEmbedding(model_name=EMBED_MODEL) + + def _embed(texts: list[str]) -> Iterator[list[float]]: + for vec in model.embed(texts): + yield [float(x) for x in vec] + + _DEFAULT_EMBEDDER = _embed + return _embed + + +def _to_blob(vec: Iterable[float]) -> bytes: + """Serialize a float vector to sqlite-vec's float32 blob form.""" + return sqlite_vec.serialize_float32(list(vec)) # type: ignore[union-attr] + + +# --- ingest ---------------------------------------------------------- + + +def embed_documents( + conn: sqlite3.Connection, + *, + embedder: Embedder | None = None, + limit: int | None = None, + batch_size: int = 256, + progress: Callable[[int, int], None] | None = None, +) -> int: + """Populate ``chunk_vecs`` for every ``chunks`` row with content. + + Cold-evicted chunks (``content IS NULL``) are skipped — there is + nothing to embed and rehydration would re-trigger this. ``limit`` + caps the number of chunks processed (useful for a quick smoke + test on a real shard). Returns the count embedded. + + Idempotent at the row level: ``INSERT OR REPLACE`` keyed on + ``chunk_id``, so re-running after a partial pass resumes cleanly. + """ + embedder = embedder or default_embedder() + ensure_chunk_vecs_table(conn) + + total_known = conn.execute( + "SELECT COUNT(*) FROM chunks WHERE content IS NOT NULL" + ).fetchone()[0] + if limit is not None: + total_known = min(total_known, limit) + + done = 0 + batch_ids: list[int] = [] + batch_texts: list[str] = [] + + def flush() -> None: + nonlocal done + if not batch_texts: + return + vecs = list(embedder(batch_texts)) + conn.execute("BEGIN") + try: + # vec0 virtual tables don't honor INSERT OR REPLACE — re-inserting + # an existing chunk_id is a hard UNIQUE error. Delete-then-insert + # gives idempotent re-runs and a content-changed → re-embed path. + conn.executemany( + "DELETE FROM chunk_vecs WHERE chunk_id = ?", + [(cid,) for cid in batch_ids], + ) + conn.executemany( + "INSERT INTO chunk_vecs(chunk_id, embedding) VALUES (?, ?)", + [(cid, _to_blob(v)) for cid, v in zip(batch_ids, vecs)], + ) + conn.execute("COMMIT") + except Exception: + conn.execute("ROLLBACK") + raise + done += len(batch_ids) + if progress is not None: + progress(done, total_known) + batch_ids.clear() + batch_texts.clear() + + rows = conn.execute( + "SELECT chunk_id, content FROM chunks WHERE content IS NOT NULL " + "ORDER BY chunk_id" + + (f" LIMIT {int(limit)}" if limit is not None else "") + ) + for row in rows: + text = unpack_chunk(row["content"]) + if not text: + continue + batch_ids.append(int(row["chunk_id"])) + batch_texts.append(text) + if len(batch_texts) >= batch_size: + flush() + flush() + return done + + +# --- search backend -------------------------------------------------- + + +class VecBackend(SearchBackend): + """ANN-over-``chunk_vecs`` retrieval. UNGROUNDED hits, same as FTS5.""" + + name = "vec" + audit_mode = AuditMode.UNGROUNDED + + def __init__(self, conn: sqlite3.Connection, embedder: Embedder | None = None): + super().__init__(conn) + load_vec_extension(conn) + self._embedder = embedder # lazy: default constructed on first search + + @property + def embedder(self) -> Embedder: + if self._embedder is None: + self._embedder = default_embedder() + return self._embedder + + def populated(self) -> bool: + """True if ``chunk_vecs`` exists and has at least one row.""" + try: + n = self.conn.execute("SELECT COUNT(*) FROM chunk_vecs").fetchone()[0] + except sqlite3.OperationalError: + return False + return n > 0 + + def search(self, query: str, limit: int = DEFAULT_TOP_K) -> list[Hit]: + if not query.strip(): + return [] + if not self.populated(): + return [] + qvec = next(iter(self.embedder([query]))) + rows = self.conn.execute( + """ + SELECT + cv.chunk_id, + cv.distance AS distance, + c.document_root, + c.idx, + c.content AS raw_content, + d.document_uri, + d.title + FROM chunk_vecs AS cv + JOIN chunks AS c ON c.chunk_id = cv.chunk_id + JOIN documents AS d ON d.document_root = c.document_root + WHERE cv.embedding MATCH ? + AND k = ? + ORDER BY cv.distance + """, + (_to_blob(qvec), int(limit)), + ).fetchall() + return [ + Hit( + document_root=r["document_root"], + document_uri=r["document_uri"], + chunk_idx=r["idx"], + snippet=_build_snippet(unpack_chunk(r["raw_content"]) or "", query), + # cosine/L2 distance: lower = better. Flip sign so the + # merge layer (which treats higher score as better, per + # FTS5Backend) ranks vec hits correctly. Bounded; not + # comparable to BM25 scale — hybrid fusion uses RRF + # (#000039 §4.2), not raw-score addition. + score=-float(r["distance"]) if r["distance"] is not None else 0.0, + audit_mode=self.audit_mode, + title=r["title"], + ) + for r in rows + ] + + +__all__ = [ + "VEC_AVAILABLE", + "VEC_BACKEND_VERSION", + "EMBED_MODEL", + "EMBED_DIM", + "VecBackend", + "embed_documents", + "ensure_chunk_vecs_table", + "load_vec_extension", + "default_embedder", +] diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 3dc4138..1e02503 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -100,7 +100,7 @@ Newest first. Update on every open/close. | #000042 | Term-aliases table (vocabulary-mismatch bridge) | closed · 13 rows live across geometry + classical-physics + arithmetic domains by 2026-05-10 | 2026-05-09 | — | | #000041 | Citation-aliases table (PD substitutes for proprietary cites) | closed · 74 rows live as of 2026-05-10 (count grew 40 → 54 → 74; Goldstein/Newton, Mendelson/Enderton/Jech/Landau/Gödel→{Russell IMP, Russell PoM, De Morgan, Boole, Cantor, Peano, Dedekind, SF-LF}, Stanley/Brualdi/Knuth → Bogart+Levin+Keller-Trotter, Dummit-Foote/Barendregt/Böhm-Jacopini → Judson/PLFA/SF, Kolmogorov → Grinstead-Snell+Laplace) | 2026-05-09 | — | | #000040 | Phase 5 resolver fix — phrase + content-token cascade (Hilbert terminology mismatch surfaced) | closed · cascade landed 2026-05-09; lift blocked by 1902-vs-modern vocab; follow-up #000042 | 2026-05-09 | — | -| #000039 | Optional `sqlite-vec` retrieval backend (A/B vs FTS5, hybrid not replacement) | open · awaiting go/no-go (doc-only Phase 0) | 2026-05-09 | — | +| #000039 | Optional `sqlite-vec` retrieval backend (A/B vs FTS5, hybrid not replacement) | in progress · Phase 0 doc + **Phase 1 landed 2026-05-11** (`arborist/search/vec.py` — `VecBackend` + `chunk_vecs` vec0 + `embed_documents` + `arborist embed` / `search --backend vec`; `[vec]` extra = sqlite-vec + fastembed; v1 tuning: bge-small-en-v1.5 / 384 / float32 / cosine / flat / top_k 20; 7 tests; demonstrated on `crawl_appliedcombinatorics_org.db` — 168 chunks, semantic hits topically correct, chain-check 0). Phase 2 (RRF hybrid fusion) gated on ≥5pp recall lift | 2026-05-09 | — | | #000038 | Phase 4 content acquisition — proprietary textbook license decisions for warrant coverage | closed · obviated 2026-05-10 by alias-substitution sprint under #000031 (74 rows in #000041 + 13 rows in #000042); 92/92 records now resolve. Residue (multilingual PD, Hilbert-Ackermann OCR, Knuth permission, personal-copy path B) preserved as design log §8 | 2026-05-09 | — | | #000037 | Prometheus-Σ recursive falsification controller (bicameral substrate) | in progress · Phases 0 + 1 + 1.b + 1.c + 2 landed 2026-05-10; **§12 Trigger 2 fired** (divergence variance 0.575 / N=37); §22 Findings 2 + 3 RESOLVED (kernel/llm cost split + sweep_weights §15.4 + per-mode τ_qa); `controller_events` carries 4 event kinds (decision · difficulty · budget_allocation · falsification_proposal) feeding `arborist controller-events` inspector + live-harvest third bucket in `bench/scripts/harvest_falsification_proposals.py`; §12 Trigger 1 probe wired 2026-05-11 (`trigger_1_branch_density` reads `fork_score_branches` — measurable, not yet fired); Phase 3 sleep-sweep scheduler tracked under #000045 (gating ticket) | 2026-05-09 | — | | #000036 | T3 per-window covert-channel budget bound | **closed · 2026-05-11** · Phase 1 + dav1d review → Tier-1 + Tier-2 (Option B = `b1_model=max_envelope` default, in v1, no v2 fork) + KAT-regen tooling (`scripts/generate_t3_bound_kat.py`) all landed 2026-05-11; baseline 625.87 → 6183.02 (max_envelope), `NOT_CERTIFIED_BY_BOUND` at W=10000; 53 → 83 tests; 12-entry active KAT; both dav1d closure blockers cleared, all §5 acceptance criteria met. Continuation: empirical C_B* tightening under #000043 (parks on v7) | 2026-05-09 | — | diff --git a/docs/tickets/ticket-000039-sqlite-vec-optional-backend.md b/docs/tickets/ticket-000039-sqlite-vec-optional-backend.md index 54e410b..9880354 100644 --- a/docs/tickets/ticket-000039-sqlite-vec-optional-backend.md +++ b/docs/tickets/ticket-000039-sqlite-vec-optional-backend.md @@ -1,6 +1,6 @@ # Ticket #000039 — Optional `sqlite-vec` retrieval backend (A/B vs FTS5, hybrid not replacement) -**Status:** open · awaiting go/no-go (doc-only Phase 0) +**Status:** in progress · Phase 0 (doc) + **Phase 1 landed 2026-05-11**. Phase 1: `arborist/search/vec.py` — `VecBackend(SearchBackend)` (UNGROUNDED hits, never in proof path), `chunk_vecs` vec0 virtual table + `vec_meta` (sibling tables — don't touch chunks/documents/audit chain), `embed_documents()` ingest (delete-then-insert idempotent; vec0 doesn't honor INSERT-OR-REPLACE), pluggable `Embedder` callable with a fastembed `bge-small-en-v1.5` default. CLI: `arborist embed [--limit] [--batch-size]` + `arborist search --backend vec`. `[vec]` optional extra (sqlite-vec + fastembed). **Obvious v1 tuning** (`VEC_BACKEND_VERSION = vec-v1-bge-small-en-v1.5-384float32-cosine-flat`): embedder `BAAI/bge-small-en-v1.5`, dim 384, quant float32 (int8/binary = the production storage knob per §3.1, not wired in v1), metric cosine (bge outputs L2-normalized so cosine ≡ L2 ranking), ANN flat (vec0 default), top_k 20. 7 tests (`tests/test_search_vec.py`, stub embedder — plumbing only; semantic quality demonstrated on a real shard). **Demonstrated on `crawl_appliedcombinatorics_org.db`** (168 chunks embedded in ~37s incl. model load; semantic queries return topically-correct hits — "how many ways to choose k things from n" → top hit "AC Combinations"; chain-check on that shard reports 0 after embedding). The 5 hyperparams fold into `governance_policy_hash` in a later phase (§6 — not wired yet). **Phase 2** (RRF hybrid fusion in `query.py`) gated on a ≥5pp recall-lift measurement on bench fixtures with no STRICT-rate regression (§8). **Opened:** 2026-05-09 **Scope:** Spec an optional `sqlite-vec` backend that runs **alongside** the existing FTS5 retrieval pipeline (never replacing it), with phased gates diff --git a/pyproject.toml b/pyproject.toml index 9e1afde..0ab52bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,20 @@ crawler = [ "cairosvg>=2.7", "pypdf>=4.0", ] +vec = [ + # Optional sqlite-vec semantic retrieval backend (ticket #000039). + # sqlite-vec ships only the loadable SQLite extension (~1 MB); + # fastembed pulls onnxruntime + tokenizers + huggingface-hub + # (~150 MB) and downloads the bge-small-en-v1.5 ONNX model + # (~130 MB) on first use. Gated separately so a fresh checkout + # stays python3.12 + venv + sqlite3. CLI surfaces `arborist embed` + # / `--backend vec` only when `sqlite_vec` imports. Install with: + # pip install 'arborist[vec]' + # (sentence-transformers is the heavier "official" embedder path + # the ticket §5 names; fastembed is the lightweight ONNX one.) + "sqlite-vec>=0.1.9", + "fastembed>=0.4", +] dev = [ "pytest>=8", "pytest-asyncio>=0.23", @@ -75,6 +89,7 @@ dev = [ "arborist[crawler]", "arborist[math]", "arborist[hessian]", + "arborist[vec]", ] [project.scripts] diff --git a/tests/test_search_vec.py b/tests/test_search_vec.py new file mode 100644 index 0000000..6e98b13 --- /dev/null +++ b/tests/test_search_vec.py @@ -0,0 +1,159 @@ +"""Tests for the optional sqlite-vec semantic-retrieval backend (#000039). + +Skips entirely when the ``[vec]`` extra (sqlite-vec) is not installed. +Uses a deterministic *stub* embedder (a hash → unit-vector map) so the +suite exercises the sqlite-vec plumbing — extension load, ``chunk_vecs`` +schema, ingest, KNN query, JOIN back to chunks/documents, Hit shape — +without pulling in the heavy fastembed ONNX model. Real semantic +quality is demonstrated end-to-end on a corpus shard, not unit-tested +(embedding quality isn't a property a unit test can pin). +""" + +from __future__ import annotations + +import hashlib + +import pytest + +from arborist.search import VEC_AVAILABLE + +pytestmark = pytest.mark.skipif( + not VEC_AVAILABLE, reason="sqlite-vec not installed — pip install 'arborist[vec]'" +) + +if VEC_AVAILABLE: + from arborist.search.base import AuditMode + from arborist.search.vec import ( + EMBED_DIM, + VEC_BACKEND_VERSION, + VecBackend, + embed_documents, + ensure_chunk_vecs_table, + ) + from arborist.store import connect + + +# --- deterministic stub embedder ------------------------------------- + + +def _stub_vec(text: str) -> list[float]: + """Map text → a fixed unit vector in R^EMBED_DIM, deterministically. + + Expand SHA-256(text) into EMBED_DIM bytes via counter mode, scale to + [-1, 1), then L2-normalize. Same text → same vector (so a query that + equals a chunk's text gets distance 0); different texts → (almost + surely) different vectors. + """ + raw = bytearray() + i = 0 + while len(raw) < EMBED_DIM: + raw += hashlib.sha256(text.encode("utf-8") + i.to_bytes(4, "little")).digest() + i += 1 + vals = [2.0 * (b / 256.0) - 1.0 for b in raw[:EMBED_DIM]] + norm = sum(v * v for v in vals) ** 0.5 or 1.0 + return [v / norm for v in vals] + + +def _stub_embedder(texts: list[str]) -> list[list[float]]: + return [_stub_vec(t) for t in texts] + + +# --- fixtures -------------------------------------------------------- + + +@pytest.fixture() +def db(tmp_path): + conn = connect(tmp_path / "vec-test.db") + # Three minimal documents, one chunk each. + docs = [ + ("root_a" * 8, "uri://a", "Alpha doc", "alpha content about combinations"), + ("root_b" * 8, "uri://b", "Beta doc", "beta content about permutations"), + ("root_c" * 8, "uri://c", "Gamma doc", "gamma content about pigeonhole"), + ] + for root, uri, title, content in docs: + conn.execute( + "INSERT INTO documents(" + " document_root, document_uri, source_type, kind, title," + " chunking_version, canonicalization_version, schema_version, ingest_ts" + ") VALUES (?, ?, 'test', 'surface', ?, 'tok-512-v1', 'norm-v1', 'v9.8.0', 0)", + (root, uri, title), + ) + conn.execute( + "INSERT INTO chunks(document_root, idx, leaf_hash, content, tier) " + "VALUES (?, 0, ?, ?, 'hot')", + (root, hashlib.sha256(content.encode()).hexdigest(), content), + ) + yield conn, docs + conn.close() + + +# --- tests ----------------------------------------------------------- + + +def test_ensure_table_records_backend_version(db): + conn, _ = db + ensure_chunk_vecs_table(conn) + row = conn.execute( + "SELECT value FROM vec_meta WHERE key='backend_version'" + ).fetchone() + assert row[0] == VEC_BACKEND_VERSION + + +def test_embed_then_search_roundtrips(db): + conn, docs = db + n = embed_documents(conn, embedder=_stub_embedder) + assert n == 3, "all three chunks should embed (none cold)" + + backend = VecBackend(conn, embedder=_stub_embedder) + assert backend.populated() + + # Query with text == doc B's content → distance 0 → top hit is doc B. + target_content = docs[1][3] + hits = backend.search(target_content, limit=3) + assert hits, "search should return hits" + assert hits[0].document_root == docs[1][0] + assert hits[0].audit_mode is AuditMode.UNGROUNDED + assert hits[0].title == "Beta doc" + assert hits[0].chunk_idx == 0 + # Score is the negated distance: exact match → ~0; ordering is + # monotone (best first), so score is non-increasing down the list. + assert hits[0].score >= hits[-1].score + assert hits[0].snippet # non-empty snippet rendered + + +def test_search_respects_limit(db): + conn, _ = db + embed_documents(conn, embedder=_stub_embedder) + backend = VecBackend(conn, embedder=_stub_embedder) + hits = backend.search("anything", limit=2) + assert len(hits) <= 2 + + +def test_search_empty_query_returns_nothing(db): + conn, _ = db + embed_documents(conn, embedder=_stub_embedder) + backend = VecBackend(conn, embedder=_stub_embedder) + assert backend.search(" ", limit=5) == [] + + +def test_unpopulated_backend_returns_nothing(db): + conn, _ = db + backend = VecBackend(conn, embedder=_stub_embedder) + assert not backend.populated() + assert backend.search("anything", limit=5) == [] + + +def test_embed_limit_caps_count(db): + conn, _ = db + n = embed_documents(conn, embedder=_stub_embedder, limit=2) + assert n == 2 + cnt = conn.execute("SELECT COUNT(*) FROM chunk_vecs").fetchone()[0] + assert cnt == 2 + + +def test_embed_is_idempotent(db): + conn, _ = db + embed_documents(conn, embedder=_stub_embedder) + embed_documents(conn, embedder=_stub_embedder) # re-run; INSERT OR REPLACE + cnt = conn.execute("SELECT COUNT(*) FROM chunk_vecs").fetchone()[0] + assert cnt == 3