ticket #000039: --quant {float32,int8} + int8 head-to-head
Wire vector quantization (the §3.1 production knob): arborist embed
--quant int8 [--rebuild]. The chunk_vecs vec0 column becomes int8[384]
vs float[384] per quant; the quant folds into VEC_BACKEND_VERSION
(...-384int8-... / ...-384float32-...) and vec_meta records it per
shard. Switching quant on an existing chunk_vecs requires --rebuild
(the vec0 element type can't be altered in place — embed_documents
raises ValueError telling you to --rebuild).
int8 serialization: scale each bge component by 127 (theoretical
[-1,1] range), clamp to [-127,127], round, serialize_int8. The same
scaling on the query vector → distances comparable; cosine is
scale-invariant so the uniform x127 cancels in the ranking.
sqlite-vec v0.1.9 quirk worked around: a bare blob inserted into a
vec0 column is interpreted as float32 regardless of the column's
declared type — int8 vectors MUST be wrapped in vec_int8(...). So
the INSERT and the MATCH now wrap the blob in vec_f32(?) (float32)
or vec_int8(?) (int8) — constructor name from a fixed dict, no
injection surface. (Discovered the hard way: a bare int8 blob into
an int8[384] column → "expected int8, but a float32 vector was
provided".)
VecBackend reads the quant from the existing chunk_vecs schema (or
defaults to float32) so search uses the matching wrapper. New module
exports: QUANTS, EMBED_QUANT, vec_backend_version(quant), existing_quant.
int8 head-to-head on crawl_appliedcombinatorics_org.db (168 chunks):
- storage: float32 1,597,440 B -> int8 417,792 B = 3.8x smaller
(~4x at corpus scale where the 1024-vector blocks fill; the
~28 KB of vec0 metadata doesn't quarter, hence 3.8 not 4.0).
- recall vs the float32 baseline:
Q "how many ways to choose k things from n":
identical top-5 (Combinations, Permutations, Exercises,
Derangements, Graph Coloring).
Q "pigeonhole principle counting":
identical top-2 (Graph Coloring, Exercises); ranks 3-4 swap
Derangements <-> Permutations at Δdistance 0.002 — sub-noise.
- embed speed unchanged (~3.9 chunks/s — model-load-dominated).
Conclusion: int8 is the obvious production config (§3.1's +6%-tax
recommendation confirmed empirically). v1 default stays float32 for
max fidelity; flipping the default to int8 is a fox call.
CLI (arborist/cli.py): arborist embed --quant {float32,int8}; output
JSON gains "quant"; embed_documents ValueError → exit 2 with the
"--rebuild" hint.
tests/test_search_vec.py (9 -> 16): test_int8_quant_roundtrips
(int8[384] schema, vec_meta version, search round-trip, quant
inferred by VecBackend), test_quant_mismatch_requires_rebuild,
test_invalid_quant_rejected.
#000039 status updated. Full suite: 2343 passed, 28 skipped.
(Unrelated parallel-clone work in the tree — Makefile, arborist/qa/
verify.py, bench/fixtures/5f/*, tests/test_bench_batteries.py,
tests/test_verify.py — is #000046's hard-fixture tier, not touched.)
This commit is contained in:
parent
7a64939e2b
commit
06f5a11651
4 changed files with 202 additions and 41 deletions
|
|
@ -302,11 +302,10 @@ def _cmd_embed(args: argparse.Namespace) -> int:
|
|||
"""
|
||||
import time as _time
|
||||
|
||||
from arborist.search.vec import (
|
||||
VEC_BACKEND_VERSION,
|
||||
embed_documents,
|
||||
)
|
||||
from arborist.search.vec import embed_documents, vec_backend_version
|
||||
|
||||
quant = getattr(args, "quant", "float32")
|
||||
rebuild = getattr(args, "rebuild", False)
|
||||
conn = connect(args.db)
|
||||
t0 = _time.monotonic()
|
||||
last_print = [0.0]
|
||||
|
|
@ -327,17 +326,22 @@ def _cmd_embed(args: argparse.Namespace) -> int:
|
|||
conn,
|
||||
limit=args.limit,
|
||||
batch_size=args.batch_size,
|
||||
incremental=not getattr(args, "rebuild", False),
|
||||
rebuild=getattr(args, "rebuild", False),
|
||||
incremental=not rebuild,
|
||||
rebuild=rebuild,
|
||||
quant=quant,
|
||||
progress=_progress,
|
||||
)
|
||||
except ValueError as e:
|
||||
print(f"embed error: {e}", file=sys.stderr)
|
||||
return 2
|
||||
finally:
|
||||
conn.close()
|
||||
elapsed = _time.monotonic() - t0
|
||||
print(json.dumps({
|
||||
"db": str(args.db),
|
||||
"backend_version": VEC_BACKEND_VERSION,
|
||||
"mode": "rebuild" if getattr(args, "rebuild", False) else "incremental",
|
||||
"backend_version": vec_backend_version(quant),
|
||||
"quant": quant,
|
||||
"mode": "rebuild" if rebuild else "incremental",
|
||||
"chunks_embedded": n,
|
||||
"elapsed_s": round(elapsed, 2),
|
||||
"rate_per_s": round(n / max(elapsed, 1e-6), 1),
|
||||
|
|
@ -4737,12 +4741,22 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
help="cap the number of chunks embedded (smoke-test on a real shard)",
|
||||
)
|
||||
embed_cmd.add_argument("--batch-size", type=int, default=256)
|
||||
embed_cmd.add_argument(
|
||||
"--quant", choices=["float32", "int8"], default="float32",
|
||||
help=(
|
||||
"vector quantization: float32 (default — +25%% corpus tax, max "
|
||||
"fidelity) or int8 (~4x smaller, +6%% tax, ~1-3%% recall hit — "
|
||||
"the recommended production default; switching quant on an "
|
||||
"existing chunk_vecs requires --rebuild)"
|
||||
),
|
||||
)
|
||||
embed_cmd.add_argument(
|
||||
"--rebuild", action="store_true",
|
||||
help=(
|
||||
"DROP + recreate chunk_vecs, then full re-embed — the clean "
|
||||
"VEC_BACKEND_VERSION-bump path (default is incremental: embed "
|
||||
"only chunks not already in chunk_vecs)"
|
||||
"DROP + recreate chunk_vecs (at --quant), then full re-embed — "
|
||||
"the clean version-bump path and the only way to switch quant "
|
||||
"(default is incremental: embed only chunks not already in "
|
||||
"chunk_vecs)"
|
||||
),
|
||||
)
|
||||
embed_cmd.set_defaults(func=_cmd_embed)
|
||||
|
|
|
|||
|
|
@ -61,17 +61,44 @@ except ImportError: # pragma: no cover - exercised in CI matrices w/o the extra
|
|||
|
||||
EMBED_MODEL = "BAAI/bge-small-en-v1.5"
|
||||
EMBED_DIM = 384
|
||||
EMBED_QUANT = "float32"
|
||||
EMBED_METRIC = "cosine"
|
||||
ANN_INDEX = "flat"
|
||||
DEFAULT_TOP_K = 20
|
||||
|
||||
# Rotation discriminator. Bumping any hyperparameter above changes this
|
||||
# token; the vec_meta table records it per shard so a mismatch is
|
||||
# detectable at lookup time (and a `make rebuild-vec` re-embeds).
|
||||
VEC_BACKEND_VERSION = (
|
||||
f"vec-v1-bge-small-en-v1.5-{EMBED_DIM}{EMBED_QUANT}-{EMBED_METRIC}-{ANN_INDEX}"
|
||||
)
|
||||
# Quantization knob (#000039 §3.1). float32 = +25% corpus tax (the v1
|
||||
# default — simplest, max fidelity); int8 = ~4× smaller (+6% tax, ~1-3%
|
||||
# recall hit — the recommended production default). binary is a future
|
||||
# pass (needs a two-stage re-rank). bge outputs L2-normalized [-1,1]-ish
|
||||
# components; int8 scales by 127 (theoretical-range scaling — leaves
|
||||
# headroom but plenty of resolution for cosine ranking).
|
||||
QUANTS = ("float32", "int8")
|
||||
EMBED_QUANT = "float32" # the v1 default
|
||||
_INT8_SCALE = 127.0
|
||||
|
||||
# Per-quant vec0 column element type and the SQL constructor that tags a
|
||||
# blob as that type. sqlite-vec v0.1.9 treats a bare blob as float32 —
|
||||
# int8 blobs MUST be wrapped in vec_int8(...); we wrap float32 in
|
||||
# vec_f32(...) too for symmetry/clarity. (Constructor names come from
|
||||
# this fixed dict, never from user input — no SQL-injection surface.)
|
||||
_VEC0_ELEM_TYPE = {"float32": "float", "int8": "int8"}
|
||||
_VEC_CTOR = {"float32": "vec_f32", "int8": "vec_int8"}
|
||||
|
||||
|
||||
def vec_backend_version(quant: str = EMBED_QUANT) -> str:
|
||||
"""Rotation discriminator for a given quantization.
|
||||
|
||||
Bumping the model / dim / metric / ann / quant changes this token;
|
||||
``vec_meta.backend_version`` records it per shard so a mismatch is
|
||||
detectable at lookup time (and an ``arborist embed --rebuild`` under
|
||||
a different quant re-embeds + recreates the table at the new type).
|
||||
"""
|
||||
if quant not in QUANTS:
|
||||
raise ValueError(f"quant must be one of {QUANTS}; got {quant!r}")
|
||||
return f"vec-v1-bge-small-en-v1.5-{EMBED_DIM}{quant}-{EMBED_METRIC}-{ANN_INDEX}"
|
||||
|
||||
|
||||
# Back-compat constant — the float32 (v1 default) version token.
|
||||
VEC_BACKEND_VERSION = vec_backend_version(EMBED_QUANT)
|
||||
|
||||
|
||||
# --- extension load + schema -----------------------------------------
|
||||
|
|
@ -95,19 +122,51 @@ def load_vec_extension(conn: sqlite3.Connection) -> None:
|
|||
conn.enable_load_extension(False)
|
||||
|
||||
|
||||
def ensure_chunk_vecs_table(conn: sqlite3.Connection) -> None:
|
||||
def existing_quant(conn: sqlite3.Connection) -> str | None:
|
||||
"""Return the quant of an existing ``chunk_vecs`` table, or None.
|
||||
|
||||
Parsed from the table's stored CREATE statement (``... embedding
|
||||
int8[384] ...`` vs ``... embedding float[384] ...``).
|
||||
"""
|
||||
row = conn.execute(
|
||||
"SELECT sql FROM sqlite_master WHERE type='table' AND name='chunk_vecs'"
|
||||
).fetchone()
|
||||
if not row or not row[0]:
|
||||
return None
|
||||
sql = row[0]
|
||||
for quant, elem in _VEC0_ELEM_TYPE.items():
|
||||
if f"embedding {elem}[" in sql:
|
||||
return quant
|
||||
return None
|
||||
|
||||
|
||||
def ensure_chunk_vecs_table(conn: sqlite3.Connection, *, quant: str = EMBED_QUANT) -> None:
|
||||
"""Create the ``chunk_vecs`` vec0 virtual table + ``vec_meta`` if absent.
|
||||
|
||||
``chunk_vecs`` is a *sibling* table (additive — does not touch
|
||||
``chunks`` / ``documents`` / the audit chain). ``vec_meta`` records
|
||||
the ``VEC_BACKEND_VERSION`` that populated this shard so a
|
||||
hyperparameter rotation is detectable.
|
||||
the ``vec_backend_version(quant)`` that populated this shard so a
|
||||
rotation (including a quant change) is detectable.
|
||||
|
||||
Raises ``ValueError`` if ``chunk_vecs`` already exists at a *different*
|
||||
quant — switching quant requires recreating the table (use
|
||||
``embed_documents(..., rebuild=True)`` / ``arborist embed --rebuild
|
||||
--quant ...``), since the vec0 column element type can't be altered
|
||||
in place.
|
||||
"""
|
||||
if quant not in QUANTS:
|
||||
raise ValueError(f"quant must be one of {QUANTS}; got {quant!r}")
|
||||
load_vec_extension(conn)
|
||||
have = existing_quant(conn)
|
||||
if have is not None and have != quant:
|
||||
raise ValueError(
|
||||
f"chunk_vecs already exists at quant={have!r}; cannot switch to "
|
||||
f"{quant!r} in place — re-run with --rebuild (DROP + recreate)."
|
||||
)
|
||||
conn.execute(
|
||||
f"CREATE VIRTUAL TABLE IF NOT EXISTS chunk_vecs USING vec0("
|
||||
f" chunk_id INTEGER PRIMARY KEY,"
|
||||
f" embedding float[{EMBED_DIM}] distance_metric={EMBED_METRIC}"
|
||||
f" embedding {_VEC0_ELEM_TYPE[quant]}[{EMBED_DIM}] distance_metric={EMBED_METRIC}"
|
||||
f")"
|
||||
)
|
||||
conn.execute(
|
||||
|
|
@ -118,7 +177,7 @@ def ensure_chunk_vecs_table(conn: sqlite3.Connection) -> None:
|
|||
)
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO vec_meta(key, value) VALUES ('backend_version', ?)",
|
||||
(VEC_BACKEND_VERSION,),
|
||||
(vec_backend_version(quant),),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -159,9 +218,25 @@ def default_embedder() -> Embedder:
|
|||
return _embed
|
||||
|
||||
|
||||
def _to_blob(vec: Iterable[float]) -> bytes:
|
||||
"""Serialize a float vector to sqlite-vec's float32 blob form."""
|
||||
return sqlite_vec.serialize_float32(list(vec)) # type: ignore[union-attr]
|
||||
def _to_blob(vec: Iterable[float], quant: str = EMBED_QUANT) -> bytes:
|
||||
"""Serialize a float vector to sqlite-vec's blob form for ``quant``.
|
||||
|
||||
float32: ``serialize_float32`` verbatim. int8: scale each component
|
||||
by 127 (theoretical [-1,1] range), clamp to [-127, 127], round, then
|
||||
``serialize_int8``. The same scaling is applied to the query vector
|
||||
so distances are comparable. (Cosine is scale-invariant, so the
|
||||
uniform ×127 cancels in the ranking; clamping touches only the rare
|
||||
component beyond ±1.)
|
||||
"""
|
||||
vals = list(vec)
|
||||
if quant == "float32":
|
||||
return sqlite_vec.serialize_float32(vals) # type: ignore[union-attr]
|
||||
if quant == "int8":
|
||||
scaled = [
|
||||
max(-127, min(127, int(round(v * _INT8_SCALE)))) for v in vals
|
||||
]
|
||||
return sqlite_vec.serialize_int8(scaled) # type: ignore[union-attr]
|
||||
raise ValueError(f"quant must be one of {QUANTS}; got {quant!r}")
|
||||
|
||||
|
||||
# --- ingest ----------------------------------------------------------
|
||||
|
|
@ -175,13 +250,14 @@ def embed_documents(
|
|||
batch_size: int = 256,
|
||||
incremental: bool = True,
|
||||
rebuild: bool = False,
|
||||
quant: str = EMBED_QUANT,
|
||||
progress: Callable[[int, int], None] | None = None,
|
||||
) -> int:
|
||||
"""Populate ``chunk_vecs`` for ``chunks`` rows with content.
|
||||
|
||||
Idempotency model (see #000039 — chunk content is immutable per
|
||||
``chunk_id``: a chunk, once embedded, never needs re-embedding
|
||||
unless the *embedder* changes):
|
||||
unless the *embedder* or *quant* changes):
|
||||
|
||||
- ``incremental=True`` (default): embed only chunks **not already
|
||||
in chunk_vecs** (``chunk_id NOT IN (SELECT chunk_id FROM chunk_vecs)``).
|
||||
|
|
@ -190,26 +266,38 @@ def embed_documents(
|
|||
(no-op once everything's embedded).
|
||||
- ``incremental=False``: re-embed every chunk with content
|
||||
(delete-then-insert all). The embedder-changed case.
|
||||
- ``rebuild=True``: DROP + recreate ``chunk_vecs`` first, then a
|
||||
full pass — the clean ``VEC_BACKEND_VERSION`` bump (so a search
|
||||
mid-rebuild never mixes old- and new-model embeddings: it's all
|
||||
new-model from the recreated table, growing as the pass runs).
|
||||
Implies a full (non-incremental) pass.
|
||||
- ``rebuild=True``: DROP + recreate ``chunk_vecs`` first (at the
|
||||
requested ``quant``), then a full pass — the clean
|
||||
``vec_backend_version`` bump *and* the only way to switch quant
|
||||
(the vec0 column element type can't be altered in place). A search
|
||||
mid-rebuild never mixes configs: it's all new from the recreated
|
||||
table, growing as the pass runs. Implies a full (non-incremental)
|
||||
pass. **Required when changing ``quant`` on an existing table.**
|
||||
|
||||
``quant`` ∈ {"float32" (default, +25% corpus tax, max fidelity),
|
||||
"int8" (~4× smaller, +6% tax, ~1-3% recall hit — production default)}.
|
||||
|
||||
Cold-evicted chunks (``content IS NULL``) are skipped — the vec
|
||||
row, if it exists, stays valid (content is the same on rehydrate).
|
||||
``limit`` caps chunks processed (smoke-test knob). Returns the
|
||||
count newly embedded this call.
|
||||
"""
|
||||
if quant not in QUANTS:
|
||||
raise ValueError(f"quant must be one of {QUANTS}; got {quant!r}")
|
||||
embedder = embedder or default_embedder()
|
||||
load_vec_extension(conn)
|
||||
if not rebuild and existing_quant(conn) not in (None, quant):
|
||||
raise ValueError(
|
||||
f"chunk_vecs exists at quant={existing_quant(conn)!r}; pass "
|
||||
f"rebuild=True (--rebuild) to switch to {quant!r}."
|
||||
)
|
||||
if rebuild:
|
||||
try:
|
||||
load_vec_extension(conn)
|
||||
conn.execute("DROP TABLE IF EXISTS chunk_vecs")
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
incremental = False # recreated table is empty → full pass anyway
|
||||
ensure_chunk_vecs_table(conn)
|
||||
ensure_chunk_vecs_table(conn, quant=quant)
|
||||
|
||||
where = "content IS NOT NULL"
|
||||
if incremental:
|
||||
|
|
@ -241,8 +329,9 @@ def embed_documents(
|
|||
[(cid,) for cid in batch_ids],
|
||||
)
|
||||
conn.executemany(
|
||||
"INSERT INTO chunk_vecs(chunk_id, embedding) VALUES (?, ?)",
|
||||
[(cid, _to_blob(v)) for cid, v in zip(batch_ids, vecs)],
|
||||
f"INSERT INTO chunk_vecs(chunk_id, embedding) "
|
||||
f"VALUES (?, {_VEC_CTOR[quant]}(?))",
|
||||
[(cid, _to_blob(v, quant)) for cid, v in zip(batch_ids, vecs)],
|
||||
)
|
||||
conn.execute("COMMIT")
|
||||
except Exception:
|
||||
|
|
@ -279,10 +368,20 @@ class VecBackend(SearchBackend):
|
|||
name = "vec"
|
||||
audit_mode = AuditMode.UNGROUNDED
|
||||
|
||||
def __init__(self, conn: sqlite3.Connection, embedder: Embedder | None = None):
|
||||
def __init__(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
embedder: Embedder | None = None,
|
||||
*,
|
||||
quant: str | None = None,
|
||||
):
|
||||
super().__init__(conn)
|
||||
load_vec_extension(conn)
|
||||
self._embedder = embedder # lazy: default constructed on first search
|
||||
# The quant the existing chunk_vecs was populated at; falls back
|
||||
# to the v1 default if the table doesn't exist yet (search will
|
||||
# return [] in that case anyway — see populated()).
|
||||
self.quant = quant or existing_quant(conn) or EMBED_QUANT
|
||||
|
||||
@property
|
||||
def embedder(self) -> Embedder:
|
||||
|
|
@ -305,7 +404,7 @@ class VecBackend(SearchBackend):
|
|||
return []
|
||||
qvec = next(iter(self.embedder([query])))
|
||||
rows = self.conn.execute(
|
||||
"""
|
||||
f"""
|
||||
SELECT
|
||||
cv.chunk_id,
|
||||
cv.distance AS distance,
|
||||
|
|
@ -317,11 +416,11 @@ class VecBackend(SearchBackend):
|
|||
FROM chunk_vecs AS cv
|
||||
JOIN chunks AS c ON c.chunk_id = cv.chunk_id
|
||||
JOIN documents AS d ON d.document_root = c.document_root
|
||||
WHERE cv.embedding MATCH ?
|
||||
WHERE cv.embedding MATCH {_VEC_CTOR[self.quant]}(?)
|
||||
AND k = ?
|
||||
ORDER BY cv.distance
|
||||
""",
|
||||
(_to_blob(qvec), int(limit)),
|
||||
(_to_blob(qvec, self.quant), int(limit)),
|
||||
).fetchall()
|
||||
return [
|
||||
Hit(
|
||||
|
|
@ -347,6 +446,10 @@ __all__ = [
|
|||
"VEC_BACKEND_VERSION",
|
||||
"EMBED_MODEL",
|
||||
"EMBED_DIM",
|
||||
"EMBED_QUANT",
|
||||
"QUANTS",
|
||||
"vec_backend_version",
|
||||
"existing_quant",
|
||||
"VecBackend",
|
||||
"embed_documents",
|
||||
"ensure_chunk_vecs_table",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Ticket #000039 — Optional `sqlite-vec` retrieval backend (A/B vs FTS5, hybrid not replacement)
|
||||
|
||||
**Status:** in progress · Phase 0 (doc) + **Phase 1 landed 2026-05-11**. Phase 1: `arborist/search/vec.py` — `VecBackend(SearchBackend)` (UNGROUNDED hits, never in proof path), `chunk_vecs` vec0 virtual table + `vec_meta` (sibling tables — don't touch chunks/documents/audit chain), `embed_documents()` ingest (delete-then-insert idempotent; vec0 doesn't honor INSERT-OR-REPLACE), pluggable `Embedder` callable with a fastembed `bge-small-en-v1.5` default. CLI: `arborist embed [--limit] [--batch-size]` + `arborist search --backend vec`. `[vec]` optional extra (sqlite-vec + fastembed). **Obvious v1 tuning** (`VEC_BACKEND_VERSION = vec-v1-bge-small-en-v1.5-384float32-cosine-flat`): embedder `BAAI/bge-small-en-v1.5`, dim 384, quant float32 (int8/binary = the production storage knob per §3.1, not wired in v1), metric cosine (bge outputs L2-normalized so cosine ≡ L2 ranking), ANN flat (vec0 default), top_k 20. 7 tests (`tests/test_search_vec.py`, stub embedder — plumbing only; semantic quality demonstrated on a real shard). **Demonstrated on `crawl_appliedcombinatorics_org.db`** (168 chunks embedded in ~37s incl. model load; semantic queries return topically-correct hits — "how many ways to choose k things from n" → top hit "AC Combinations"; chain-check on that shard reports 0 after embedding). The 5 hyperparams fold into `governance_policy_hash` in a later phase (§6 — not wired yet). **Ingest integration landed 2026-05-11** (see §14): `embed_documents()` is now **incremental by default** (embeds only `chunk_id`s not already in `chunk_vecs` — re-runs are cheap no-ops); `arborist embed --rebuild` does the DROP+recreate+full-re-embed for a `VEC_BACKEND_VERSION` bump; `arborist ingest --embed` is the **eager opt-in** (embed this run's new chunks after the chunk+Merkle-commit pass; default ingest does NOT embed — the lazy `arborist embed` pass / cron / Prometheus-Σ unconscious sweep is the usual path). 9 vec tests. **Phase 2** (RRF hybrid fusion in `query.py`) gated on a ≥5pp recall-lift measurement on bench fixtures with no STRICT-rate regression (§8).
|
||||
**Status:** in progress · Phase 0 (doc) + **Phase 1 landed 2026-05-11**. Phase 1: `arborist/search/vec.py` — `VecBackend(SearchBackend)` (UNGROUNDED hits, never in proof path), `chunk_vecs` vec0 virtual table + `vec_meta` (sibling tables — don't touch chunks/documents/audit chain), `embed_documents()` ingest (delete-then-insert idempotent; vec0 doesn't honor INSERT-OR-REPLACE), pluggable `Embedder` callable with a fastembed `bge-small-en-v1.5` default. CLI: `arborist embed [--limit] [--batch-size]` + `arborist search --backend vec`. `[vec]` optional extra (sqlite-vec + fastembed). **Obvious v1 tuning** (`VEC_BACKEND_VERSION = vec-v1-bge-small-en-v1.5-384float32-cosine-flat`): embedder `BAAI/bge-small-en-v1.5`, dim 384, quant float32 (int8/binary = the production storage knob per §3.1, not wired in v1), metric cosine (bge outputs L2-normalized so cosine ≡ L2 ranking), ANN flat (vec0 default), top_k 20. 7 tests (`tests/test_search_vec.py`, stub embedder — plumbing only; semantic quality demonstrated on a real shard). **Demonstrated on `crawl_appliedcombinatorics_org.db`** (168 chunks embedded in ~37s incl. model load; semantic queries return topically-correct hits — "how many ways to choose k things from n" → top hit "AC Combinations"; chain-check on that shard reports 0 after embedding). The 5 hyperparams fold into `governance_policy_hash` in a later phase (§6 — not wired yet). **Ingest integration landed 2026-05-11** (see §14): `embed_documents()` is now **incremental by default** (embeds only `chunk_id`s not already in `chunk_vecs` — re-runs are cheap no-ops); `arborist embed --rebuild` does the DROP+recreate+full-re-embed for a `VEC_BACKEND_VERSION` bump; `arborist ingest --embed` is the **eager opt-in** (embed this run's new chunks after the chunk+Merkle-commit pass; default ingest does NOT embed — the lazy `arborist embed` pass / cron / Prometheus-Σ unconscious sweep is the usual path). **`--quant {float32,int8}` landed 2026-05-11** (`arborist embed --quant int8 [--rebuild]`; the `chunk_vecs` vec0 column is `int8[384]` vs `float[384]` per quant; int8 blobs are `vec_int8(?)`-wrapped — sqlite-vec v0.1.9 treats a bare blob as float32; quant change on an existing table requires `--rebuild` since the vec0 element type can't be altered in place; the quant folds into `VEC_BACKEND_VERSION` → `...-384int8-...`, recorded in `vec_meta`). **int8 head-to-head on `crawl_appliedcombinatorics_org.db`**: storage 1.60 MB → **0.42 MB (3.8× smaller; ~4× at corpus scale where blocks fill)**; recall ≈ float32 — Q2 "how many ways to choose k things from n" identical top-5, Q1 "pigeonhole principle counting" identical top-2 with a sub-noise rank-3/4 swap (Δdistance 0.002). So int8 is the obvious production config (§3.1's +6%-tax recommendation confirmed) — v1 default stays float32 for max fidelity; switching the default to int8 is a fox call. 16 vec tests. **Phase 2** (RRF hybrid fusion in `query.py`) gated on a ≥5pp recall-lift measurement on bench fixtures with no STRICT-rate regression (§8).
|
||||
**Opened:** 2026-05-09
|
||||
**Scope:** Spec an optional `sqlite-vec` backend that runs **alongside** the
|
||||
existing FTS5 retrieval pipeline (never replacing it), with phased gates
|
||||
|
|
|
|||
|
|
@ -203,3 +203,47 @@ def test_embed_rebuild_re_embeds_all(db):
|
|||
"SELECT value FROM vec_meta WHERE key='backend_version'"
|
||||
).fetchone()
|
||||
assert row[0] == VEC_BACKEND_VERSION
|
||||
|
||||
|
||||
def test_int8_quant_roundtrips(db):
|
||||
"""quant='int8': table is int8[384], vec_meta records the int8 version,
|
||||
search round-trips (a query equal to a chunk's content → that chunk on
|
||||
top, since same float vec → same int8 vec → distance ~0)."""
|
||||
from arborist.search.vec import existing_quant, vec_backend_version
|
||||
conn, docs = db
|
||||
n = embed_documents(conn, embedder=_stub_embedder, quant="int8", rebuild=True)
|
||||
assert n == 3
|
||||
assert existing_quant(conn) == "int8"
|
||||
sql = conn.execute(
|
||||
"SELECT sql FROM sqlite_master WHERE name='chunk_vecs'"
|
||||
).fetchone()[0]
|
||||
assert "int8[384]" in sql
|
||||
assert conn.execute(
|
||||
"SELECT value FROM vec_meta WHERE key='backend_version'"
|
||||
).fetchone()[0] == vec_backend_version("int8")
|
||||
|
||||
backend = VecBackend(conn, embedder=_stub_embedder) # quant inferred from table
|
||||
assert backend.quant == "int8"
|
||||
hits = backend.search(docs[2][3], limit=3) # query == doc C's content
|
||||
assert hits and hits[0].document_root == docs[2][0]
|
||||
assert hits[0].audit_mode is AuditMode.UNGROUNDED
|
||||
|
||||
|
||||
def test_quant_mismatch_requires_rebuild(db):
|
||||
"""Switching quant on an existing chunk_vecs without --rebuild errors —
|
||||
the vec0 column element type can't be altered in place."""
|
||||
conn, _ = db
|
||||
embed_documents(conn, embedder=_stub_embedder, quant="float32") # default
|
||||
with pytest.raises(ValueError, match="rebuild"):
|
||||
embed_documents(conn, embedder=_stub_embedder, quant="int8") # no rebuild
|
||||
# ...but with rebuild it switches cleanly.
|
||||
embed_documents(conn, embedder=_stub_embedder, quant="int8", rebuild=True)
|
||||
from arborist.search.vec import existing_quant
|
||||
assert existing_quant(conn) == "int8"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad", ["fp16", "binary", "int4", "", None])
|
||||
def test_invalid_quant_rejected(db, bad):
|
||||
conn, _ = db
|
||||
with pytest.raises(ValueError, match="quant"):
|
||||
embed_documents(conn, embedder=_stub_embedder, quant=bad) # type: ignore[arg-type]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue