ticket #000039: ingest integration + incremental embed (both models)
Fox: "i like both" — keep the lazy out-of-band pass as the default
AND add the eager opt-in. Plus the Phase-1 gap fix (incremental embed).
Key insight folded into the design (new ticket section 14): a chunk_id's
content is immutable in arborist — same content → same chunk_id;
different content → a NEW chunk_id (re-ingest makes a new doc_root +
new chunk_ids linked by supersedes; a chunker bump re-chunks → new
chunk_ids). So a chunk, once embedded, never needs re-embedding — the
ONLY re-embed trigger is the embedder changing (VEC_BACKEND_VERSION
bump). That makes the idempotency story clean.
arborist/search/vec.py — embed_documents() now:
- incremental=True (default): embed only chunk_ids NOT already in
chunk_vecs (chunk_id NOT IN (SELECT chunk_id FROM chunk_vecs)).
This is the after-ingest / cron / Prometheus-Sigma-sweep path —
it picks up exactly the newly-ingested chunks; re-running is a
cheap no-op once everything's embedded.
- incremental=False: re-embed every chunk with content (delete-then-
insert all) — the embedder-changed case.
- rebuild=True: DROP + recreate chunk_vecs first, then a full pass —
the clean VEC_BACKEND_VERSION-bump path (a search mid-rebuild never
mixes old- and new-model embeddings: the recreated table starts
empty and grows new-model as the pass runs). Implies non-incremental.
- Cold-evicted chunks (content NULL) still skipped; vec rows persist
and stay valid (content is identical on rehydrate).
CLI (arborist/cli.py):
- arborist embed --rebuild — the DROP+recreate+full-re-embed path
(default is incremental). Output JSON now reports "mode".
- arborist ingest --embed — eager opt-in: after the chunk+Merkle-
commit pass, incremental-embed this run's new chunks. Default
ingest does NOT embed. ingest output gains "chunks_embedded" when
--embed is set. Only surfaced when the [vec] extra is installed.
- Hoisted the _vec_ok check up to the top of build_parser so both
the ingest --embed flag and the search --backend / embed subcommand
can gate on it.
tests/test_search_vec.py (7 -> 9): test_embed_incremental_only_embeds
_new_chunks (second pass after a follow-up ingest embeds only the new
chunk; third pass is a no-op), test_embed_rebuild_re_embeds_all
(DROP+recreate+full pass; vec_meta still records the version).
Verified on crawl_appliedcombinatorics_org.db: incremental on an
already-embedded shard reports chunks_embedded=0 in ~1.8s; --rebuild
re-embeds all 168 in ~61s; semantic search after rebuild still returns
topically-correct hits ("pigeonhole principle counting" -> "AC Graph
Coloring" chunk containing "Generalized Pigeon Hole Principle").
Ticket section 14 added: the idempotency table (re-ingest / chunker
bump / cold eviction / embedder bump / superseded docs), the two
integration models (lazy default + eager opt-in; the lazy pass's
natural home is a Prometheus-Sigma unconscious-sweep task per #000037
section 3.1), the command matrix, concurrency notes, versioning.
Status line updated.
Full suite: 2339 passed, 28 skipped.
(Unrelated parallel-clone work in the working tree — Makefile,
arborist/qa/verify.py, bench/fixtures/5f/*, tests/test_bench_batteries.py,
tests/test_verify.py — is #000046's hard-fixture tier, not touched here.)
This commit is contained in:
parent
0c1bad8494
commit
86d47e6aee
4 changed files with 202 additions and 21 deletions
|
|
@ -211,6 +211,7 @@ def _cmd_ingest(args: argparse.Namespace) -> int:
|
|||
)
|
||||
|
||||
conn = connect(target_db)
|
||||
embedded = None
|
||||
try:
|
||||
result = ingest_source(
|
||||
conn,
|
||||
|
|
@ -224,9 +225,20 @@ def _cmd_ingest(args: argparse.Namespace) -> int:
|
|||
loss_report_excerpts=not args.no_loss_excerpts,
|
||||
loss_report_max_excerpt_bytes=args.loss_excerpt_bytes,
|
||||
)
|
||||
if getattr(args, "embed", False):
|
||||
# Eager opt-in: after the chunk+Merkle-commit pass, embed the
|
||||
# chunks this run added (incremental — only chunk_ids not
|
||||
# already in chunk_vecs). Default ingest does NOT embed; the
|
||||
# lazy out-of-band pass (`arborist embed`, or a cron, or the
|
||||
# Prometheus-Σ unconscious sweep) is the usual path. See #000039.
|
||||
from arborist.search.vec import embed_documents
|
||||
embedded = embed_documents(conn, incremental=True)
|
||||
finally:
|
||||
conn.close()
|
||||
print(json.dumps(result.__dict__, indent=2, ensure_ascii=False))
|
||||
out = dict(result.__dict__)
|
||||
if embedded is not None:
|
||||
out["chunks_embedded"] = embedded
|
||||
print(json.dumps(out, indent=2, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
|
|
@ -315,6 +327,8 @@ def _cmd_embed(args: argparse.Namespace) -> int:
|
|||
conn,
|
||||
limit=args.limit,
|
||||
batch_size=args.batch_size,
|
||||
incremental=not getattr(args, "rebuild", False),
|
||||
rebuild=getattr(args, "rebuild", False),
|
||||
progress=_progress,
|
||||
)
|
||||
finally:
|
||||
|
|
@ -323,6 +337,7 @@ def _cmd_embed(args: argparse.Namespace) -> int:
|
|||
print(json.dumps({
|
||||
"db": str(args.db),
|
||||
"backend_version": VEC_BACKEND_VERSION,
|
||||
"mode": "rebuild" if getattr(args, "rebuild", False) else "incremental",
|
||||
"chunks_embedded": n,
|
||||
"elapsed_s": round(elapsed, 2),
|
||||
"rate_per_s": round(n / max(elapsed, 1e-6), 1),
|
||||
|
|
@ -4514,6 +4529,14 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
)
|
||||
sub = p.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
# Optional sqlite-vec backend (#000039): only surface --embed /
|
||||
# `arborist embed` / `search --backend vec` when the [vec] extra
|
||||
# is installed.
|
||||
try:
|
||||
from arborist.search import VEC_AVAILABLE as _vec_ok
|
||||
except Exception: # pragma: no cover
|
||||
_vec_ok = False
|
||||
|
||||
ingest = sub.add_parser("ingest", help="ingest documents from a source")
|
||||
ingest.add_argument(
|
||||
"--source",
|
||||
|
|
@ -4676,19 +4699,23 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
default=200,
|
||||
help="max excerpt byte length (default 200)",
|
||||
)
|
||||
if _vec_ok:
|
||||
ingest.add_argument(
|
||||
"--embed", action="store_true",
|
||||
help=(
|
||||
"after ingest, embed the chunks this run added (incremental "
|
||||
"chunk_vecs population — #000039; default ingest does NOT "
|
||||
"embed, the lazy `arborist embed` pass / cron / Prometheus-Σ "
|
||||
"sweep is the usual path)"
|
||||
),
|
||||
)
|
||||
ingest.set_defaults(func=_cmd_ingest)
|
||||
|
||||
search = sub.add_parser("search", help="keyword/semantic search (UNGROUNDED audit mode)")
|
||||
search.add_argument("query", help="query string")
|
||||
search.add_argument("--limit", type=int, default=20)
|
||||
search.add_argument("--json", action="store_true", help="output JSON")
|
||||
_search_backends = ["fts5"]
|
||||
try:
|
||||
from arborist.search import VEC_AVAILABLE as _vec_ok
|
||||
if _vec_ok:
|
||||
_search_backends.append("vec")
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
_search_backends = ["fts5"] + (["vec"] if _vec_ok else [])
|
||||
search.add_argument(
|
||||
"--backend", choices=_search_backends, default="fts5",
|
||||
help=(
|
||||
|
|
@ -4710,6 +4737,14 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
help="cap the number of chunks embedded (smoke-test on a real shard)",
|
||||
)
|
||||
embed_cmd.add_argument("--batch-size", type=int, default=256)
|
||||
embed_cmd.add_argument(
|
||||
"--rebuild", action="store_true",
|
||||
help=(
|
||||
"DROP + recreate chunk_vecs, then full re-embed — the clean "
|
||||
"VEC_BACKEND_VERSION-bump path (default is incremental: embed "
|
||||
"only chunks not already in chunk_vecs)"
|
||||
),
|
||||
)
|
||||
embed_cmd.set_defaults(func=_cmd_embed)
|
||||
|
||||
verify = sub.add_parser(
|
||||
|
|
|
|||
|
|
@ -173,23 +173,50 @@ def embed_documents(
|
|||
embedder: Embedder | None = None,
|
||||
limit: int | None = None,
|
||||
batch_size: int = 256,
|
||||
incremental: bool = True,
|
||||
rebuild: bool = False,
|
||||
progress: Callable[[int, int], None] | None = None,
|
||||
) -> int:
|
||||
"""Populate ``chunk_vecs`` for every ``chunks`` row with content.
|
||||
"""Populate ``chunk_vecs`` for ``chunks`` rows with content.
|
||||
|
||||
Cold-evicted chunks (``content IS NULL``) are skipped — there is
|
||||
nothing to embed and rehydration would re-trigger this. ``limit``
|
||||
caps the number of chunks processed (useful for a quick smoke
|
||||
test on a real shard). Returns the count embedded.
|
||||
Idempotency model (see #000039 — chunk content is immutable per
|
||||
``chunk_id``: a chunk, once embedded, never needs re-embedding
|
||||
unless the *embedder* changes):
|
||||
|
||||
Idempotent at the row level: ``INSERT OR REPLACE`` keyed on
|
||||
``chunk_id``, so re-running after a partial pass resumes cleanly.
|
||||
- ``incremental=True`` (default): embed only chunks **not already
|
||||
in chunk_vecs** (``chunk_id NOT IN (SELECT chunk_id FROM chunk_vecs)``).
|
||||
This is the after-ingest / cron / Prometheus-Σ-sweep path — it
|
||||
picks up exactly the newly-ingested chunks; re-running is cheap
|
||||
(no-op once everything's embedded).
|
||||
- ``incremental=False``: re-embed every chunk with content
|
||||
(delete-then-insert all). The embedder-changed case.
|
||||
- ``rebuild=True``: DROP + recreate ``chunk_vecs`` first, then a
|
||||
full pass — the clean ``VEC_BACKEND_VERSION`` bump (so a search
|
||||
mid-rebuild never mixes old- and new-model embeddings: it's all
|
||||
new-model from the recreated table, growing as the pass runs).
|
||||
Implies a full (non-incremental) pass.
|
||||
|
||||
Cold-evicted chunks (``content IS NULL``) are skipped — the vec
|
||||
row, if it exists, stays valid (content is the same on rehydrate).
|
||||
``limit`` caps chunks processed (smoke-test knob). Returns the
|
||||
count newly embedded this call.
|
||||
"""
|
||||
embedder = embedder or default_embedder()
|
||||
if rebuild:
|
||||
try:
|
||||
load_vec_extension(conn)
|
||||
conn.execute("DROP TABLE IF EXISTS chunk_vecs")
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
incremental = False # recreated table is empty → full pass anyway
|
||||
ensure_chunk_vecs_table(conn)
|
||||
|
||||
where = "content IS NOT NULL"
|
||||
if incremental:
|
||||
where += " AND chunk_id NOT IN (SELECT chunk_id FROM chunk_vecs)"
|
||||
|
||||
total_known = conn.execute(
|
||||
"SELECT COUNT(*) FROM chunks WHERE content IS NOT NULL"
|
||||
f"SELECT COUNT(*) FROM chunks WHERE {where}"
|
||||
).fetchone()[0]
|
||||
if limit is not None:
|
||||
total_known = min(total_known, limit)
|
||||
|
|
@ -207,7 +234,8 @@ def embed_documents(
|
|||
try:
|
||||
# vec0 virtual tables don't honor INSERT OR REPLACE — re-inserting
|
||||
# an existing chunk_id is a hard UNIQUE error. Delete-then-insert
|
||||
# gives idempotent re-runs and a content-changed → re-embed path.
|
||||
# is the upsert; harmless no-op on rows that aren't present yet
|
||||
# (the incremental path) and the re-embed path otherwise.
|
||||
conn.executemany(
|
||||
"DELETE FROM chunk_vecs WHERE chunk_id = ?",
|
||||
[(cid,) for cid in batch_ids],
|
||||
|
|
@ -227,8 +255,7 @@ def embed_documents(
|
|||
batch_texts.clear()
|
||||
|
||||
rows = conn.execute(
|
||||
"SELECT chunk_id, content FROM chunks WHERE content IS NOT NULL "
|
||||
"ORDER BY chunk_id"
|
||||
f"SELECT chunk_id, content FROM chunks WHERE {where} ORDER BY chunk_id"
|
||||
+ (f" LIMIT {int(limit)}" if limit is not None else "")
|
||||
)
|
||||
for row in rows:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Ticket #000039 — Optional `sqlite-vec` retrieval backend (A/B vs FTS5, hybrid not replacement)
|
||||
|
||||
**Status:** in progress · Phase 0 (doc) + **Phase 1 landed 2026-05-11**. Phase 1: `arborist/search/vec.py` — `VecBackend(SearchBackend)` (UNGROUNDED hits, never in proof path), `chunk_vecs` vec0 virtual table + `vec_meta` (sibling tables — don't touch chunks/documents/audit chain), `embed_documents()` ingest (delete-then-insert idempotent; vec0 doesn't honor INSERT-OR-REPLACE), pluggable `Embedder` callable with a fastembed `bge-small-en-v1.5` default. CLI: `arborist embed [--limit] [--batch-size]` + `arborist search --backend vec`. `[vec]` optional extra (sqlite-vec + fastembed). **Obvious v1 tuning** (`VEC_BACKEND_VERSION = vec-v1-bge-small-en-v1.5-384float32-cosine-flat`): embedder `BAAI/bge-small-en-v1.5`, dim 384, quant float32 (int8/binary = the production storage knob per §3.1, not wired in v1), metric cosine (bge outputs L2-normalized so cosine ≡ L2 ranking), ANN flat (vec0 default), top_k 20. 7 tests (`tests/test_search_vec.py`, stub embedder — plumbing only; semantic quality demonstrated on a real shard). **Demonstrated on `crawl_appliedcombinatorics_org.db`** (168 chunks embedded in ~37s incl. model load; semantic queries return topically-correct hits — "how many ways to choose k things from n" → top hit "AC Combinations"; chain-check on that shard reports 0 after embedding). The 5 hyperparams fold into `governance_policy_hash` in a later phase (§6 — not wired yet). **Phase 2** (RRF hybrid fusion in `query.py`) gated on a ≥5pp recall-lift measurement on bench fixtures with no STRICT-rate regression (§8).
|
||||
**Status:** in progress · Phase 0 (doc) + **Phase 1 landed 2026-05-11**. Phase 1: `arborist/search/vec.py` — `VecBackend(SearchBackend)` (UNGROUNDED hits, never in proof path), `chunk_vecs` vec0 virtual table + `vec_meta` (sibling tables — don't touch chunks/documents/audit chain), `embed_documents()` ingest (delete-then-insert idempotent; vec0 doesn't honor INSERT-OR-REPLACE), pluggable `Embedder` callable with a fastembed `bge-small-en-v1.5` default. CLI: `arborist embed [--limit] [--batch-size]` + `arborist search --backend vec`. `[vec]` optional extra (sqlite-vec + fastembed). **Obvious v1 tuning** (`VEC_BACKEND_VERSION = vec-v1-bge-small-en-v1.5-384float32-cosine-flat`): embedder `BAAI/bge-small-en-v1.5`, dim 384, quant float32 (int8/binary = the production storage knob per §3.1, not wired in v1), metric cosine (bge outputs L2-normalized so cosine ≡ L2 ranking), ANN flat (vec0 default), top_k 20. 7 tests (`tests/test_search_vec.py`, stub embedder — plumbing only; semantic quality demonstrated on a real shard). **Demonstrated on `crawl_appliedcombinatorics_org.db`** (168 chunks embedded in ~37s incl. model load; semantic queries return topically-correct hits — "how many ways to choose k things from n" → top hit "AC Combinations"; chain-check on that shard reports 0 after embedding). The 5 hyperparams fold into `governance_policy_hash` in a later phase (§6 — not wired yet). **Ingest integration landed 2026-05-11** (see §14): `embed_documents()` is now **incremental by default** (embeds only `chunk_id`s not already in `chunk_vecs` — re-runs are cheap no-ops); `arborist embed --rebuild` does the DROP+recreate+full-re-embed for a `VEC_BACKEND_VERSION` bump; `arborist ingest --embed` is the **eager opt-in** (embed this run's new chunks after the chunk+Merkle-commit pass; default ingest does NOT embed — the lazy `arborist embed` pass / cron / Prometheus-Σ unconscious sweep is the usual path). 9 vec tests. **Phase 2** (RRF hybrid fusion in `query.py`) gated on a ≥5pp recall-lift measurement on bench fixtures with no STRICT-rate regression (§8).
|
||||
**Opened:** 2026-05-09
|
||||
**Scope:** Spec an optional `sqlite-vec` backend that runs **alongside** the
|
||||
existing FTS5 retrieval pipeline (never replacing it), with phased gates
|
||||
|
|
@ -815,3 +815,76 @@ If fox rejects the dep footprint → fall back to the lighter
|
|||
`onnxruntime` path with a `bge-small` ONNX model (still ~33 MB
|
||||
weights but smaller runtime). Adds complexity but keeps the
|
||||
no-server constraint.
|
||||
|
||||
---
|
||||
|
||||
## 14. Ingest integration & idempotency (landed 2026-05-11)
|
||||
|
||||
The load-bearing fact: **a `chunk_id`'s content is immutable in
|
||||
arborist.** A chunk is content-addressed-ish (`leaf_hash`) — same
|
||||
content → same `chunk_id`; different content → a *new* `chunk_id`
|
||||
(re-ingest with changed content makes a new `document_root` + new
|
||||
chunk_ids, linked by `supersedes`; a chunker-version bump re-chunks
|
||||
→ new chunk_ids). So **a chunk, once embedded, never needs
|
||||
re-embedding** — the only re-embed trigger is the *embedder*
|
||||
changing (`VEC_BACKEND_VERSION` bump).
|
||||
|
||||
### 14.1 Idempotency table
|
||||
|
||||
| Event | `chunk_vecs` effect |
|
||||
|---|---|
|
||||
| Re-ingest same content | doc_root unchanged, chunks no-op-insert → already embedded → no-op |
|
||||
| Re-ingest changed content (new doc_root, new chunk_ids) | old chunks + old vecs stay (lossless history); new chunk_ids embedded on next pass |
|
||||
| Chunker bump (`tok-512-v1 → v2`) | re-chunk → new chunk_ids → embedded next pass; old chunk_ids' vecs are orphan-ish but harmless (old chunks still exist) |
|
||||
| Cold eviction (`content = NULL`) | vec row persists — still valid; on rehydrate the same `chunk_id` returns with the same content |
|
||||
| Embedder bump (`VEC_BACKEND_VERSION` changes) | the *only* full-re-embed case — `arborist embed --rebuild` (DROP+recreate+full pass); the `vec_meta.backend_version` row records the populating version |
|
||||
| Superseded docs | old chunks stay embedded → searchable via vec — consistent with `chunks_fts` keeping superseded docs; the retrieval `supersedes`-filter applies equally |
|
||||
|
||||
### 14.2 Two integration models
|
||||
|
||||
- **Lazy / out-of-band (default).** `arborist ingest` does *not*
|
||||
embed. `arborist embed` (incremental — embeds only `chunk_id`s
|
||||
not in `chunk_vecs`) populates on demand; the natural ops home is
|
||||
a cron after the nightly ingest, or — better — a **Prometheus-Σ
|
||||
unconscious-sweep task** (#000037 §3.1 already walks "ingested-
|
||||
but-not-yet-probed" documents; embed-the-new-chunks is exactly
|
||||
that shape, so vec needn't carry its own cron). Ingest stays fast;
|
||||
embedding is a separable, resumable, optional layer; `chunk_vecs`
|
||||
trails ingest by ≤ one pass.
|
||||
- **Eager opt-in.** `arborist ingest --embed` — after the
|
||||
chunk+Merkle-commit pass, embed this run's new chunks (incremental,
|
||||
so it's just the delta). `chunk_vecs` stays in sync. Pays
|
||||
~10-20 ms/chunk CPU per ingest — fine for incremental ingests,
|
||||
not for a bulk wiki backfill.
|
||||
|
||||
(#000039 §5.1's "embed at ingest, never at query" is about *when not*
|
||||
to embed the query corpus — it doesn't mandate embedding inside the
|
||||
ingest call.)
|
||||
|
||||
### 14.3 Commands
|
||||
|
||||
| Command | Behavior |
|
||||
|---|---|
|
||||
| `arborist --db D embed` | incremental — embed `chunk_id`s not yet in `chunk_vecs` (cheap no-op once everything's embedded) |
|
||||
| `arborist --db D embed --rebuild` | DROP + recreate `chunk_vecs`, then full re-embed — the clean `VEC_BACKEND_VERSION`-bump path (a search mid-rebuild never mixes old/new-model embeddings: the recreated table starts empty and grows new-model) |
|
||||
| `arborist --db D embed --limit N` | cap chunks processed — smoke-test on a real shard |
|
||||
| `arborist ... ingest --source ... --embed` | eager: after ingest, incremental-embed this run's new chunks |
|
||||
| `arborist --db D search Q --backend vec` | semantic ANN; errors with an install/embed hint if `[vec]` missing or `chunk_vecs` empty |
|
||||
|
||||
### 14.4 Concurrency
|
||||
|
||||
WAL serializes writers → `arborist embed` and `arborist ingest` on
|
||||
the *same* shard take turns. In the `--shards-dir` layout each shard
|
||||
is its own file → embed-shard-0 and ingest-shard-1 don't contend.
|
||||
Embed-during-ingest is safe under the lazy model: embed sees a
|
||||
consistent per-transaction snapshot; ingest's new chunks get picked
|
||||
up on the next incremental embed pass.
|
||||
|
||||
### 14.5 Versioning
|
||||
|
||||
`vec_meta.backend_version` (written by `ensure_chunk_vecs_table`) is
|
||||
the per-shard discriminator. The 5 vec hyperparams fold into
|
||||
`governance_policy_hash` in a later phase (§6 — not wired yet); the
|
||||
8-dim `cache_key` invariant is untouched (vec config goes into the
|
||||
existing `governance_policy_hash` dim, not a new one). Same
|
||||
versioned-default discipline as `tok-512-v1` / `norm-v1` etc.
|
||||
|
|
|
|||
|
|
@ -154,6 +154,52 @@ def test_embed_limit_caps_count(db):
|
|||
def test_embed_is_idempotent(db):
|
||||
conn, _ = db
|
||||
embed_documents(conn, embedder=_stub_embedder)
|
||||
embed_documents(conn, embedder=_stub_embedder) # re-run; INSERT OR REPLACE
|
||||
embed_documents(conn, embedder=_stub_embedder) # re-run
|
||||
cnt = conn.execute("SELECT COUNT(*) FROM chunk_vecs").fetchone()[0]
|
||||
assert cnt == 3
|
||||
|
||||
|
||||
def test_embed_incremental_only_embeds_new_chunks(db):
|
||||
"""Default (incremental=True): a second embed pass after ingesting a
|
||||
new doc embeds only the new chunk — re-running is cheap (#000039)."""
|
||||
conn, _ = db
|
||||
first = embed_documents(conn, embedder=_stub_embedder)
|
||||
assert first == 3
|
||||
# Simulate a follow-up ingest: one new document + chunk.
|
||||
new_root = "root_d" * 8
|
||||
conn.execute(
|
||||
"INSERT INTO documents("
|
||||
" document_root, document_uri, source_type, kind, title,"
|
||||
" chunking_version, canonicalization_version, schema_version, ingest_ts"
|
||||
") VALUES (?, 'uri://d', 'test', 'surface', 'Delta doc',"
|
||||
" 'tok-512-v1', 'norm-v1', 'v9.8.0', 0)",
|
||||
(new_root,),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO chunks(document_root, idx, leaf_hash, content, tier) "
|
||||
"VALUES (?, 0, ?, 'delta content about derangements', 'hot')",
|
||||
(new_root, hashlib.sha256(b"delta").hexdigest()),
|
||||
)
|
||||
second = embed_documents(conn, embedder=_stub_embedder) # incremental
|
||||
assert second == 1, "only the new chunk should embed on the second pass"
|
||||
assert conn.execute("SELECT COUNT(*) FROM chunk_vecs").fetchone()[0] == 4
|
||||
# A third pass with nothing new is a no-op.
|
||||
third = embed_documents(conn, embedder=_stub_embedder)
|
||||
assert third == 0
|
||||
|
||||
|
||||
def test_embed_rebuild_re_embeds_all(db):
|
||||
"""rebuild=True: DROP + recreate + full pass — every chunk re-embedded
|
||||
(the clean VEC_BACKEND_VERSION-bump path)."""
|
||||
conn, _ = db
|
||||
embed_documents(conn, embedder=_stub_embedder)
|
||||
assert conn.execute("SELECT COUNT(*) FROM chunk_vecs").fetchone()[0] == 3
|
||||
n = embed_documents(conn, embedder=_stub_embedder, rebuild=True)
|
||||
assert n == 3, "rebuild re-embeds all chunks"
|
||||
assert conn.execute("SELECT COUNT(*) FROM chunk_vecs").fetchone()[0] == 3
|
||||
# vec_meta still records the backend version after the recreate.
|
||||
from arborist.search.vec import VEC_BACKEND_VERSION
|
||||
row = conn.execute(
|
||||
"SELECT value FROM vec_meta WHERE key='backend_version'"
|
||||
).fetchone()
|
||||
assert row[0] == VEC_BACKEND_VERSION
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue