qa/corpus: wire fts_title (documents_fts) + fts_phrase (FTS5 phrase MATCH)
Completes the Corpus protocol surface. Both routes already worked at the
storage layer (every shard has chunks_fts AND documents_fts virtual
tables; FTS5 supports MATCH '"phrase"' natively); they just weren't
plumbed through. Now SqliteShardCorpus, MultiShardSqliteCorpus,
BucketClient, FtsSidecarShardClient, MultiShardSidecarCorpus, and
SidecarBucketCorpus all expose fts_title + fts_phrase end-to-end.
Bonus fix in arborist/ingest.py: every shard's INSERT INTO documents
now also INSERTs into documents_fts in the same transaction. Without
this, fts_title returned empty on freshly-ingested shards — only
migrated genesis shards had documents_fts populated. The cost is one
FTS5 row per new doc, negligible vs the chunk inserts.
Also includes operational cleanup (E):
- Deleted s3://arborist/clones/manifest-sidecar.json
- Deleted s3://arborist/clones/full-bench-64k/00[0-3].sidecar.bin
(~3.1 GB reclaimed; the slim FTS5 manifest is now the only cloud
surface)
- Fixed MultiShardSidecarCorpus.stats() — was calling .bucket on
FtsSidecarShardClient (attribute went away when SidecarShardClient
was deleted); now calls .stats() directly on whichever client.
Validation
----------
- Local fts_title("dinosaur"): "Dinosaur" main article ranks #1
- Local fts_phrase(["always been at war"]): "Nineteen Eighty-Four" ranks
#1 (the canonical phrase-route test from CLAUDE.md)
- Cloud (slim FTS5 sidecar) fts_title / fts_phrase: IDENTICAL ranking to
local on the same query (bit-for-bit FTS5 parity preserved)
- Full pytest suite: 2737 passed, 28 skipped, 1 xfailed (the same two
pre-existing failures from main HEAD)
This commit is contained in:
parent
bc3f94efeb
commit
18db3c2caf
4 changed files with 317 additions and 43 deletions
|
|
@ -159,40 +159,76 @@ def test_sqlite_fts_body_returns_hit_shape(shard_conn):
|
|||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Routes that should NotSupportedError
|
||||
# fts_title + fts_phrase routes (documents_fts + FTS5 phrase MATCH)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_sqlite_fts_title_not_supported(shard_conn):
|
||||
def test_sqlite_fts_title_via_documents_fts(shard_conn):
|
||||
"""fts_title returns title-bm25 ranked hits from documents_fts."""
|
||||
corpus = SqliteShardCorpus(shard_conn)
|
||||
with pytest.raises(NotSupportedError):
|
||||
corpus.fts_title("anarchism")
|
||||
hits = corpus.fts_title("anarchism", limit=5)
|
||||
# Should rank the Anarchism doc highly via title MATCH.
|
||||
assert any("Anarchism" in h.title for h in hits)
|
||||
|
||||
|
||||
def test_sqlite_fts_phrase_not_supported(shard_conn):
|
||||
def test_sqlite_fts_title_stopword_only_returns_empty(shard_conn):
|
||||
"""All-stopword queries reduce to empty FTS5 — short-circuit."""
|
||||
corpus = SqliteShardCorpus(shard_conn)
|
||||
with pytest.raises(NotSupportedError):
|
||||
corpus.fts_phrase(["anarchism is a"])
|
||||
assert corpus.fts_title("the of and") == []
|
||||
|
||||
|
||||
def test_sidecar_fts_title_not_supported():
|
||||
def test_sqlite_fts_phrase_matches_verbatim(shard_conn):
|
||||
"""FTS5 phrase MATCH '"phrase"' returns chunks containing the
|
||||
verbatim phrase. Use a phrase known to appear in the fixture doc."""
|
||||
corpus = SqliteShardCorpus(shard_conn)
|
||||
# The Anarchism fixture has "political philosophy" verbatim.
|
||||
hits = corpus.fts_phrase(["political philosophy"], limit=5)
|
||||
assert hits, "expected at least one hit for verbatim phrase"
|
||||
assert any("Anarchism" in h.title for h in hits)
|
||||
# Phrase tokens that DON'T appear together should return nothing.
|
||||
misses = corpus.fts_phrase(["stateless capital"], limit=5)
|
||||
assert misses == []
|
||||
|
||||
|
||||
def test_sqlite_fts_phrase_empty_input_returns_empty(shard_conn):
|
||||
corpus = SqliteShardCorpus(shard_conn)
|
||||
assert corpus.fts_phrase([]) == []
|
||||
assert corpus.fts_phrase(["", " "]) == []
|
||||
|
||||
|
||||
def test_sidecar_fts_title_delegates_to_multishard():
|
||||
"""SidecarBucketCorpus.fts_title delegates to the multi-shard
|
||||
fts_title_search method. Stub the multi-shard and confirm shape."""
|
||||
class _StubMulti:
|
||||
manifest = type("M", (), {"shards": [], "snapshot_root": None})()
|
||||
def fts_search(self, q, *, limit=8): return []
|
||||
def fts_title_search(self, q, *, limit=8): return [
|
||||
{"document_root": "abc", "document_uri": "test://x",
|
||||
"title": "X", "score": -12.0, "_shard_url": "s0"},
|
||||
]
|
||||
def fts_phrase_search(self, ngs, *, limit=8): return []
|
||||
def conn_for_shard(self, url): raise KeyError(url)
|
||||
corpus = SidecarBucketCorpus(_StubMulti())
|
||||
with pytest.raises(NotSupportedError):
|
||||
corpus.fts_title("anarchism")
|
||||
hits = corpus.fts_title("anything")
|
||||
assert len(hits) == 1
|
||||
assert hits[0].title == "X"
|
||||
assert hits[0].shard_id == "s0"
|
||||
|
||||
|
||||
def test_sidecar_fts_phrase_not_supported():
|
||||
def test_sidecar_fts_phrase_delegates_to_multishard():
|
||||
class _StubMulti:
|
||||
manifest = type("M", (), {"shards": [], "snapshot_root": None})()
|
||||
def fts_search(self, q, *, limit=8): return []
|
||||
def fts_title_search(self, q, *, limit=8): return []
|
||||
def fts_phrase_search(self, ngs, *, limit=8): return [
|
||||
{"document_root": "abc", "document_uri": "test://x",
|
||||
"title": "X", "score": -23.5},
|
||||
]
|
||||
def conn_for_shard(self, url): raise KeyError(url)
|
||||
corpus = SidecarBucketCorpus(_StubMulti())
|
||||
with pytest.raises(NotSupportedError):
|
||||
corpus.fts_phrase(["a b c"])
|
||||
hits = corpus.fts_phrase(["any phrase"])
|
||||
assert len(hits) == 1
|
||||
assert hits[0].score == -23.5
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue