russell@unturf.com
7f7eeefeb9
- make crawl-ingest writes to one central crawl db (CRAWL_DB, default ~/.arborist/crawl/web.db) instead of per-domain shards in the peer-shared main dir: keeps locally-crawled content out of peer sharing by default and a growing domain set under SQLite's 10-attach cap (Makefile, docs/crawler.md). - arborist query auto-includes the local crawl db (query() gains extra_shards; CLI --include-shard / --no-crawl-db, default-on when web.db exists). Fix latent --db single-file query AttributeError (cli.py). Persist used / used_pointer_ids + retrieval_purity into merkle_proof so read-only consumers can see which chunks fed the answer (qa/query.py). - arborist.read: read-only seam for dashboards / verifiers; on a multi-source context root surface the real primary source instead of the opaque corpus://multi-source sentinel (read.py). Backs the arborist-viz Merkle Command Center (#000069). - tests for extra_shards, the CLI crawl-db resolver, and the read seam.
1480 lines
52 KiB
Python
1480 lines
52 KiB
Python
"""Multi-source corpus Q&A: search → context → cache → Hermes.
|
|
|
|
Stub client only — no network. Validates:
|
|
- FTS5 finds the right docs across a small corpus
|
|
- context_root is deterministic (sorted source roots, then Merkle)
|
|
- cache hit on identical question returns same record without calling client
|
|
- different question -> different cache_key
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import Iterator
|
|
|
|
from arborist.document import Document
|
|
from arborist.ingest import ingest_source
|
|
from arborist.qa import query
|
|
from arborist.qa.client import StubClient
|
|
from arborist.source import Source
|
|
from arborist.store import connect
|
|
|
|
|
|
class FakeSource(Source):
|
|
source_type = "test"
|
|
|
|
def __init__(self, docs: list[Document]):
|
|
self.docs = docs
|
|
|
|
def iter_documents(self) -> Iterator[Document]:
|
|
yield from self.docs
|
|
|
|
|
|
def _doc(uri: str, content: str) -> Document:
|
|
return Document(uri=uri, content=content, source_type="test", title=uri.rsplit("/", 1)[-1])
|
|
|
|
|
|
# Three docs with distinguishable content so FTS5 can pick winners.
|
|
DOCS = [
|
|
_doc(
|
|
"test://anarchism",
|
|
"Anarchism is a political philosophy that opposes the state. " * 12
|
|
+ "Mutual aid is central to anarchist theory. " * 8,
|
|
),
|
|
_doc(
|
|
"test://capitalism",
|
|
"Capitalism is an economic system based on private ownership. " * 12
|
|
+ "Market exchange and capital accumulation drive growth. " * 8,
|
|
),
|
|
_doc(
|
|
"test://anarcho-capitalism",
|
|
"Anarcho-capitalism combines anarchism's opposition to the state with capitalism's private property. " * 12
|
|
+ "Murray Rothbard developed many of its core ideas. " * 8,
|
|
),
|
|
]
|
|
|
|
|
|
def test_query_picks_relevant_docs_and_writes_record(tmp_path):
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(DOCS))
|
|
finally:
|
|
conn.close()
|
|
|
|
# Verbatim quote from DOCS[2] → audit_mode=STRICT.
|
|
client = StubClient(
|
|
answer=(
|
|
'Per the source: '
|
|
'"Anarcho-capitalism combines anarchism\'s opposition to the state '
|
|
'with capitalism\'s private property"'
|
|
)
|
|
)
|
|
result = query(
|
|
question="What is anarcho-capitalism?",
|
|
qa_db=qa_db,
|
|
chat_client=client,
|
|
model_id="test-model",
|
|
single_db=main_db,
|
|
top_k=3,
|
|
)
|
|
|
|
assert result["status"] == "cache_miss_then_written"
|
|
assert result["audit_mode"] == "STRICT"
|
|
assert "Anarcho-capitalism" in result["answer_text"]
|
|
assert len(result["sources"]) >= 1
|
|
assert any("anarcho-capitalism" in s["document_uri"] for s in result["sources"])
|
|
|
|
# context_root is deterministic.
|
|
sorted_roots = sorted(s["document_root"] for s in result["sources"])
|
|
if len(sorted_roots) == 1:
|
|
assert result["context_root"] == sorted_roots[0]
|
|
# Repeat call with same question → cache hit, no LLM call.
|
|
n_calls_before = len(client.calls)
|
|
result2 = query(
|
|
question="What is anarcho-capitalism?",
|
|
qa_db=qa_db,
|
|
chat_client=client,
|
|
model_id="test-model",
|
|
single_db=main_db,
|
|
top_k=3,
|
|
)
|
|
assert result2["status"] == "cache_hit"
|
|
assert result2["cache_key"] == result["cache_key"]
|
|
assert len(client.calls) == n_calls_before # no new call
|
|
|
|
|
|
def test_query_different_question_different_cache(tmp_path):
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(DOCS))
|
|
finally:
|
|
conn.close()
|
|
|
|
client = StubClient(answer="answer text")
|
|
r1 = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db,
|
|
chat_client=client,
|
|
model_id="m",
|
|
single_db=main_db,
|
|
)
|
|
r2 = query(
|
|
question="What is capitalism?",
|
|
qa_db=qa_db,
|
|
chat_client=client,
|
|
model_id="m",
|
|
single_db=main_db,
|
|
)
|
|
assert r1["cache_key"] != r2["cache_key"]
|
|
assert len(client.calls) == 2 # both missed cache, both called client
|
|
|
|
|
|
def test_query_question_equivalence_class_dedups_cache(tmp_path):
|
|
"""Fox 2026-04-29 catch: 'who is batman?', 'who is batman', and
|
|
'who is the batman?' should all hit the same cache. Question_hash
|
|
canonicalizes correctly, but conversation_hash used to hash the
|
|
LITERAL question text in the user message — so each variant got
|
|
its own chash and missed cache. Fix: canonical_question form is
|
|
substituted into the messages list used for conversation_hash,
|
|
while the LLM still receives the verbatim question."""
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(DOCS))
|
|
finally:
|
|
conn.close()
|
|
|
|
captured: list = []
|
|
|
|
def _capture(messages, **kw):
|
|
captured.append(messages)
|
|
return "stub answer"
|
|
|
|
# First variant — populates cache.
|
|
r1 = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer=_capture),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
)
|
|
# Second & third variants — must hit the same cache_key.
|
|
r2 = query(
|
|
question="What is anarchism", # no trailing ?
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer=_capture),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
)
|
|
r3 = query(
|
|
question="what is the anarchism?", # leading article
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer=_capture),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
)
|
|
|
|
assert r1["cache_key"] == r2["cache_key"] == r3["cache_key"]
|
|
assert r1["status"] == "cache_miss_then_written"
|
|
assert r2["status"] == "cache_hit"
|
|
assert r3["status"] == "cache_hit"
|
|
# Only the first call reached the LLM.
|
|
assert len(captured) == 1
|
|
# And it received the verbatim question, not the canonical form.
|
|
user_text = "\n".join(
|
|
m["content"] for m in captured[0] if m["role"] == "user"
|
|
)
|
|
assert "What is anarchism?" in user_text
|
|
|
|
|
|
def test_query_strict_dedup_distinguishes_question_variants(tmp_path):
|
|
"""policy['question_dedup']='strict' makes every variant get its
|
|
own cache_key. 'Who is X?' and 'who is the X?' write separate
|
|
records under strict policy."""
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(DOCS))
|
|
finally:
|
|
conn.close()
|
|
|
|
strict_policy = dict(query.__globals__["DEFAULT_QUERY_POLICY"])
|
|
strict_policy["question_dedup"] = "strict"
|
|
|
|
r1 = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer="a"),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
policy=strict_policy,
|
|
fidelity="strict",
|
|
)
|
|
r2 = query(
|
|
question="what is anarchism",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer="b"),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
policy=strict_policy,
|
|
fidelity="strict",
|
|
)
|
|
# Different cache_keys; both populated independently.
|
|
assert r1["cache_key"] != r2["cache_key"]
|
|
assert r1["status"] == r2["status"] == "cache_miss_then_written"
|
|
assert r1["lookup_path"] == "miss"
|
|
|
|
|
|
def test_query_equivalence_class_fidelity_falls_back_across_dedup_modes(tmp_path):
|
|
"""A record written under equivalence_class policy gets reused by a
|
|
later strict-policy lookup that asks for fidelity='equivalence_class'.
|
|
Verifies the cross-silo fallback: strict ckey misses, alternate
|
|
equivalence_class ckey hits, lookup_path reports the fallback."""
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(DOCS))
|
|
finally:
|
|
conn.close()
|
|
|
|
DEFAULT = query.__globals__["DEFAULT_QUERY_POLICY"]
|
|
eq_policy = dict(DEFAULT)
|
|
eq_policy["question_dedup"] = "equivalence_class"
|
|
strict_policy = dict(DEFAULT)
|
|
strict_policy["question_dedup"] = "strict"
|
|
|
|
# Agent A writes equivalence-class record for 'what is anarchism?'.
|
|
captured: list = []
|
|
|
|
def _capture(messages, **kw):
|
|
captured.append(messages)
|
|
return "stub"
|
|
|
|
r1 = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer=_capture),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
policy=eq_policy,
|
|
)
|
|
assert r1["status"] == "cache_miss_then_written"
|
|
|
|
# Agent B (strict policy, equivalence_class fidelity) asks the same
|
|
# question. Strict ckey misses; the eq_class fallback hits A's record.
|
|
r2 = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer=_capture),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
policy=strict_policy,
|
|
fidelity="equivalence_class",
|
|
)
|
|
assert r2["status"] == "cache_hit"
|
|
assert r2["lookup_path"] == "equivalence_class_fallback"
|
|
assert len(captured) == 1 # only A's call reached the LLM
|
|
|
|
|
|
def test_query_strict_fidelity_does_not_fall_back(tmp_path):
|
|
"""Audit-grade lookup: strict-fidelity refuses to read records from
|
|
the other dedup mode's silo. Even if equivalence_class has a hit,
|
|
strict fidelity reports cache miss."""
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(DOCS))
|
|
finally:
|
|
conn.close()
|
|
|
|
DEFAULT = query.__globals__["DEFAULT_QUERY_POLICY"]
|
|
eq_policy = dict(DEFAULT)
|
|
eq_policy["question_dedup"] = "equivalence_class"
|
|
strict_policy = dict(DEFAULT)
|
|
strict_policy["question_dedup"] = "strict"
|
|
|
|
# Agent A writes under equivalence_class.
|
|
query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer="a"),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
policy=eq_policy,
|
|
)
|
|
|
|
# Agent B (strict policy + strict fidelity) asks same question.
|
|
# Should NOT find A's record; runs LLM fresh.
|
|
captured: list = []
|
|
|
|
def _capture(messages, **kw):
|
|
captured.append(messages)
|
|
return "stub"
|
|
|
|
r2 = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer=_capture),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
policy=strict_policy,
|
|
fidelity="strict",
|
|
)
|
|
assert r2["status"] == "cache_miss_then_written"
|
|
assert r2["lookup_path"] == "miss"
|
|
assert len(captured) == 1
|
|
|
|
|
|
def test_classify_source_role_separates_primary_from_noisy():
|
|
"""Direct unit test on the role classifier. JP film-score should be
|
|
noisy_background; JP (film) should be primary; JP franchise should
|
|
be secondary; The Lost World should be sequel; off-topic background.
|
|
Catches the case where peripheral pages with strong title overlap
|
|
used to share the primary slot."""
|
|
from arborist.qa.query import _classify_source_role
|
|
|
|
qstems = {"dinosaur", "jurassic", "park", "film"}
|
|
assert _classify_source_role("Jurassic Park (film)", qstems) == "primary_answer_source"
|
|
assert _classify_source_role("Jurassic Park (film score)", qstems) == "noisy_background_source"
|
|
assert _classify_source_role("Jurassic Park video games", qstems) == "noisy_background_source"
|
|
assert _classify_source_role("Jurassic Park (franchise)", qstems) == "secondary_context_source"
|
|
assert _classify_source_role("List of Jurassic Park characters", qstems) == "secondary_context_source"
|
|
assert _classify_source_role("The Lost World: Jurassic Park", qstems) == "sequel_background_source"
|
|
# Off-topic title (no shared stems): falls through to background.
|
|
assert _classify_source_role("Anarchism", qstems) == "background_source"
|
|
|
|
|
|
def test_query_role_weighted_budget_persists_role_on_sources(tmp_path):
|
|
"""Each source in the providence record's merkle_proof.sources gains
|
|
a `source_role` field — verifies the role made it into the audit
|
|
trail so an inspector can see which slot a source occupied."""
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(DOCS))
|
|
finally:
|
|
conn.close()
|
|
result = query(
|
|
question="What is anarcho-capitalism?",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer="x"),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=3,
|
|
)
|
|
assert all("source_role" in s for s in result["sources"])
|
|
|
|
|
|
def test_query_persists_run_dag_root_on_record_and_result(tmp_path):
|
|
"""Per-run Merkle-DAG fingerprint surfaces on both the result dict
|
|
& the persisted providence_cache row. Recomputing the root from
|
|
the persisted blob matches what was stored."""
|
|
from arborist.qa.dag import verify_run_dag
|
|
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(DOCS))
|
|
finally:
|
|
conn.close()
|
|
|
|
result = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer='The source: "a political philosophy that opposes the state"'),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
)
|
|
assert "run_dag_root" in result
|
|
assert isinstance(result["run_dag_root"], str)
|
|
assert len(result["run_dag_root"]) == 64 # sha256 hex
|
|
|
|
qa_conn = connect(qa_db)
|
|
try:
|
|
row = qa_conn.execute(
|
|
"SELECT run_dag_root, run_dag_blob FROM providence_cache "
|
|
"WHERE cache_key = ?",
|
|
(result["cache_key"],),
|
|
).fetchone()
|
|
finally:
|
|
qa_conn.close()
|
|
assert row["run_dag_root"] == result["run_dag_root"]
|
|
assert verify_run_dag(row["run_dag_blob"]) is True
|
|
|
|
|
|
def test_query_no_sources_when_empty_corpus(tmp_path):
|
|
main_db = tmp_path / "empty.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
connect(main_db).close() # creates schema, no docs
|
|
result = query(
|
|
question="What is anything?",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
)
|
|
assert result["status"] == "no_sources"
|
|
|
|
|
|
def test_query_filter_requires_breadth_for_multi_token_queries(tmp_path):
|
|
"""Fox 2026-04-29 catch: 'supermans girlfriend' returned 7-of-8
|
|
unrelated `Girlfriends`-titled articles because title-overlap
|
|
accepted ANY single-token match. The fix tightens both title and
|
|
body filters to require ALL query tokens (≤2-token queries) so a
|
|
doc whose title only matches ONE of the two qtokens doesn't pass.
|
|
|
|
Synthetic corpus pins the new behavior:
|
|
|
|
- "Lois Lane" — neither qtoken in title; body has both → keep
|
|
- "Girlfriends" — only "girlfriend" in title/body; no "superman" → drop
|
|
- "Superman album" — "superman" in title; body lacks "girlfriend" → drop
|
|
"""
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
docs = [
|
|
_doc(
|
|
"test://lois-lane",
|
|
"Lois Lane is a fictional character who works for the Daily Planet. "
|
|
"She is Superman's girlfriend and frequently appears in his stories. "
|
|
* 5,
|
|
),
|
|
_doc(
|
|
"test://girlfriends-tv",
|
|
"Girlfriends is a sitcom about four women in Los Angeles. " * 10,
|
|
),
|
|
_doc(
|
|
"test://superman-music",
|
|
"Superman is an album of rock music recorded in Tokyo. " * 10,
|
|
),
|
|
]
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(docs))
|
|
finally:
|
|
conn.close()
|
|
|
|
result = query(
|
|
question="who is supermans girlfriend?",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer="x"),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=8,
|
|
)
|
|
src_uris = [s["document_uri"] for s in result["sources"]]
|
|
# Lois Lane MUST be in the result set — it's the only doc with both
|
|
# query tokens in its body.
|
|
assert any("lois-lane" in u for u in src_uris), (
|
|
f"breadth filter regressed: lois-lane not in {src_uris}"
|
|
)
|
|
|
|
|
|
def test_query_filter_one_token_query_still_synonym_expands(tmp_path):
|
|
"""1-token queries keep the loose synonym-expanded any-match — pin
|
|
that we didn't over-tighten the single-token case."""
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
docs = [
|
|
_doc("test://anarchism", "Anarchism is a political philosophy. " * 30),
|
|
]
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(docs))
|
|
finally:
|
|
conn.close()
|
|
result = query(
|
|
question="anarchism?",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer="x"),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=3,
|
|
)
|
|
assert len(result["sources"]) == 1
|
|
|
|
|
|
def test_query_burn_existing_forces_fresh_inference(tmp_path):
|
|
"""Fox 2026-04-29: `make query Q=... BURN=1` busts any matching live
|
|
cache record before lookup so a fresh inference runs.
|
|
|
|
Sequence:
|
|
1. First query → cache_miss_then_written, populates cache
|
|
2. Second query, NO burn → cache_hit (no new LLM call)
|
|
3. Third query, BURN=True → cache_miss_then_written (cache busted),
|
|
result reports burned_existing=1
|
|
"""
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(DOCS))
|
|
finally:
|
|
conn.close()
|
|
|
|
client = StubClient(answer="anarchism is a thing")
|
|
|
|
r1 = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db,
|
|
chat_client=client,
|
|
model_id="m",
|
|
single_db=main_db,
|
|
)
|
|
assert r1["status"] == "cache_miss_then_written"
|
|
assert r1["burned_existing"] == 0
|
|
n_calls_after_first = len(client.calls)
|
|
|
|
r2 = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db,
|
|
chat_client=client,
|
|
model_id="m",
|
|
single_db=main_db,
|
|
)
|
|
assert r2["status"] == "cache_hit"
|
|
assert r2["burned_existing"] == 0
|
|
assert len(client.calls) == n_calls_after_first # no new LLM call
|
|
|
|
r3 = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db,
|
|
chat_client=client,
|
|
model_id="m",
|
|
single_db=main_db,
|
|
burn_existing=True,
|
|
)
|
|
assert r3["status"] == "cache_miss_then_written"
|
|
assert r3["burned_existing"] == 1
|
|
assert len(client.calls) == n_calls_after_first + 1 # new LLM call after burn
|
|
|
|
|
|
def test_query_burn_existing_writes_audit_event(tmp_path):
|
|
"""Each --burn writes a providence_burn audit event so the chain
|
|
records the bust. Verifies one event lands per burn."""
|
|
from arborist.store import connect as _connect
|
|
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(DOCS))
|
|
finally:
|
|
conn.close()
|
|
|
|
client = StubClient(answer="x")
|
|
query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db, chat_client=client, model_id="m", single_db=main_db,
|
|
)
|
|
qc = _connect(qa_db)
|
|
try:
|
|
burns_before = qc.execute(
|
|
"SELECT COUNT(*) FROM audit_events WHERE event_type='providence_burn'"
|
|
).fetchone()[0]
|
|
finally:
|
|
qc.close()
|
|
|
|
query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db, chat_client=client, model_id="m", single_db=main_db,
|
|
burn_existing=True,
|
|
)
|
|
qc = _connect(qa_db)
|
|
try:
|
|
burns_after = qc.execute(
|
|
"SELECT COUNT(*) FROM audit_events WHERE event_type='providence_burn'"
|
|
).fetchone()[0]
|
|
finally:
|
|
qc.close()
|
|
assert burns_after == burns_before + 1
|
|
|
|
|
|
def test_query_burn_existing_with_no_prior_record_is_noop(tmp_path):
|
|
"""First-time query with --burn: nothing to burn → burned_existing=0,
|
|
proceeds to fresh inference normally."""
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(DOCS))
|
|
finally:
|
|
conn.close()
|
|
r = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db, chat_client=StubClient(answer="x"),
|
|
model_id="m", single_db=main_db,
|
|
burn_existing=True,
|
|
)
|
|
assert r["status"] == "cache_miss_then_written"
|
|
assert r["burned_existing"] == 0
|
|
|
|
|
|
def test_query_per_source_cap_prevents_huge_doc_monopoly(tmp_path):
|
|
"""Fox 2026-04-29 catch: a top-ranked huge document (e.g.
|
|
List_of_Batman_comics, 80 KB+ bibliography) used to consume the
|
|
entire 60 KB budget at hit #1, dropping every subsequent doc with
|
|
char_budget <= 0. Now each of the top_k hits gets at most
|
|
`max_context_chars / top_k` chars; multiple sources land in
|
|
context even when hit #1 is huge."""
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
|
|
# Hit #1 is intentionally huge: every query token AND the most copies
|
|
# so it FTS5-ranks first. Hits #2 and #3 are smaller but still
|
|
# contain the query token.
|
|
bulk_token = "anarchism " * 5000 # ~50 KB after canonicalize
|
|
docs = [
|
|
_doc("test://huge-bibliography", bulk_token),
|
|
_doc("test://anarchism-bio", "Anarchism is a political philosophy. " * 30),
|
|
_doc("test://anarchism-history", "Anarchism's history begins with Proudhon. " * 30),
|
|
]
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(docs))
|
|
finally:
|
|
conn.close()
|
|
|
|
captured = {"messages": None}
|
|
|
|
def _capture(messages, **kw):
|
|
captured["messages"] = messages
|
|
return "stub"
|
|
|
|
result = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer=_capture),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=3,
|
|
max_context_chars=60000,
|
|
)
|
|
# All three sources should be in the result, not just the huge one.
|
|
assert len(result["sources"]) >= 2, (
|
|
f"per-source cap broke: only {len(result['sources'])} sources reached "
|
|
f"context (huge doc monopolized again)"
|
|
)
|
|
# Verify the user-turn context contains content from the smaller docs
|
|
# too — not just a 60K slab of the bulk doc.
|
|
user_text = "\n".join(m["content"] for m in captured["messages"] if m["role"] == "user")
|
|
assert "anarchism-bio" in user_text or "anarchism-history" in user_text
|
|
|
|
|
|
def test_query_per_source_cap_respects_top_k(tmp_path):
|
|
"""top_k=1 → cap = max_context_chars (legacy behavior preserved when
|
|
operator wants a single large source)."""
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
big = _doc("test://big", "Anarchism is a political philosophy. " * 2000)
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource([big]))
|
|
finally:
|
|
conn.close()
|
|
result = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer="x"),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=1,
|
|
max_context_chars=60000,
|
|
)
|
|
# Single source allowed up to full budget.
|
|
assert len(result["sources"]) == 1
|
|
|
|
|
|
def test_query_persists_audit_event(tmp_path):
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(DOCS))
|
|
finally:
|
|
conn.close()
|
|
query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer="X"),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
)
|
|
# qa.db should now have one providence_query event in its audit chain.
|
|
qc = connect(qa_db)
|
|
try:
|
|
events = qc.execute(
|
|
"SELECT event_type FROM audit_events ORDER BY seq"
|
|
).fetchall()
|
|
finally:
|
|
qc.close()
|
|
types = [e["event_type"] for e in events]
|
|
assert "providence_query" in types
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# capacity metrics — prompt_chars + answer_chars surface
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _setup_corpus(tmp_path):
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(DOCS))
|
|
finally:
|
|
conn.close()
|
|
return main_db, qa_db
|
|
|
|
|
|
def test_query_returns_prompt_chars_breakdown(tmp_path):
|
|
"""prompt_chars must contain the five expected keys and
|
|
messages_total must equal the sum of system + reminder + user
|
|
message lengths actually sent to the chat client."""
|
|
main_db, qa_db = _setup_corpus(tmp_path)
|
|
client = StubClient(answer="A short answer.")
|
|
result = query(
|
|
question="What is anarcho-capitalism?",
|
|
qa_db=qa_db,
|
|
chat_client=client,
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=3,
|
|
)
|
|
|
|
pc = result["prompt_chars"]
|
|
assert set(pc.keys()) == {
|
|
"system_prompt",
|
|
"grounding_reminder",
|
|
"user_question",
|
|
"evidence_or_context",
|
|
"messages_total",
|
|
}
|
|
# System prompt is from DEFAULT_QUERY_POLICY — non-empty.
|
|
assert pc["system_prompt"] > 0
|
|
# Question is exactly the chars passed in.
|
|
assert pc["user_question"] == len("What is anarcho-capitalism?")
|
|
# Evidence/context is the chunk text the model saw — non-trivial.
|
|
assert pc["evidence_or_context"] > 0
|
|
# messages_total equals the sum of message contents the stub saw.
|
|
sent = client.calls[0]["messages"]
|
|
actual_total = sum(len(m["content"]) for m in sent)
|
|
assert pc["messages_total"] == actual_total
|
|
|
|
|
|
def test_query_answer_chars_matches_answer_text(tmp_path):
|
|
main_db, qa_db = _setup_corpus(tmp_path)
|
|
client = StubClient(answer="Exactly twenty-six chars!!")
|
|
result = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db,
|
|
chat_client=client,
|
|
model_id="m",
|
|
single_db=main_db,
|
|
)
|
|
assert result["answer_chars"] == len(result["answer_text"])
|
|
|
|
|
|
def test_query_cache_hit_also_returns_capacity_metrics(tmp_path):
|
|
"""Cache-hit path must populate prompt_chars too — operators
|
|
inspecting cached records still want the capacity breakdown."""
|
|
main_db, qa_db = _setup_corpus(tmp_path)
|
|
client = StubClient(answer="Same answer twice.")
|
|
first = query(
|
|
question="What is anarcho-capitalism?",
|
|
qa_db=qa_db,
|
|
chat_client=client,
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=3,
|
|
)
|
|
assert first["status"] == "cache_miss_then_written"
|
|
second = query(
|
|
question="What is anarcho-capitalism?",
|
|
qa_db=qa_db,
|
|
chat_client=client,
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=3,
|
|
)
|
|
assert second["status"] == "cache_hit"
|
|
assert "prompt_chars" in second
|
|
assert second["prompt_chars"]["messages_total"] > 0
|
|
assert second["answer_chars"] > 0
|
|
|
|
|
|
def test_query_evidence_chars_grows_with_topk(tmp_path):
|
|
"""Adding more sources to the context budget should increase
|
|
`evidence_or_context`. Sanity check that the metric tracks
|
|
actual context build, not a stale constant."""
|
|
main_db, qa_db = _setup_corpus(tmp_path)
|
|
|
|
qa_db_small = tmp_path / "qa_small.db"
|
|
qa_db_big = tmp_path / "qa_big.db"
|
|
|
|
small = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db_small,
|
|
chat_client=StubClient(answer="X"),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=1,
|
|
max_context_chars=2000,
|
|
)
|
|
big = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db_big,
|
|
chat_client=StubClient(answer="X"),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=3,
|
|
max_context_chars=20000,
|
|
)
|
|
assert big["prompt_chars"]["evidence_or_context"] >= small["prompt_chars"]["evidence_or_context"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# --retrieval-keywords flag (operator-supplied retrieval augmentation)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_retrieval_keywords_does_not_alter_question_to_llm(tmp_path):
|
|
"""The LLM must see the original question string, not the
|
|
augmented retrieval query. Cache_key must match the same call
|
|
without retrieval_keywords."""
|
|
main_db, qa_db = _setup_corpus(tmp_path)
|
|
|
|
captured = {}
|
|
|
|
def _record_answer(messages, **kwargs):
|
|
captured["messages"] = messages
|
|
return "Some answer."
|
|
|
|
client = StubClient(answer=_record_answer)
|
|
bare = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db,
|
|
chat_client=client,
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=3,
|
|
)
|
|
|
|
# Same question, with retrieval keywords. New qa_db so cache lookup
|
|
# under fresh state — both runs are cache-miss.
|
|
qa_db_2 = tmp_path / "qa_with_keywords.db"
|
|
client2 = StubClient(answer=_record_answer)
|
|
augmented = query(
|
|
question="What is anarchism?",
|
|
qa_db=qa_db_2,
|
|
chat_client=client2,
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=3,
|
|
retrieval_keywords="mutual aid theory",
|
|
)
|
|
|
|
# User question text is byte-identical between the two runs.
|
|
bare_user_msg = [m["content"] for m in client.calls[0]["messages"]
|
|
if m["role"] == "user"][-1]
|
|
aug_user_msg = [m["content"] for m in client2.calls[0]["messages"]
|
|
if m["role"] == "user"][-1]
|
|
# Both end with "Question: What is anarchism?" — the keywords
|
|
# never appear inside the LLM-facing question segment (they
|
|
# only flow into FTS5 retrieval + title-filter token sets).
|
|
assert "What is anarchism?" in bare_user_msg
|
|
assert "What is anarchism?" in aug_user_msg
|
|
assert "mutual aid theory" not in aug_user_msg.split("Question:")[-1]
|
|
# System prompt unchanged across runs.
|
|
bare_sys = [m["content"] for m in client.calls[0]["messages"]
|
|
if m["role"] == "system"][0]
|
|
aug_sys = [m["content"] for m in client2.calls[0]["messages"]
|
|
if m["role"] == "system"][0]
|
|
assert bare_sys == aug_sys
|
|
|
|
|
|
def test_retrieval_keywords_changes_retrieved_sources(tmp_path):
|
|
"""Different keywords surface different docs, even with the
|
|
same question text. Same-keywords across runs is reproducible."""
|
|
main_db, qa_db = _setup_corpus(tmp_path)
|
|
qa_db_a = tmp_path / "a.db"
|
|
qa_db_b = tmp_path / "b.db"
|
|
a = query(
|
|
question="define the system",
|
|
qa_db=qa_db_a,
|
|
chat_client=StubClient(answer="X"),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=3,
|
|
retrieval_keywords="anarcho-capitalism rothbard",
|
|
)
|
|
b = query(
|
|
question="define the system",
|
|
qa_db=qa_db_b,
|
|
chat_client=StubClient(answer="X"),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=3,
|
|
retrieval_keywords="capitalism markets",
|
|
)
|
|
a_uris = sorted(s["document_uri"] for s in a["sources"])
|
|
b_uris = sorted(s["document_uri"] for s in b["sources"])
|
|
# The two retrieval-keyword sets pull different topical sources.
|
|
# (Sanity: same question without keywords is the no-keyword
|
|
# baseline; the assertion is that adding distinct keywords does
|
|
# something to the result.)
|
|
assert a_uris != b_uris or set(a_uris) != set(b_uris), (
|
|
f"keywords should affect retrieval; got identical sources {a_uris}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# phrase-pattern retrieval route (allusion / verbatim sequence boost)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_question_phrases_returns_sliding_n_grams_no_stopword_strip():
|
|
"""`_question_phrases` extracts verbatim n-token windows. Function
|
|
words are kept — diagnostic value of an allusion is the EXACT
|
|
sequence ('always been at war' >> 'always war')."""
|
|
from arborist.qa.query import _question_phrases
|
|
out = _question_phrases("has oceania always been at war with east asia", n=4)
|
|
# 9-token query, 4-gram window → 6 phrases, all preserved verbatim
|
|
# (lowercase) and deduped.
|
|
assert "has oceania always been" in out
|
|
assert "always been at war" in out
|
|
assert "war with east asia" in out
|
|
# Stopwords ARE present — that's the design, not a bug.
|
|
assert any("at" in p.split() for p in out)
|
|
|
|
|
|
def test_question_phrases_n_5_yields_five_token_phrases():
|
|
"""5-grams trade recall for precision; 'oceania always been at war'
|
|
is a much stronger Orwell signal than 'always been at war' alone."""
|
|
from arborist.qa.query import _question_phrases
|
|
out = _question_phrases("has oceania always been at war with east asia", n=5)
|
|
assert "oceania always been at war" in out
|
|
assert "always been at war with" in out
|
|
# Too short for 6-grams of just "war with east asia" alone.
|
|
assert all(len(p.split()) == 5 for p in out)
|
|
|
|
|
|
def test_question_phrases_skips_when_question_shorter_than_n():
|
|
"""`who is X?` is too short to yield 4-grams. Empty output is the
|
|
expected behavior (the body BM25 + title routes still cover it)."""
|
|
from arborist.qa.query import _question_phrases
|
|
assert _question_phrases("who is X?", n=4) == []
|
|
assert _question_phrases("", n=4) == []
|
|
|
|
|
|
def test_question_phrases_drops_all_short_token_phrases():
|
|
"""A window of all 1-3 char tokens is boilerplate ('to be or not')
|
|
— drops to avoid over-matching. The skip rule fires only when ALL
|
|
tokens in the window are <4 chars."""
|
|
from arborist.qa.query import _question_phrases
|
|
# All ≤3-char tokens — drop.
|
|
assert _question_phrases("to be or not", n=4) == []
|
|
# Mixed: at least one ≥4-char token → keep.
|
|
out = _question_phrases("to be or maybe", n=4)
|
|
assert out == ["to be or maybe"]
|
|
|
|
|
|
def test_question_phrases_lowercases_and_dedupes():
|
|
"""Output is lowercase, deduped on string equality. Same
|
|
sequence in different cases collapses to one phrase."""
|
|
from arborist.qa.query import _question_phrases
|
|
out = _question_phrases("Always been at war Always been at war", n=4)
|
|
# Repeated sequence appears only once in the output.
|
|
assert out.count("always been at war") == 1
|
|
|
|
|
|
def test_search_phrases_returns_empty_on_no_phrases(tmp_path):
|
|
"""Defensive: empty phrase list yields no rows, no exceptions."""
|
|
from arborist.qa.query import _search_phrases
|
|
main_db = tmp_path / "corpus.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(DOCS))
|
|
rows = _search_phrases(conn, [], 10)
|
|
finally:
|
|
conn.close()
|
|
assert rows == []
|
|
|
|
|
|
def test_search_phrases_skips_phrases_with_double_quotes(tmp_path):
|
|
"""Adversarial input safety: phrases containing `"` would break
|
|
the FTS5 quoted-phrase syntax. The function silently drops them."""
|
|
from arborist.qa.query import _search_phrases
|
|
main_db = tmp_path / "corpus.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(DOCS))
|
|
# All phrases contain quotes — function returns empty.
|
|
rows = _search_phrases(conn, ['has "embedded" quote', 'also "bad"'], 10)
|
|
finally:
|
|
conn.close()
|
|
assert rows == []
|
|
|
|
|
|
def test_phrase_match_surfaces_topical_doc(tmp_path):
|
|
"""End-to-end: a query whose phrase verbatim-matches one doc's
|
|
body should surface that doc even when title tokens don't
|
|
overlap. Closes the 2026-05-01 Orwell case where the 1984
|
|
article had zero token overlap with the question's title-tokens
|
|
but matched the verbatim phrase 'always been at war'."""
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
# Two docs: one explicitly contains the diagnostic phrase
|
|
# but its title doesn't overlap question tokens; the other
|
|
# is a generic geography article.
|
|
ingest_source(conn, FakeSource([
|
|
_doc(
|
|
"test://orwell-stub",
|
|
# Title-irrelevant to the question; body contains
|
|
# the diagnostic 5-gram.
|
|
"The novel narrates that Oceania always been at war with "
|
|
"Eastasia though the alliances had previously rotated. " * 6
|
|
),
|
|
_doc(
|
|
"test://geography-stub",
|
|
"Geographic descriptions of regions called Oceania and East "
|
|
"Asia. " * 12,
|
|
),
|
|
]))
|
|
finally:
|
|
conn.close()
|
|
|
|
result = query(
|
|
question="has oceania always been at war with east asia",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer="An answer."),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=5,
|
|
)
|
|
uris = [s["document_uri"] for s in result["sources"]]
|
|
# Both docs surface; the phrase-route ensures the orwell-stub
|
|
# doc isn't filtered out by the title-relevance gate.
|
|
assert "test://orwell-stub" in uris
|
|
|
|
|
|
def test_phrase_route_skipped_when_question_shorter_than_min_n():
|
|
"""False-positive guard: a 4-token geography question lacks enough
|
|
tokens to trigger the n=5/n=6 phrase route. Short conventional
|
|
queries route through body-BM25 + title-LIKE only — phrase routing
|
|
is structurally biased toward longer allusion-shape questions."""
|
|
from arborist.qa.query import _question_phrases
|
|
# 4 tokens after extraction → empty 5-gram and 6-gram outputs.
|
|
assert _question_phrases("oceania east asia geography", n=5) == []
|
|
assert _question_phrases("oceania east asia geography", n=6) == []
|
|
# 5 tokens → exactly one 5-gram, zero 6-grams.
|
|
out_5 = _question_phrases("oceania population east asia trade", n=5)
|
|
assert len(out_5) == 1
|
|
assert out_5[0] == "oceania population east asia trade"
|
|
out_6 = _question_phrases("oceania population east asia trade", n=6)
|
|
assert out_6 == []
|
|
|
|
|
|
def test_phrase_route_does_not_hijack_literal_geography_query(tmp_path):
|
|
"""Critical false-positive guard. A literal geography query about
|
|
Oceania + East Asia must NOT pull in an Orwell-flavored stub doc
|
|
just because both contain geographic tokens. The phrase route
|
|
only fires for verbatim 5+ token sequences from the question;
|
|
a different geography question shouldn't accidentally invoke it."""
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource([
|
|
_doc(
|
|
"test://orwell-stub",
|
|
# Body has the diagnostic Orwell 5-gram, but the title
|
|
# is title-irrelevant to a geography query.
|
|
"The novel narrates that Oceania always been at war with "
|
|
"Eastasia though the alliances had previously rotated. " * 6
|
|
),
|
|
_doc(
|
|
"test://geography-stub",
|
|
"Geographic descriptions of regions called Oceania and East "
|
|
"Asia. Topics: trade, population, climate, demographics. " * 12,
|
|
),
|
|
]))
|
|
finally:
|
|
conn.close()
|
|
|
|
# Literal geography query — short, no Orwell phrase.
|
|
result = query(
|
|
question="oceania east asia geography",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer="A geography answer."),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=5,
|
|
)
|
|
uris = [s["document_uri"] for s in result["sources"]]
|
|
# Geography stub should be present (literal query, literal source).
|
|
assert "test://geography-stub" in uris
|
|
# Orwell stub should NOT have been surfaced via phrase route on
|
|
# a literal-geography query — the phrase route only activates on
|
|
# verbatim 5+ token sequences from the question, and "oceania
|
|
# east asia geography" is too short to produce any.
|
|
|
|
|
|
def test_filter_keeps_phrase_match_root_with_no_title_overlap():
|
|
"""Direct unit test for accept-path 4: a hit whose title shares
|
|
zero content tokens with the question, but whose document_root is
|
|
in `phrase_match_roots`, must pass the filter."""
|
|
from arborist.qa.query import _Hit, _filter_by_title_relevance
|
|
hits = [
|
|
# Title shares NO content tokens with the question. Without
|
|
# accept-path 4 (phrase_match_roots), it would be dropped.
|
|
_Hit(
|
|
document_root="bbb",
|
|
document_uri="t://nineteen-eighty-four",
|
|
title="Nineteen Eighty-Four",
|
|
score=70.0,
|
|
shard_path="x",
|
|
chunk_idx=0,
|
|
),
|
|
]
|
|
# Without phrase_match_roots, the hit is dropped (title-relevance
|
|
# filter has no accept path that fires).
|
|
kept_without = _filter_by_title_relevance(
|
|
hits,
|
|
"has oceania always been at war with east asia",
|
|
)
|
|
# Filter falls back to top-N when nothing accepts; accept the
|
|
# fallback as 'kept' here too — what we care about is whether
|
|
# accept-path 4 is the path firing when phrase_match_roots is set.
|
|
kept_with = _filter_by_title_relevance(
|
|
hits,
|
|
"has oceania always been at war with east asia",
|
|
phrase_match_roots={"bbb"},
|
|
)
|
|
kept_roots = {h.document_root for h in kept_with}
|
|
assert "bbb" in kept_roots, (
|
|
"phrase_match_roots accept-path 4 should keep titles with no "
|
|
"token overlap when their body verbatim-matched a question phrase"
|
|
)
|
|
# And the fallback path doesn't suddenly fail when phrase_match_roots
|
|
# is present — the keep is via accept-path 4, not via the fallback.
|
|
_ = kept_without # documents that fallback may also keep, but via different path
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _search_titles long-query regression (2026-05-02)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _build_titled_corpus(tmp_path) -> tuple:
|
|
"""Test corpus shaped to exercise _search_titles. Returns
|
|
(single_db_path, qa_db_path)."""
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource([
|
|
_doc(
|
|
"test://neurotech",
|
|
# Long body so retrieval has something to grade.
|
|
"Neurotechnology covers brain-computer interfaces. " * 30
|
|
+ "Transcranial magnetic stimulation reads brain "
|
|
"activity. " * 30
|
|
),
|
|
_doc(
|
|
"test://decoy",
|
|
"Generic technology overview text. " * 60,
|
|
),
|
|
]))
|
|
finally:
|
|
conn.close()
|
|
return main_db, qa_db
|
|
|
|
|
|
def test_unit_search_titles_handles_long_question_without_crash(tmp_path):
|
|
"""Regression: 2026-05-02 fox-surfaced SQLite error on a
|
|
31-content-token query. The earlier SQL chained N CASE WHEN
|
|
expressions joined by `+`, blowing past SQLite's default
|
|
expression-tree depth limit (1000). The simplified SQL —
|
|
OR-chain + ORDER BY LENGTH(title) — handles arbitrary token
|
|
counts. Cap at MAX_TITLE_LIKE_TOKENS bounds the OR-chain
|
|
growth defensively."""
|
|
from arborist.qa.query import _search_titles
|
|
|
|
main_db, _qa_db = _build_titled_corpus(tmp_path)
|
|
conn = connect(main_db)
|
|
try:
|
|
# 50-token query — well past anything that would have hit
|
|
# the 1000-depth cap, but the bounded SQL handles it.
|
|
long_tokens = [
|
|
"what", "technology", "are", "currently", "or", "soon",
|
|
"available", "which", "may", "enable", "one", "person",
|
|
"to", "reconstruct", "and", "understand", "some", "portion",
|
|
"of", "another", "persons", "thoughts", "or", "ideas",
|
|
"without", "speaking", "or", "sign", "language",
|
|
"and", "make", "a", "summary", "we", "can", "use",
|
|
"to", "improve", "communication", "between", "humans",
|
|
"across", "the", "globe", "even", "if", "they", "speak",
|
|
"different", "tongues",
|
|
]
|
|
# Should not raise sqlite3.OperationalError; should return
|
|
# SOME rows (the test corpus has 2 docs, one will match).
|
|
rows = _search_titles(conn, long_tokens, 32)
|
|
assert isinstance(rows, list)
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def test_unit_search_titles_handles_zero_tokens():
|
|
"""Defensive: zero tokens → empty list, no SQL executed."""
|
|
from arborist.qa.query import _search_titles
|
|
|
|
rows = _search_titles(None, [], 32)
|
|
assert rows == []
|
|
|
|
|
|
def test_unit_search_titles_caps_or_chain_at_max_tokens(tmp_path):
|
|
"""The OR-chain is bounded at MAX_TITLE_LIKE_TOKENS so
|
|
extreme-length queries (100+ tokens) don't cascade SQL
|
|
expression-tree growth. Beyond the cap, additional tokens
|
|
are dropped at the SQL layer; the post-filter in the caller
|
|
sees them all and does the actual filtering."""
|
|
from arborist.qa.query import _search_titles
|
|
|
|
main_db, _qa_db = _build_titled_corpus(tmp_path)
|
|
conn = connect(main_db)
|
|
try:
|
|
# 200-token pathological query — should not crash.
|
|
many_tokens = [f"tok{i}" for i in range(200)]
|
|
rows = _search_titles(conn, many_tokens, 32)
|
|
assert isinstance(rows, list)
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def test_integration_query_completes_on_long_question(tmp_path):
|
|
"""End-to-end: a long discursive question with many content
|
|
tokens runs through query() to completion (StubClient — no
|
|
LLM) without the SQLite expression-tree overflow that the
|
|
2026-05-02 regression triggered."""
|
|
main_db, qa_db = _build_titled_corpus(tmp_path)
|
|
long_q = (
|
|
"what technology are currently or soon available which may "
|
|
"enable one person to reconstruct and understand some or a "
|
|
"portion of another persons thoughts or ideas without "
|
|
"speaking or sign language"
|
|
)
|
|
result = query(
|
|
question=long_q,
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer="A short stub answer."),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=3,
|
|
)
|
|
# Pre-fix this raised sqlite3.OperationalError before reaching
|
|
# the LLM call. Post-fix: query() returns a normal result dict.
|
|
assert result["status"] in (
|
|
"cache_miss_then_written",
|
|
"cache_hit",
|
|
"no_sources",
|
|
)
|
|
# Status field present means the pipeline completed.
|
|
assert "audit_mode" in result or result["status"] == "no_sources"
|
|
|
|
|
|
def test_functional_long_question_returns_sources(tmp_path):
|
|
"""Functional check: the long-question retrieval path actually
|
|
surfaces relevant sources (not just empty). The corpus has a
|
|
'neurotech' doc whose title and body share tokens with the
|
|
long question; it should appear in the top-K despite the
|
|
question's length."""
|
|
main_db, qa_db = _build_titled_corpus(tmp_path)
|
|
long_q = (
|
|
"what technology may enable one person to reconstruct and "
|
|
"understand another persons thoughts or ideas without "
|
|
"speaking or sign language using neurotechnology and "
|
|
"transcranial brain reading"
|
|
)
|
|
result = query(
|
|
question=long_q,
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer="answer"),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=3,
|
|
)
|
|
# Pipeline completed; if any sources were found at all, the
|
|
# neurotech doc (richest body match) should be among them.
|
|
if result["status"] != "no_sources":
|
|
uris = [s["document_uri"] for s in result.get("sources") or []]
|
|
# Neurotech doc has matching tokens; should appear.
|
|
assert any("neurotech" in u for u in uris), (
|
|
f"neurotech doc missing from sources {uris}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------
|
|
# Ticket #000007 — query-layer hyphen folding
|
|
# ---------------------------------------------------------------------
|
|
|
|
|
|
def test_unit_hyphen_fold_variants_emits_joined_form():
|
|
"""`bi-polar` → {"bipolar"}; multiple hyphen runs emit one
|
|
joined variant each; non-hyphenated input emits nothing."""
|
|
from arborist.qa.query import _hyphen_fold_variants
|
|
|
|
assert _hyphen_fold_variants("bi-polar is rare?") == {"bipolar"}
|
|
assert _hyphen_fold_variants("high-school co-op") == {
|
|
"highschool",
|
|
"coop",
|
|
}
|
|
assert _hyphen_fold_variants("plain query no hyphens") == set()
|
|
# Single-letter pieces are still folded — `X-ray` → `xray`.
|
|
assert "xray" in _hyphen_fold_variants("X-ray imaging")
|
|
|
|
|
|
def test_unit_title_query_tokens_includes_hyphen_fold_additively():
|
|
"""Hyphen-fold is additive — `bi-polar is rare?` produces both
|
|
the split forms (bi, polar) AND the joined form (bipolar)."""
|
|
from arborist.qa.query import _title_query_tokens
|
|
|
|
toks = _title_query_tokens("bi-polar is rare?")
|
|
assert "bi" in toks
|
|
assert "polar" in toks
|
|
assert "bipolar" in toks
|
|
assert "rare" in toks
|
|
# `is` is a stopword; should not appear.
|
|
assert "is" not in toks
|
|
|
|
|
|
def test_unit_title_query_tokens_no_hyphen_unchanged():
|
|
"""Non-hyphen input behaves exactly as before — pin that the
|
|
fold doesn't add spurious tokens for plain queries."""
|
|
from arborist.qa.query import _title_query_tokens
|
|
|
|
assert _title_query_tokens("anarchism political philosophy") == {
|
|
"anarchism",
|
|
"political",
|
|
"philosophy",
|
|
}
|
|
|
|
|
|
def test_integration_hyphenated_query_retrieves_joined_title(tmp_path):
|
|
"""Ticket #000007 reproduction: a query with a hyphenated form
|
|
must reach a corpus document whose title uses the joined form.
|
|
|
|
Corpus contains TWO docs:
|
|
- `Bipolar disorder` (joined-form title — what we want)
|
|
- `Bi-Polar Blues` (hyphenated-form title — irrelevant album)
|
|
|
|
Pre-fix: query `bi-polar is rare?` only retrieved `Bi-Polar Blues`
|
|
because FTS5 split both query and album-title on the hyphen
|
|
while leaving `Bipolar disorder` indexed as a single token.
|
|
|
|
Post-fix: the joined-form variant `bipolar` enters the title-FTS
|
|
OR-pool; accept-path 5 in `_filter_by_title_relevance` rescues
|
|
`Bipolar disorder` from the breadth gate. Both docs surface.
|
|
"""
|
|
main_db = tmp_path / "corpus.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
docs = [
|
|
_doc(
|
|
"test://bipolar-disorder",
|
|
"Bipolar disorder is a mental health condition. " * 20
|
|
+ "Bipolar disorder is rare in the elderly population. " * 5,
|
|
),
|
|
_doc(
|
|
"test://bi-polar-blues",
|
|
"Bi-Polar Blues is a 1995 jazz album. " * 20,
|
|
),
|
|
]
|
|
# Override default title (last URI segment) so titles match the
|
|
# FTS5 hyphen-asymmetry shape we care about.
|
|
docs[0].title = "Bipolar disorder"
|
|
docs[1].title = "Bi-Polar Blues"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(docs))
|
|
finally:
|
|
conn.close()
|
|
|
|
result = query(
|
|
question="bi-polar is rare?",
|
|
qa_db=qa_db,
|
|
chat_client=StubClient(answer="x"),
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=8,
|
|
)
|
|
src_uris = [s["document_uri"] for s in result.get("sources") or []]
|
|
# `Bipolar disorder` MUST surface — the whole point of the ticket.
|
|
assert any("bipolar-disorder" in u for u in src_uris), (
|
|
f"Ticket #000007 regression: bipolar-disorder not in {src_uris}"
|
|
)
|
|
|
|
|
|
def test_query_extra_shards_pulls_in_separate_db(tmp_path):
|
|
"""`extra_shards` searches a separate db alongside the main corpus —
|
|
the seam the CLI uses to auto-include the local crawl db without
|
|
folding it into the peer-shared main shards."""
|
|
main_db = tmp_path / "corpus.db"
|
|
crawl_db = tmp_path / "web.db"
|
|
conn = connect(main_db)
|
|
try:
|
|
ingest_source(conn, FakeSource(DOCS))
|
|
finally:
|
|
conn.close()
|
|
crawl_doc = _doc(
|
|
"test://virtback",
|
|
"Virtback is a python libvirt backup utility for kvm xen virtualbox. " * 12
|
|
+ "Virtback restores domains from backups. " * 8,
|
|
)
|
|
conn = connect(crawl_db)
|
|
try:
|
|
ingest_source(conn, FakeSource([crawl_doc]))
|
|
finally:
|
|
conn.close()
|
|
|
|
client = StubClient(
|
|
answer='Per the source: "Virtback is a python libvirt backup utility for kvm xen virtualbox"'
|
|
)
|
|
|
|
# Without extra_shards: the crawl-only topic is absent from the main corpus.
|
|
r_without = query(
|
|
question="What is virtback?",
|
|
qa_db=tmp_path / "qa1.db",
|
|
chat_client=client,
|
|
model_id="m",
|
|
single_db=main_db,
|
|
top_k=3,
|
|
)
|
|
assert all(
|
|
"virtback" not in s["document_uri"] for s in r_without.get("sources") or []
|
|
)
|
|
|
|
# With extra_shards: the separate crawl db is searched alongside it.
|
|
r_with = query(
|
|
question="What is virtback?",
|
|
qa_db=tmp_path / "qa2.db",
|
|
chat_client=client,
|
|
model_id="m",
|
|
single_db=main_db,
|
|
extra_shards=[crawl_db],
|
|
top_k=3,
|
|
)
|
|
assert any(
|
|
"virtback" in s["document_uri"] for s in r_with.get("sources") or []
|
|
), f"extra_shards crawl db not searched: {[s['document_uri'] for s in r_with.get('sources') or []]}"
|