russell@unturf.com
71507f9327
Both ask() (single-doc) and query() (multi-source) now run the assembled context through aborist.wikitext.to_base() before message construction. Hermes sees clean prose instead of raw wikitext, which: - lets the model quote verbatim (raw [[Cloud Strife]] was unquotable in fluent prose; the model paraphrased to bare "Cloud Strife" and the verifier flagged that as UNGROUNDED) - shrinks token bills ~43% on Wikipedia chunks (measured: 125,152 → 71,771 chars on the FF7 main-character query) - aligns LLM-input with what the verifier already strips, so the prose-vs-prose substring check is end-to-end consistent Gated on policy["base_version"] (= wikitext-base-v1) so: - governance_policy_hash flows the choice — prior answers cached under raw-wikitext policy retain distinct cache_keys, no migration - operators can opt out (set base_version=None) for debugging - environments without mwparserfromhell soft-fall to raw wikitext with no other behavior change Tests (4): default policy carries base_version (tripwire), ask() user messages have wikitext markers stripped, opt-out reverts to raw wikitext, governance_hash differs with vs without base_version.
894 lines
34 KiB
Python
894 lines
34 KiB
Python
"""Multi-source corpus Q&A.
|
|
|
|
Pose a question, the tree finds related cached docs, assembles them as context,
|
|
asks Hermes, caches the answer.
|
|
|
|
The flow:
|
|
1. FTS5 search across all shards (chunks_fts can't be UNION'd in views,
|
|
so we query each shard's index independently and merge by score).
|
|
2. Title-boost rerank: hits whose title contains query tokens get a
|
|
score bump. Title is a strong topical signal that BM25 alone misses
|
|
(BM25 favors short docs with rare body tokens — Tell_(poker) outranks
|
|
Back_to_the_Future without a title boost).
|
|
3. Pick top-K distinct documents within a character budget.
|
|
4. Compute `context_root` = Merkle root over the sorted source
|
|
document_roots — that's the "source" dimension of v9.8's 8-dim
|
|
cache_key for this multi-source answer.
|
|
5. Cache lookup; hit returns the persisted audit_mode.
|
|
6. Miss calls Hermes via the OpenAI-compatible client, then runs the
|
|
faithfulness check (`verify_quotes`) — every double-quoted span in
|
|
the answer is verbatim-matched against the assembled context.
|
|
Result classifies the answer:
|
|
STRICT every quote (>=1) verified against context
|
|
HYBRID some claims sourced, some emergent (training-derived)
|
|
UNGROUNDED no quotes verify — purely emergent
|
|
7. Persist record with merkle_proof = {context_root, sources: [...]},
|
|
audit_mode, and unverified_quotes (the spans the model produced
|
|
that didn't appear in any source — corpus-growth signal).
|
|
|
|
Per-source proofs are not bundled here (the source roots themselves are
|
|
already content-addressed). A verifier asks the shards for any specific
|
|
chunk's proof on demand.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import time
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
from aborist import (
|
|
CANONICALIZATION_VERSION,
|
|
CHUNKING_VERSION,
|
|
SCHEMA_VERSION,
|
|
)
|
|
from aborist.compress import unpack_chunk
|
|
from aborist.merkle import MerkleTree
|
|
from aborist.qa.client import ChatClient
|
|
from aborist.qa.concepts import (
|
|
has_compare_phrasing,
|
|
rivalry_excluded,
|
|
synonym_expand,
|
|
)
|
|
from aborist.qa.keys import (
|
|
cache_key,
|
|
conversation_hash,
|
|
governance_policy_hash,
|
|
model_profile_hash,
|
|
question_hash,
|
|
)
|
|
from aborist.qa.verify import verify_quotes
|
|
|
|
try:
|
|
from aborist.wikitext import BASE_VERSION as _WIKITEXT_BASE_VERSION
|
|
from aborist.wikitext import to_base as _wikitext_to_base
|
|
except ImportError: # pragma: no cover
|
|
_WIKITEXT_BASE_VERSION = None
|
|
_wikitext_to_base = None
|
|
from aborist.search import FTS5Backend
|
|
from aborist.store import (
|
|
append_audit,
|
|
connect,
|
|
discover_shards,
|
|
transaction,
|
|
)
|
|
|
|
|
|
_TITLE_TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z0-9]*")
|
|
# Kept in sync with FTS5 stopwords in aborist.search.fts5 — both filter
|
|
# the same set of question-shaping words. "tell" leaking into title-LIKE
|
|
# search caused "tell me about permacomputer" to pull Tell_(poker), the
|
|
# Tell-Tale_Heart movie, Tell_City Indiana, etc.
|
|
_TITLE_STOPWORDS = frozenset(
|
|
"""
|
|
the a an is are was were be been being of to in on at for with by from
|
|
as about into through during and or but not no nor so yet too very also just
|
|
what who where when why how which this that these those such i you he she
|
|
it we they me him her us them do does did have has had can could should
|
|
would will may might
|
|
tell show describe explain summarize say give list find make please
|
|
""".split()
|
|
)
|
|
|
|
|
|
def _title_query_tokens(s: str) -> set[str]:
|
|
return {
|
|
t.lower()
|
|
for t in _TITLE_TOKEN_RE.findall(s)
|
|
if t.lower() not in _TITLE_STOPWORDS and len(t) > 1
|
|
}
|
|
|
|
|
|
def _rerank_by_title(
|
|
hits: list,
|
|
question: str,
|
|
boost: float = 10.0,
|
|
) -> list:
|
|
"""Boost hits whose title overlaps query tokens. Pure ordering aid."""
|
|
qtokens = _title_query_tokens(question)
|
|
if not qtokens:
|
|
return hits
|
|
for h in hits:
|
|
if not h.title:
|
|
continue
|
|
ttokens = _title_query_tokens(h.title.replace("_", " "))
|
|
overlap = len(qtokens & ttokens)
|
|
if overlap:
|
|
h.score += overlap * boost
|
|
hits.sort(key=lambda h: -h.score)
|
|
return hits
|
|
|
|
|
|
def _filter_by_title_relevance(
|
|
hits: list,
|
|
question: str,
|
|
*,
|
|
core_match_roots: set[str] | None = None,
|
|
body_density_check: callable | None = None,
|
|
fallback_top_n: int = 5,
|
|
) -> list:
|
|
"""Concept-aware relevance filter with three accept paths:
|
|
|
|
1. Title-token overlap (after synonym expansion). Strongest signal.
|
|
2. TF-IDF core keyword overlap — `core_match_roots` is a precomputed
|
|
set of source document_roots whose TF-IDF cores contain query
|
|
tokens. Closes the gap for neologisms like "permacomputer" that
|
|
never appear in titles but are distinctive enough to be TF-IDF
|
|
keywords of conversation bodies.
|
|
3. Body density — docs mentioning the query token >= N times pass
|
|
even without title or core match. Cheap proxy for "actually about
|
|
the topic." `body_density_check(hit)` returns bool.
|
|
|
|
Rivalry exclusion (Intel-titled docs in AMD queries) still applies on
|
|
every accept path.
|
|
|
|
If all three accept paths together produce nothing, fall back to the
|
|
top `fallback_top_n` body-BM25 hits — the LLM gets enough context to
|
|
say "I don't know" rather than fabricating from a single tangential
|
|
source.
|
|
"""
|
|
qtokens = _title_query_tokens(question)
|
|
if not qtokens:
|
|
return hits
|
|
accept = synonym_expand(qtokens)
|
|
exclude = rivalry_excluded(qtokens, compare_phrasing=has_compare_phrasing(question))
|
|
core_roots = core_match_roots or set()
|
|
|
|
kept: list = []
|
|
for h in hits:
|
|
ttokens = _title_query_tokens(h.title.replace("_", " ")) if h.title else set()
|
|
if exclude & ttokens:
|
|
continue # rivalry: opposing-side title, drop it
|
|
if accept & ttokens:
|
|
kept.append(h)
|
|
continue
|
|
if h.document_root in core_roots:
|
|
kept.append(h)
|
|
continue
|
|
if body_density_check is not None and body_density_check(h):
|
|
kept.append(h)
|
|
continue
|
|
if not kept:
|
|
return hits[: max(1, fallback_top_n)] if hits else []
|
|
return kept
|
|
|
|
|
|
DEFAULT_QUERY_POLICY = {
|
|
"system_prompt": (
|
|
"You are answering a question using ONLY the sources provided below. "
|
|
"Each source is delimited by '=== Source: <URI> ===' headers.\n\n"
|
|
"GROUNDING RULE (most important):\n"
|
|
"For EVERY factual claim in your answer, include a verbatim quote "
|
|
"from a source enclosed in double quotes (\"...\"). The quoted span "
|
|
"must appear word-for-word in one of the sources. If you cannot "
|
|
"find a verbatim quote that supports a claim, do not make the "
|
|
"claim. Do not paraphrase inside the quotes.\n\n"
|
|
"ATTRIBUTION RULES:\n"
|
|
"1. A single source may discuss MULTIPLE products, companies, or "
|
|
"competitors. Read carefully and only attribute facts to the entity "
|
|
"the source explicitly names for that fact.\n"
|
|
"2. If the question asks about ONE specific entity (e.g., 'fastest "
|
|
"AMD CPU'), do NOT include facts about competitors (Intel, Pentium) "
|
|
"as if they were facts about the asked entity. They are different "
|
|
"products even when discussed in the same article.\n"
|
|
"3. When referencing model numbers like 'Athlon XP 3200+', remember "
|
|
"that AMD's PR-rating numbers (3200+, 2500+) are NOT the clock speed "
|
|
"in MHz. Quote the source's own wording for clock speeds rather than "
|
|
"interpreting model numbers.\n"
|
|
"4. If the sources do not contain the answer, say 'I don't know "
|
|
"based on the provided sources.' Do not fall back on background "
|
|
"knowledge from training."
|
|
),
|
|
# Restated rule fired as a user message immediately before the sources
|
|
# turn. Hermes (and most instruction-tuned 8B models) follow recent
|
|
# user-turn instructions more reliably than a system-turn rule that
|
|
# decays under long context. Repetition is not redundancy — it raises
|
|
# the prior on the response shape we want.
|
|
"grounding_reminder": (
|
|
"REMINDER: every factual claim in your reply must be wrapped in "
|
|
"double quotes (\"...\") and the quoted span must appear "
|
|
"word-for-word in one of the sources. No quote, no claim. "
|
|
"Example:\n\n"
|
|
" Q: who founded Apple?\n"
|
|
" Sources: ...Apple Inc. was founded by Steve Jobs, Steve "
|
|
"Wozniak, and Ronald Wayne in 1976...\n"
|
|
" A: Apple's founders are named in the source: \"Apple Inc. was "
|
|
"founded by Steve Jobs, Steve Wozniak, and Ronald Wayne\".\n\n"
|
|
"Now answer the question on the next message."
|
|
),
|
|
"temperature": 0.1,
|
|
"top_p": 1.0,
|
|
"max_tokens": 768,
|
|
# Entity-path policy for the faithfulness verifier. See
|
|
# aborist/qa/verify.py:ENTITY_POLICIES. Default 'proximity' promotes
|
|
# to STRICT only if N verified entities cluster within W chars of
|
|
# each other in source — separating "source documents these entities
|
|
# as a group" (cast list / infobox / roster) from "source mentions
|
|
# them incidentally in scattered prose". Lives in policy so changing
|
|
# it bumps governance_policy_hash and invalidates cache cleanly.
|
|
"entity_policy": "proximity",
|
|
"entity_proximity_n": 3,
|
|
"entity_proximity_window": 300,
|
|
# Strip wikitext markup before the LLM ever sees the context. Lets
|
|
# Hermes quote prose verbatim and shrinks token bills (~43% on
|
|
# Wikipedia chunks). Bumps governance_policy_hash so prior cached
|
|
# answers under raw-wikitext policy stay distinct on lookup. No-op
|
|
# if mwparserfromhell isn't installed.
|
|
"base_version": _WIKITEXT_BASE_VERSION,
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class _Hit:
|
|
document_root: str
|
|
document_uri: str
|
|
title: str | None
|
|
score: float
|
|
shard_path: str
|
|
chunk_idx: int
|
|
|
|
|
|
def _search_titles(conn, qtokens: list[str], limit: int) -> list[tuple]:
|
|
"""SQL LIKE over documents.title — finds the HTTP article that FTS5
|
|
misses because list-pages with many URLs have higher 'http' term
|
|
frequency than the actual protocol article. Returns rows shaped
|
|
to match the FTS5 hit tuple."""
|
|
if not qtokens:
|
|
return []
|
|
clauses = " OR ".join(["LOWER(title) LIKE ?"] * len(qtokens))
|
|
params = [f"%{t.lower()}%" for t in qtokens]
|
|
params.append(limit)
|
|
rows = conn.execute(
|
|
f"SELECT document_root, document_uri, title FROM documents "
|
|
f"WHERE {clauses} LIMIT ?",
|
|
params,
|
|
).fetchall()
|
|
return rows
|
|
|
|
|
|
def _docs_with_core_keyword_match(
|
|
conn, qtokens: list[str], limit: int
|
|
) -> list[tuple]:
|
|
"""Find SOURCE docs whose TF-IDF core keywords contain a query token.
|
|
|
|
TF-IDF cores act as enriched titles: a doc's distinctive low-frequency
|
|
terms get distilled into the core's content as comma-separated keywords.
|
|
A query for "permacomputer" (a neologism that never makes it into a
|
|
title) can match the TF-IDF core of a Grok conversation that mentioned
|
|
it, because permacomputer is a rare term that ranks high under TF-IDF.
|
|
|
|
Returns rows of (document_root, document_uri, title) for the SOURCE docs
|
|
(not the cores) — the source is what gets fed to the LLM as context.
|
|
"""
|
|
if not qtokens:
|
|
return []
|
|
# Word-boundary match against the comma-separated TF-IDF keyword list.
|
|
# Prepending/appending ", " lets one LIKE pattern (`%, token, %`) check
|
|
# for the token regardless of its position in the keyword string.
|
|
# Without this, naive `LIKE '%intel%'` would match "intelligence",
|
|
# "intellectual", "intellivision" — drowning real hits like Pentium_4
|
|
# (whose TF-IDF core has "intel" as an exact keyword) in noise.
|
|
#
|
|
# Per-row `match_count` tallies how many distinct query tokens hit
|
|
# this doc's TF-IDF core. Multi-token coverage is a strong relevance
|
|
# signal — a doc whose core has "intel" + "cpu" + "faster" beats a
|
|
# doc whose only signal is "intel" appearing in its TITLE. The
|
|
# caller boosts the score by match_count.
|
|
case_clauses = " + ".join(
|
|
[
|
|
"(CASE WHEN LOWER(', ' || c.content || ', ') LIKE ? THEN 1 ELSE 0 END)"
|
|
]
|
|
* len(qtokens)
|
|
)
|
|
where_clauses = " OR ".join(
|
|
["LOWER(', ' || c.content || ', ') LIKE ?"] * len(qtokens)
|
|
)
|
|
patterns = [f"%, {t.lower()}, %" for t in qtokens]
|
|
params = patterns + patterns + [limit]
|
|
rows = conn.execute(
|
|
f"""
|
|
SELECT
|
|
src.document_root,
|
|
src.document_uri,
|
|
src.title,
|
|
MAX({case_clauses}) AS match_count
|
|
FROM chunks c
|
|
JOIN documents core ON core.document_root = c.document_root
|
|
JOIN derivations der ON der.core_root = core.document_root
|
|
JOIN documents src ON src.document_root = der.src_root
|
|
WHERE core.source_type LIKE 'core:tfidf-%'
|
|
AND ({where_clauses})
|
|
GROUP BY src.document_root, src.document_uri, src.title
|
|
ORDER BY match_count DESC
|
|
LIMIT ?
|
|
""",
|
|
params,
|
|
).fetchall()
|
|
return rows
|
|
|
|
|
|
def _body_density_passes(
|
|
conn, document_root: str, qtokens: set[str], min_mentions: int = 3
|
|
) -> bool:
|
|
"""Body-token CO-OCCURRENCE relevance.
|
|
|
|
Pre-2026-04-27 this counted any token's mentions in the body (so
|
|
Intel_8086, with 30 "Intel" mentions, would pass for a query of
|
|
{intel, fastest, cpu} despite never mentioning "fastest"). The
|
|
revised gate requires at least HALF the query tokens to each appear
|
|
at least once, AND the total mention count to clear `min_mentions`.
|
|
|
|
For a 1-token query (e.g. {anarchism}) this simplifies to "the body
|
|
must mention the token at least min_mentions times" — same as before.
|
|
For multi-token queries, both signals must hold: breadth (most
|
|
tokens present) and depth (enough total mentions).
|
|
"""
|
|
if not qtokens:
|
|
return False
|
|
rows = conn.execute(
|
|
"SELECT content FROM chunks WHERE document_root = ? AND content IS NOT NULL",
|
|
(document_root,),
|
|
).fetchall()
|
|
if not rows:
|
|
return False
|
|
body = " ".join(unpack_chunk(r["content"]) or "" for r in rows).lower()
|
|
counts = {t: body.count(t.lower()) for t in qtokens}
|
|
distinct_present = sum(1 for n in counts.values() if n > 0)
|
|
total_mentions = sum(counts.values())
|
|
breadth_threshold = max(1, (len(qtokens) + 1) // 2)
|
|
return distinct_present >= breadth_threshold and total_mentions >= min_mentions
|
|
|
|
|
|
def _search_corpus(
|
|
shards_dir: Path | None,
|
|
single_db: Path | None,
|
|
question: str,
|
|
over_fetch: int,
|
|
) -> list[_Hit]:
|
|
"""Two parallel searches across shards, merged:
|
|
|
|
- FTS5 body search (BM25 over chunk content).
|
|
- SQL title-LIKE search (catches articles whose body is short on the
|
|
query terms but whose title is the literal topic — e.g., the HTTP
|
|
protocol article doesn't out-frequency-score URL-heavy list pages
|
|
but it IS the topic).
|
|
|
|
Dedupe by document_root. Title hits get a baseline score that
|
|
out-ranks FTS5 body hits so the actual topic article rises to the top.
|
|
"""
|
|
qtokens = _title_query_tokens(question)
|
|
# Synonym expansion: a query for "athlon" also fetches AMD-titled docs.
|
|
accept_tokens = synonym_expand(qtokens)
|
|
paths: list[Path]
|
|
if shards_dir is not None:
|
|
paths = discover_shards(shards_dir)
|
|
elif single_db is not None:
|
|
paths = [Path(single_db)]
|
|
else:
|
|
paths = []
|
|
|
|
raw: list[tuple] = []
|
|
# Roots whose TF-IDF cores contain a query token — collected across
|
|
# shards. Used downstream by _filter_by_title_relevance as accept-path 2.
|
|
core_match_roots: set[str] = set()
|
|
# Per-shard mapping of doc root -> shard path, for body-density lookups
|
|
# in accept-path 3. Lets us reach back to the source shard cheaply.
|
|
root_to_shard: dict[str, str] = {}
|
|
|
|
for p in paths:
|
|
conn = connect(p)
|
|
try:
|
|
backend = FTS5Backend(conn)
|
|
for h in backend.search(question, limit=over_fetch):
|
|
raw.append(
|
|
(
|
|
h.score,
|
|
h.document_root,
|
|
h.document_uri,
|
|
h.title,
|
|
h.chunk_idx,
|
|
str(p.resolve()),
|
|
)
|
|
)
|
|
root_to_shard[h.document_root] = str(p.resolve())
|
|
# Parallel title search using synonym-expanded tokens.
|
|
# The score starts deliberately low so this signal can't drown
|
|
# FTS5 BM25 + body relevance. _rerank_by_title later adds
|
|
# `overlap*10` to every hit (FTS5 and title-search alike) that
|
|
# has title-token overlap, so a doc whose title genuinely IS
|
|
# the topic ends up rewarded twice (once here, once in rerank).
|
|
# Single-token title matches against generic terms ("intel",
|
|
# "cpu") used to score 60+ standalone, dominating top-K with
|
|
# legacy 80486-era articles for queries like "fastest intel
|
|
# CPU?". Now that contribution is the same scale as FTS5 body
|
|
# BM25, which lets Pentium_4 (high body relevance, no title
|
|
# overlap) win on its actual topical fit.
|
|
for r in _search_titles(conn, list(accept_tokens), over_fetch):
|
|
title_lower = (r["title"] or "").lower().replace("_", " ")
|
|
overlap = sum(1 for t in accept_tokens if t in title_lower)
|
|
if overlap == 0:
|
|
continue
|
|
title_score = overlap * 10.0
|
|
raw.append(
|
|
(
|
|
title_score,
|
|
r["document_root"],
|
|
r["document_uri"],
|
|
r["title"],
|
|
0,
|
|
str(p.resolve()),
|
|
)
|
|
)
|
|
root_to_shard[r["document_root"]] = str(p.resolve())
|
|
# Core-keyword search: docs whose TF-IDF core keywords match.
|
|
# The query token doesn't need to be in title or even in body —
|
|
# being a TF-IDF keyword of the doc's core is enough signal.
|
|
for r in _docs_with_core_keyword_match(
|
|
conn, list(accept_tokens), over_fetch
|
|
):
|
|
core_match_roots.add(r["document_root"])
|
|
# Score scales with how many query tokens hit this doc's
|
|
# TF-IDF core. A 3-token coverage (e.g. Pentium_4's core
|
|
# carries "intel", "cpu", "faster" for the query "fastest
|
|
# intel CPU?") beats single-token title boosts (~80) that
|
|
# otherwise saturate the top with Intel_80486DX,
|
|
# Intel_8086, etc. — articles that share *one* word with
|
|
# the query but aren't the topical answer.
|
|
match_count = r["match_count"] or 1
|
|
kw_score = 40.0 + 25.0 * match_count
|
|
raw.append(
|
|
(
|
|
kw_score,
|
|
r["document_root"],
|
|
r["document_uri"],
|
|
r["title"],
|
|
0,
|
|
str(p.resolve()),
|
|
)
|
|
)
|
|
root_to_shard[r["document_root"]] = str(p.resolve())
|
|
finally:
|
|
conn.close()
|
|
|
|
raw.sort(key=lambda r: -r[0])
|
|
seen: set[str] = set()
|
|
out: list[_Hit] = []
|
|
for score, root, uri, title, idx, sp in raw:
|
|
if root in seen:
|
|
continue
|
|
seen.add(root)
|
|
out.append(
|
|
_Hit(
|
|
document_root=root,
|
|
document_uri=uri,
|
|
title=title,
|
|
score=score,
|
|
shard_path=sp,
|
|
chunk_idx=idx,
|
|
)
|
|
)
|
|
return out
|
|
|
|
|
|
def _rerank(
|
|
hits: list[_Hit],
|
|
question: str,
|
|
*,
|
|
core_match_roots: set[str] | None = None,
|
|
body_density_check: callable | None = None,
|
|
) -> list[_Hit]:
|
|
"""Filter off-topic, then layer in body-coverage and title-overlap boosts.
|
|
|
|
Order matters: filter first (drops noise), body-coverage rerank next
|
|
(counters BM25's short-doc bias by rewarding topical density across
|
|
distinct query tokens), title-overlap rerank last (breaks ties when
|
|
a doc IS the named topic).
|
|
"""
|
|
hits = _filter_by_title_relevance(
|
|
hits,
|
|
question,
|
|
core_match_roots=core_match_roots,
|
|
body_density_check=body_density_check,
|
|
)
|
|
hits = _rerank_by_body_coverage(hits, question)
|
|
return _rerank_by_title(hits, question)
|
|
|
|
|
|
def _rerank_by_body_coverage(
|
|
hits: list,
|
|
question: str,
|
|
*,
|
|
weight: float = 0.6,
|
|
) -> list:
|
|
"""Boost score by per-token body coverage to counteract BM25's short-doc bias.
|
|
|
|
BM25 normalizes by document length, but its `b` parameter under-penalizes
|
|
*very* short docs that happen to mention every query token. Result: a
|
|
1-paragraph stub on Intel 4040 (1974 microcontroller) outscored a 30-page
|
|
Intel Core i7 article for "what is the fastest intel CPU?".
|
|
|
|
Approach: for each hit's full body, sum sqrt(count) per query token.
|
|
Sqrt scaling lets long topical articles meaningfully out-score short
|
|
tangential ones without runaway domination by enumerative list pages.
|
|
|
|
stub article: 5 intel + 1 fastest + 4 cpu -> sqrt(5)+sqrt(1)+sqrt(4) ~ 5.2
|
|
long topical: 200 + 10 + 80 -> sqrt(200)+sqrt(10)+sqrt(80) ~ 26.2
|
|
enumeration: 1000 + 50 + 300 -> sqrt(1000)+sqrt(50)+sqrt(300) ~ 55.8
|
|
|
|
Multiply by `weight` (default 0.6) and add to the existing score. The
|
|
differentiation in the example above is decisive but bounded: long topical
|
|
articles get +15-16, enumerations get +33, stubs get +3. Combined with
|
|
FTS5 base scores in the 40s, the long topical article wins comfortably.
|
|
|
|
Cost: one body fetch per surviving candidate (typically 24-32). Bounded.
|
|
"""
|
|
import math
|
|
|
|
qtokens_lower = {t.lower() for t in _title_query_tokens(question)}
|
|
if not qtokens_lower:
|
|
return hits
|
|
for h in hits:
|
|
body = _load_doc_text(h.shard_path, h.document_root)
|
|
if body is None:
|
|
continue
|
|
body_lower = body.lower()
|
|
coverage = sum(
|
|
math.sqrt(body_lower.count(t)) for t in qtokens_lower
|
|
)
|
|
h.score += coverage * weight
|
|
hits.sort(key=lambda h: -h.score)
|
|
return hits
|
|
|
|
|
|
def _load_doc_text(shard_path: str, document_root: str) -> str | None:
|
|
"""Concatenate all hot chunks of a document. Returns None if cold or missing."""
|
|
conn = connect(shard_path)
|
|
try:
|
|
rows = conn.execute(
|
|
"SELECT content FROM chunks "
|
|
"WHERE document_root = ? AND content IS NOT NULL "
|
|
"ORDER BY idx ASC",
|
|
(document_root,),
|
|
).fetchall()
|
|
finally:
|
|
conn.close()
|
|
if not rows:
|
|
return None
|
|
return "\n\n".join(unpack_chunk(r["content"]) or "" for r in rows)
|
|
|
|
|
|
def _context_root(source_roots: list[str]) -> str:
|
|
"""Merkle root over sorted source document_roots — the v9.8 'source' dim
|
|
for multi-source answers. Sorting makes the root deterministic regardless
|
|
of search ranking order."""
|
|
if not source_roots:
|
|
return "00" * 32
|
|
sorted_roots = sorted(source_roots)
|
|
if len(sorted_roots) == 1:
|
|
return sorted_roots[0]
|
|
leaves = [bytes.fromhex(r) for r in sorted_roots]
|
|
return MerkleTree.build(leaves).root.hex()
|
|
|
|
|
|
def query(
|
|
*,
|
|
question: str,
|
|
qa_db: Path,
|
|
chat_client: ChatClient,
|
|
model_id: str,
|
|
revision: str = "",
|
|
quantization: str = "",
|
|
shards_dir: Path | None = None,
|
|
single_db: Path | None = None,
|
|
top_k: int = 8,
|
|
over_fetch: int = 32,
|
|
max_context_chars: int = 60000,
|
|
policy: dict | None = None,
|
|
chain: str = "private",
|
|
) -> dict:
|
|
"""Answer `question` using the corpus. Cache to qa_db. Returns a result dict."""
|
|
policy = policy or DEFAULT_QUERY_POLICY
|
|
t_start = time.monotonic()
|
|
|
|
# 1. Search.
|
|
t_phase = time.monotonic()
|
|
hits = _search_corpus(shards_dir, single_db, question, over_fetch)
|
|
if not hits:
|
|
return {
|
|
"status": "no_sources",
|
|
"msg": "FTS5 search returned no hits",
|
|
"timings": {
|
|
"search_ms": _ms_since(t_phase),
|
|
"total_ms": _ms_since(t_start),
|
|
},
|
|
}
|
|
core_match_roots = getattr(hits, "_core_match_roots", set())
|
|
root_to_shard = getattr(hits, "_root_to_shard", {})
|
|
qtokens_lower = {t.lower() for t in _title_query_tokens(question)}
|
|
|
|
def _body_density_check(h) -> bool:
|
|
# Lazy per-hit check: open the shard, count token mentions in this doc.
|
|
sp = root_to_shard.get(h.document_root) or h.shard_path
|
|
if not sp:
|
|
return False
|
|
c = connect(sp)
|
|
try:
|
|
return _body_density_passes(c, h.document_root, qtokens_lower)
|
|
finally:
|
|
c.close()
|
|
|
|
hits = _rerank(
|
|
hits,
|
|
question,
|
|
core_match_roots=core_match_roots,
|
|
body_density_check=_body_density_check,
|
|
)
|
|
search_ms = _ms_since(t_phase)
|
|
|
|
# 2. Pull doc texts within budget.
|
|
t_phase = time.monotonic()
|
|
chosen: list[_Hit] = []
|
|
context_parts: list[str] = []
|
|
char_budget = max_context_chars
|
|
for h in hits[:top_k]:
|
|
text = _load_doc_text(h.shard_path, h.document_root)
|
|
if not text:
|
|
continue
|
|
if len(text) > char_budget:
|
|
text = text[:char_budget]
|
|
context_parts.append(
|
|
f"=== Source: {h.document_uri} ===\n{text}"
|
|
)
|
|
chosen.append(h)
|
|
char_budget -= len(text)
|
|
if char_budget <= 0:
|
|
break
|
|
|
|
if not chosen:
|
|
return {
|
|
"status": "no_sources",
|
|
"msg": "top-k hits had cold or empty content",
|
|
"timings": {
|
|
"search_ms": search_ms,
|
|
"context_ms": _ms_since(t_phase),
|
|
"total_ms": _ms_since(t_start),
|
|
},
|
|
}
|
|
context = "\n\n".join(context_parts)
|
|
|
|
# Wikitext → prose before the LLM sees it. The model can then quote
|
|
# verbatim against the prose form; the verifier compares like-against-
|
|
# like. Idempotent if context is already plain prose. Gated on
|
|
# policy["base_version"] so this is part of governance_policy_hash.
|
|
if policy.get("base_version") and _wikitext_to_base is not None:
|
|
context = _wikitext_to_base(context)
|
|
context_ms = _ms_since(t_phase)
|
|
|
|
# 3. Build messages + hashes.
|
|
# System sets the policy; a user-turn reminder restates the verbatim-quote
|
|
# rule one message before the sources arrive (Hermes follows recent
|
|
# user-turn instructions more reliably than system-turn rules under long
|
|
# context). The sources + question land last so they are the most-recent
|
|
# tokens before generation begins.
|
|
messages = [
|
|
{"role": "system", "content": policy["system_prompt"]},
|
|
]
|
|
if policy.get("grounding_reminder"):
|
|
messages.append(
|
|
{"role": "user", "content": policy["grounding_reminder"]}
|
|
)
|
|
messages.append(
|
|
{
|
|
"role": "user",
|
|
"content": f"Sources:\n\n{context}\n\n---\n\nQuestion: {question}",
|
|
}
|
|
)
|
|
context_root = _context_root([h.document_root for h in chosen])
|
|
qhash = question_hash(question)
|
|
mhash = model_profile_hash(model_id, revision, quantization)
|
|
chash = conversation_hash(messages)
|
|
ghash = governance_policy_hash(policy)
|
|
ckey = cache_key(
|
|
context_root,
|
|
qhash,
|
|
mhash,
|
|
chash,
|
|
ghash,
|
|
SCHEMA_VERSION,
|
|
CANONICALIZATION_VERSION,
|
|
CHUNKING_VERSION,
|
|
)
|
|
|
|
qa_conn = connect(qa_db)
|
|
try:
|
|
# 4. Cache lookup.
|
|
t_phase = time.monotonic()
|
|
cached = qa_conn.execute(
|
|
"SELECT * FROM providence_cache "
|
|
"WHERE cache_key = ? AND falsification_state = 'live'",
|
|
(ckey,),
|
|
).fetchone()
|
|
cache_lookup_ms = _ms_since(t_phase)
|
|
if cached is not None:
|
|
now = int(time.time())
|
|
with transaction(qa_conn):
|
|
qa_conn.execute(
|
|
"UPDATE providence_cache "
|
|
"SET hit_count = hit_count + 1, last_hit_at = ? "
|
|
"WHERE cache_key = ?",
|
|
(now, ckey),
|
|
)
|
|
return {
|
|
"status": "cache_hit",
|
|
"audit_mode": cached["audit_mode"],
|
|
"cache_key": ckey,
|
|
"context_root": context_root,
|
|
"answer_text": cached["answer_text"],
|
|
"sources": json.loads(cached["merkle_proof"])["sources"],
|
|
"n_quotes": cached["n_quotes"],
|
|
"n_verified": cached["n_verified"],
|
|
"verifier_method": cached["verifier_method"],
|
|
"unverified_quotes": (
|
|
json.loads(cached["unverified_quotes"])
|
|
if cached["unverified_quotes"]
|
|
else []
|
|
),
|
|
"timings": {
|
|
"search_ms": search_ms,
|
|
"context_ms": context_ms,
|
|
"cache_lookup_ms": cache_lookup_ms,
|
|
"llm_ms": None,
|
|
"persist_ms": None,
|
|
"total_ms": _ms_since(t_start),
|
|
},
|
|
}
|
|
|
|
# 5. Cache miss — call LLM.
|
|
t_phase = time.monotonic()
|
|
answer_text = chat_client.chat_completion(
|
|
messages,
|
|
model=model_id,
|
|
temperature=policy["temperature"],
|
|
max_tokens=policy["max_tokens"],
|
|
top_p=policy.get("top_p", 1.0),
|
|
)
|
|
llm_ms = _ms_since(t_phase)
|
|
|
|
# 5b. Faithfulness check — pull quoted spans, verify each verbatim
|
|
# against the assembled context, classify into v9.8 audit_mode.
|
|
verdict = verify_quotes(
|
|
answer_text,
|
|
context,
|
|
entity_policy=policy.get("entity_policy", "hybrid"),
|
|
proximity_n=policy.get("entity_proximity_n", 3),
|
|
proximity_window=policy.get("entity_proximity_window", 300),
|
|
)
|
|
unverified_blob = (
|
|
json.dumps(verdict["unverified_quotes"], separators=(",", ":"))
|
|
if verdict["unverified_quotes"]
|
|
else None
|
|
)
|
|
|
|
# 6. Persist record + audit event.
|
|
t_phase = time.monotonic()
|
|
proof_obj = {
|
|
"context_root": context_root,
|
|
"sources": [
|
|
{
|
|
"document_root": h.document_root,
|
|
"document_uri": h.document_uri,
|
|
"title": h.title,
|
|
"score": h.score,
|
|
"chunk_idx": h.chunk_idx,
|
|
"shard": Path(h.shard_path).name,
|
|
}
|
|
for h in chosen
|
|
],
|
|
}
|
|
proof_blob = json.dumps(proof_obj, separators=(",", ":"))
|
|
now = int(time.time())
|
|
with transaction(qa_conn):
|
|
event_hash = append_audit(
|
|
qa_conn,
|
|
event_type="providence_query",
|
|
subject_root=ckey,
|
|
body={
|
|
"context_root": context_root,
|
|
"n_sources": len(chosen),
|
|
"model_id": model_id,
|
|
"revision": revision,
|
|
"quantization": quantization,
|
|
"answer_chars": len(answer_text),
|
|
"context_chars": len(context),
|
|
"audit_mode": verdict["audit_mode"],
|
|
"n_quotes": verdict["n_quotes"],
|
|
"n_verified": verdict["n_verified"],
|
|
"verifier_method": verdict["verifier_method"],
|
|
},
|
|
ts=now,
|
|
)
|
|
qa_conn.execute(
|
|
"INSERT INTO providence_cache "
|
|
"(cache_key, source_root, document_uri, question_hash, question_text, "
|
|
" answer_text, merkle_proof, model_profile_hash, conversation_hash, "
|
|
" governance_policy_hash, schema_version, canonicalization_version, "
|
|
" chunking_version, falsification_state, chain, audit_event_hash, "
|
|
" created_at, hit_count, audit_mode, n_quotes, n_verified, "
|
|
" unverified_quotes, verifier_method) "
|
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'live', ?, ?, ?, 0, "
|
|
" ?, ?, ?, ?, ?)",
|
|
(
|
|
ckey,
|
|
context_root,
|
|
"corpus://multi-source",
|
|
qhash,
|
|
question,
|
|
answer_text,
|
|
proof_blob,
|
|
mhash,
|
|
chash,
|
|
ghash,
|
|
SCHEMA_VERSION,
|
|
CANONICALIZATION_VERSION,
|
|
CHUNKING_VERSION,
|
|
chain,
|
|
event_hash,
|
|
now,
|
|
verdict["audit_mode"],
|
|
verdict["n_quotes"],
|
|
verdict["n_verified"],
|
|
unverified_blob,
|
|
verdict["verifier_method"],
|
|
),
|
|
)
|
|
finally:
|
|
qa_conn.close()
|
|
|
|
persist_ms = _ms_since(t_phase)
|
|
return {
|
|
"status": "cache_miss_then_written",
|
|
"audit_mode": verdict["audit_mode"],
|
|
"cache_key": ckey,
|
|
"context_root": context_root,
|
|
"answer_text": answer_text,
|
|
"sources": proof_obj["sources"],
|
|
"n_quotes": verdict["n_quotes"],
|
|
"n_verified": verdict["n_verified"],
|
|
"verifier_method": verdict["verifier_method"],
|
|
"unverified_quotes": verdict["unverified_quotes"],
|
|
"timings": {
|
|
"search_ms": search_ms,
|
|
"context_ms": context_ms,
|
|
"cache_lookup_ms": cache_lookup_ms,
|
|
"llm_ms": llm_ms,
|
|
"persist_ms": persist_ms,
|
|
"total_ms": _ms_since(t_start),
|
|
},
|
|
}
|
|
|
|
|
|
def _ms_since(t: float) -> float:
|
|
"""Wall-time elapsed in milliseconds, rounded to 1 decimal."""
|
|
return round((time.monotonic() - t) * 1000, 1)
|