From 71507f93277366b2eac6496a3d0cd789a5102619 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Tue, 28 Apr 2026 17:08:49 -0400 Subject: [PATCH] qa: pipe wikitext-base-v1 into LLM context (was verifier-only) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both ask() (single-doc) and query() (multi-source) now run the assembled context through aborist.wikitext.to_base() before message construction. Hermes sees clean prose instead of raw wikitext, which: - lets the model quote verbatim (raw [[Cloud Strife]] was unquotable in fluent prose; the model paraphrased to bare "Cloud Strife" and the verifier flagged that as UNGROUNDED) - shrinks token bills ~43% on Wikipedia chunks (measured: 125,152 → 71,771 chars on the FF7 main-character query) - aligns LLM-input with what the verifier already strips, so the prose-vs-prose substring check is end-to-end consistent Gated on policy["base_version"] (= wikitext-base-v1) so: - governance_policy_hash flows the choice — prior answers cached under raw-wikitext policy retain distinct cache_keys, no migration - operators can opt out (set base_version=None) for debugging - environments without mwparserfromhell soft-fall to raw wikitext with no other behavior change Tests (4): default policy carries base_version (tripwire), ask() user messages have wikitext markers stripped, opt-out reverts to raw wikitext, governance_hash differs with vs without base_version. --- aborist/qa/query.py | 20 ++++ aborist/qa/runner.py | 20 ++++ tests/test_llm_context_base.py | 168 +++++++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+) create mode 100644 tests/test_llm_context_base.py diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 7f76d50..663ab80 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -60,6 +60,13 @@ from aborist.qa.keys import ( question_hash, ) from aborist.qa.verify import verify_quotes + +try: + from aborist.wikitext import BASE_VERSION as _WIKITEXT_BASE_VERSION + from aborist.wikitext import to_base as _wikitext_to_base +except ImportError: # pragma: no cover + _WIKITEXT_BASE_VERSION = None + _wikitext_to_base = None from aborist.search import FTS5Backend from aborist.store import ( append_audit, @@ -224,6 +231,12 @@ DEFAULT_QUERY_POLICY = { "entity_policy": "proximity", "entity_proximity_n": 3, "entity_proximity_window": 300, + # Strip wikitext markup before the LLM ever sees the context. Lets + # Hermes quote prose verbatim and shrinks token bills (~43% on + # Wikipedia chunks). Bumps governance_policy_hash so prior cached + # answers under raw-wikitext policy stay distinct on lookup. No-op + # if mwparserfromhell isn't installed. + "base_version": _WIKITEXT_BASE_VERSION, } @@ -664,6 +677,13 @@ def query( }, } context = "\n\n".join(context_parts) + + # Wikitext → prose before the LLM sees it. The model can then quote + # verbatim against the prose form; the verifier compares like-against- + # like. Idempotent if context is already plain prose. Gated on + # policy["base_version"] so this is part of governance_policy_hash. + if policy.get("base_version") and _wikitext_to_base is not None: + context = _wikitext_to_base(context) context_ms = _ms_since(t_phase) # 3. Build messages + hashes. diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index c06f915..c5b9fdd 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -32,6 +32,13 @@ from aborist.qa.keys import ( from aborist.qa.verify import verify_quotes from aborist.store import append_audit, transaction +try: + from aborist.wikitext import BASE_VERSION as _WIKITEXT_BASE_VERSION + from aborist.wikitext import to_base as _wikitext_to_base +except ImportError: # pragma: no cover + _WIKITEXT_BASE_VERSION = None + _wikitext_to_base = None + DEFAULT_POLICY = { "system_prompt": ( @@ -59,6 +66,12 @@ DEFAULT_POLICY = { "entity_policy": "proximity", "entity_proximity_n": 3, "entity_proximity_window": 300, + # Strip wikitext markup before the LLM ever sees the context. Lets + # Hermes quote prose verbatim and shrinks token bills (~43% on + # Wikipedia chunks). Bumps governance_policy_hash so prior cached + # answers under raw-wikitext policy stay distinct on lookup. Set + # via the wikitext extras; no-op if mwparserfromhell isn't installed. + "base_version": _WIKITEXT_BASE_VERSION, } @@ -102,6 +115,13 @@ def ask( document_text = "\n\n".join(unpack_chunk(r["content"]) for r in chunk_rows) + # Wikitext → prose before the LLM sees it. The model can then quote + # verbatim against the prose form; the verifier compares like-against- + # like. Idempotent if context is already plain prose. Gated on + # policy["base_version"] so this is part of governance_policy_hash. + if policy.get("base_version") and _wikitext_to_base is not None: + document_text = _wikitext_to_base(document_text) + # System sets the policy; a user-turn reminder restates the verbatim-quote # rule one message before the document arrives. Document + question land # last as the most-recent tokens before generation. diff --git a/tests/test_llm_context_base.py b/tests/test_llm_context_base.py new file mode 100644 index 0000000..b3281dc --- /dev/null +++ b/tests/test_llm_context_base.py @@ -0,0 +1,168 @@ +"""LLM context is wikitext-stripped before send. + +Before this lands, runner.ask() and query.query() handed Hermes raw +[[wikitext]] markup. The model paraphrased it into clean prose then +labeled the result as a verbatim quote — verifier correctly flagged +those as UNGROUNDED. With base_version in policy, both paths run +to_base() on the assembled context before message construction; Hermes +sees prose so it CAN quote verbatim, and the verifier compares prose- +to-prose end-to-end (idempotent — verify_quotes also runs to_base). + +These tests use StubClient to capture exactly what messages would be +sent to the LLM, asserting the wikitext markup is gone before the +model would see it. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from aborist.document import Document +from aborist.ingest import ingest_source +from aborist.qa.client import StubClient +from aborist.qa.runner import DEFAULT_POLICY, ask +from aborist.store import connect +from aborist.wikitext import BASE_VERSION + + +WIKITEXT_DOC = ( + "{{Refimprove|date=October 2009}}\n" + "[[File:foo.jpg|thumb|caption]]\n" + "''[[Final Fantasy VII]]'' follows [[protagonist]] [[Cloud Strife]], " + "a troubled mercenary.Smith 2010 " + "He joins [[AVALANCHE]] to stop [[Shinra Electric Power Company|Shinra]]." +) + + +class _OneDocSource: + source_type = "llm_ctx_test" + + def __init__(self, content: str): + self._doc = Document( + uri="https://example.com/ff7", + content=content, + source_type=self.source_type, + title="Test", + ) + + def iter_documents(self): + yield self._doc + + +def _ingest_one(db_path: Path, content: str) -> str: + conn = connect(db_path) + try: + ingest_source(conn, _OneDocSource(content)) + row = conn.execute( + "SELECT document_root FROM documents WHERE document_uri=?", + ("https://example.com/ff7",), + ).fetchone() + finally: + conn.close() + return row["document_root"] + + +# --------------------------------------------------------------------------- +# DEFAULT_POLICY carries base_version +# --------------------------------------------------------------------------- + + +def test_default_policy_includes_base_version(): + """If base_version disappears from default policy, the LLM goes back + to seeing raw wikitext silently. Tripwire.""" + assert DEFAULT_POLICY["base_version"] == BASE_VERSION + + +# --------------------------------------------------------------------------- +# Stripped context flows to the LLM (ask path) +# --------------------------------------------------------------------------- + + +def _user_messages_text(client: StubClient) -> str: + """Concatenate every user-role message from the LAST captured call.""" + last = client.calls[-1] + return "\n".join(m["content"] for m in last["messages"] if m["role"] == "user") + + +def test_ask_passes_base_form_to_llm(tmp_path): + db = tmp_path / "qa.db" + root = _ingest_one(db, WIKITEXT_DOC) + + client = StubClient(answer="I don't know based on the provided document.") + conn = connect(db) + try: + ask( + conn, + document_root=root, + question="who is the protagonist?", + client=client, + model_id="stub-model", + ) + finally: + conn.close() + + sent = _user_messages_text(client) + # Wikitext markers MUST be gone. + assert "[[" not in sent + assert "]]" not in sent + assert "{{Refimprove" not in sent + assert "" not in sent + assert "File:" not in sent + # Topical content survives. + assert "Cloud Strife" in sent + assert "Final Fantasy VII" in sent + assert "AVALANCHE" in sent + + +def test_ask_with_base_version_disabled_passes_raw_wikitext(tmp_path): + """Operator escape hatch: setting base_version=None reverts to raw + wikitext context. Confirms the gate is effective and reversible.""" + db = tmp_path / "qa.db" + root = _ingest_one(db, WIKITEXT_DOC) + + custom_policy = dict(DEFAULT_POLICY) + custom_policy["base_version"] = None + + client = StubClient(answer="stub") + conn = connect(db) + try: + ask( + conn, + document_root=root, + question="who is the protagonist?", + client=client, + model_id="stub-model", + policy=custom_policy, + ) + finally: + conn.close() + + sent = _user_messages_text(client) + # Raw wikitext markers should now BE present in what the LLM saw. + assert "[[" in sent + assert "]]" in sent + + +# --------------------------------------------------------------------------- +# governance_policy_hash flows base_version (cache_key separation) +# --------------------------------------------------------------------------- + + +def test_base_version_changes_governance_hash(tmp_path): + """Two policies that differ only in base_version produce different + governance_policy_hash values. This is what keeps prior cached + answers (under raw-wikitext policy) from satisfying lookups under + the new (stripped) policy on the same question/source — they remain + distinct cache_keys.""" + from aborist.qa.keys import governance_policy_hash + + p_with = dict(DEFAULT_POLICY) + p_with["base_version"] = "wikitext-base-v1" + p_without = dict(DEFAULT_POLICY) + p_without["base_version"] = None + + h_with = governance_policy_hash(p_with) + h_without = governance_policy_hash(p_without) + assert h_with != h_without