diff --git a/aborist/qa/query.py b/aborist/qa/query.py
index 7f76d50..663ab80 100644
--- a/aborist/qa/query.py
+++ b/aborist/qa/query.py
@@ -60,6 +60,13 @@ from aborist.qa.keys import (
question_hash,
)
from aborist.qa.verify import verify_quotes
+
+try:
+ from aborist.wikitext import BASE_VERSION as _WIKITEXT_BASE_VERSION
+ from aborist.wikitext import to_base as _wikitext_to_base
+except ImportError: # pragma: no cover
+ _WIKITEXT_BASE_VERSION = None
+ _wikitext_to_base = None
from aborist.search import FTS5Backend
from aborist.store import (
append_audit,
@@ -224,6 +231,12 @@ DEFAULT_QUERY_POLICY = {
"entity_policy": "proximity",
"entity_proximity_n": 3,
"entity_proximity_window": 300,
+ # Strip wikitext markup before the LLM ever sees the context. Lets
+ # Hermes quote prose verbatim and shrinks token bills (~43% on
+ # Wikipedia chunks). Bumps governance_policy_hash so prior cached
+ # answers under raw-wikitext policy stay distinct on lookup. No-op
+ # if mwparserfromhell isn't installed.
+ "base_version": _WIKITEXT_BASE_VERSION,
}
@@ -664,6 +677,13 @@ def query(
},
}
context = "\n\n".join(context_parts)
+
+ # Wikitext → prose before the LLM sees it. The model can then quote
+ # verbatim against the prose form; the verifier compares like-against-
+ # like. Idempotent if context is already plain prose. Gated on
+ # policy["base_version"] so this is part of governance_policy_hash.
+ if policy.get("base_version") and _wikitext_to_base is not None:
+ context = _wikitext_to_base(context)
context_ms = _ms_since(t_phase)
# 3. Build messages + hashes.
diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py
index c06f915..c5b9fdd 100644
--- a/aborist/qa/runner.py
+++ b/aborist/qa/runner.py
@@ -32,6 +32,13 @@ from aborist.qa.keys import (
from aborist.qa.verify import verify_quotes
from aborist.store import append_audit, transaction
+try:
+ from aborist.wikitext import BASE_VERSION as _WIKITEXT_BASE_VERSION
+ from aborist.wikitext import to_base as _wikitext_to_base
+except ImportError: # pragma: no cover
+ _WIKITEXT_BASE_VERSION = None
+ _wikitext_to_base = None
+
DEFAULT_POLICY = {
"system_prompt": (
@@ -59,6 +66,12 @@ DEFAULT_POLICY = {
"entity_policy": "proximity",
"entity_proximity_n": 3,
"entity_proximity_window": 300,
+ # Strip wikitext markup before the LLM ever sees the context. Lets
+ # Hermes quote prose verbatim and shrinks token bills (~43% on
+ # Wikipedia chunks). Bumps governance_policy_hash so prior cached
+ # answers under raw-wikitext policy stay distinct on lookup. Set
+ # via the wikitext extras; no-op if mwparserfromhell isn't installed.
+ "base_version": _WIKITEXT_BASE_VERSION,
}
@@ -102,6 +115,13 @@ def ask(
document_text = "\n\n".join(unpack_chunk(r["content"]) for r in chunk_rows)
+ # Wikitext → prose before the LLM sees it. The model can then quote
+ # verbatim against the prose form; the verifier compares like-against-
+ # like. Idempotent if context is already plain prose. Gated on
+ # policy["base_version"] so this is part of governance_policy_hash.
+ if policy.get("base_version") and _wikitext_to_base is not None:
+ document_text = _wikitext_to_base(document_text)
+
# System sets the policy; a user-turn reminder restates the verbatim-quote
# rule one message before the document arrives. Document + question land
# last as the most-recent tokens before generation.
diff --git a/tests/test_llm_context_base.py b/tests/test_llm_context_base.py
new file mode 100644
index 0000000..b3281dc
--- /dev/null
+++ b/tests/test_llm_context_base.py
@@ -0,0 +1,168 @@
+"""LLM context is wikitext-stripped before send.
+
+Before this lands, runner.ask() and query.query() handed Hermes raw
+[[wikitext]] markup. The model paraphrased it into clean prose then
+labeled the result as a verbatim quote — verifier correctly flagged
+those as UNGROUNDED. With base_version in policy, both paths run
+to_base() on the assembled context before message construction; Hermes
+sees prose so it CAN quote verbatim, and the verifier compares prose-
+to-prose end-to-end (idempotent — verify_quotes also runs to_base).
+
+These tests use StubClient to capture exactly what messages would be
+sent to the LLM, asserting the wikitext markup is gone before the
+model would see it.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from aborist.document import Document
+from aborist.ingest import ingest_source
+from aborist.qa.client import StubClient
+from aborist.qa.runner import DEFAULT_POLICY, ask
+from aborist.store import connect
+from aborist.wikitext import BASE_VERSION
+
+
+WIKITEXT_DOC = (
+ "{{Refimprove|date=October 2009}}\n"
+ "[[File:foo.jpg|thumb|caption]]\n"
+ "''[[Final Fantasy VII]]'' follows [[protagonist]] [[Cloud Strife]], "
+ "a troubled mercenary.[Smith 2010] "
+ "He joins [[AVALANCHE]] to stop [[Shinra Electric Power Company|Shinra]]."
+)
+
+
+class _OneDocSource:
+ source_type = "llm_ctx_test"
+
+ def __init__(self, content: str):
+ self._doc = Document(
+ uri="https://example.com/ff7",
+ content=content,
+ source_type=self.source_type,
+ title="Test",
+ )
+
+ def iter_documents(self):
+ yield self._doc
+
+
+def _ingest_one(db_path: Path, content: str) -> str:
+ conn = connect(db_path)
+ try:
+ ingest_source(conn, _OneDocSource(content))
+ row = conn.execute(
+ "SELECT document_root FROM documents WHERE document_uri=?",
+ ("https://example.com/ff7",),
+ ).fetchone()
+ finally:
+ conn.close()
+ return row["document_root"]
+
+
+# ---------------------------------------------------------------------------
+# DEFAULT_POLICY carries base_version
+# ---------------------------------------------------------------------------
+
+
+def test_default_policy_includes_base_version():
+ """If base_version disappears from default policy, the LLM goes back
+ to seeing raw wikitext silently. Tripwire."""
+ assert DEFAULT_POLICY["base_version"] == BASE_VERSION
+
+
+# ---------------------------------------------------------------------------
+# Stripped context flows to the LLM (ask path)
+# ---------------------------------------------------------------------------
+
+
+def _user_messages_text(client: StubClient) -> str:
+ """Concatenate every user-role message from the LAST captured call."""
+ last = client.calls[-1]
+ return "\n".join(m["content"] for m in last["messages"] if m["role"] == "user")
+
+
+def test_ask_passes_base_form_to_llm(tmp_path):
+ db = tmp_path / "qa.db"
+ root = _ingest_one(db, WIKITEXT_DOC)
+
+ client = StubClient(answer="I don't know based on the provided document.")
+ conn = connect(db)
+ try:
+ ask(
+ conn,
+ document_root=root,
+ question="who is the protagonist?",
+ client=client,
+ model_id="stub-model",
+ )
+ finally:
+ conn.close()
+
+ sent = _user_messages_text(client)
+ # Wikitext markers MUST be gone.
+ assert "[[" not in sent
+ assert "]]" not in sent
+ assert "{{Refimprove" not in sent
+ assert "[" not in sent
+ assert "File:" not in sent
+ # Topical content survives.
+ assert "Cloud Strife" in sent
+ assert "Final Fantasy VII" in sent
+ assert "AVALANCHE" in sent
+
+
+def test_ask_with_base_version_disabled_passes_raw_wikitext(tmp_path):
+ """Operator escape hatch: setting base_version=None reverts to raw
+ wikitext context. Confirms the gate is effective and reversible."""
+ db = tmp_path / "qa.db"
+ root = _ingest_one(db, WIKITEXT_DOC)
+
+ custom_policy = dict(DEFAULT_POLICY)
+ custom_policy["base_version"] = None
+
+ client = StubClient(answer="stub")
+ conn = connect(db)
+ try:
+ ask(
+ conn,
+ document_root=root,
+ question="who is the protagonist?",
+ client=client,
+ model_id="stub-model",
+ policy=custom_policy,
+ )
+ finally:
+ conn.close()
+
+ sent = _user_messages_text(client)
+ # Raw wikitext markers should now BE present in what the LLM saw.
+ assert "[[" in sent
+ assert "]]" in sent
+
+
+# ---------------------------------------------------------------------------
+# governance_policy_hash flows base_version (cache_key separation)
+# ---------------------------------------------------------------------------
+
+
+def test_base_version_changes_governance_hash(tmp_path):
+ """Two policies that differ only in base_version produce different
+ governance_policy_hash values. This is what keeps prior cached
+ answers (under raw-wikitext policy) from satisfying lookups under
+ the new (stripped) policy on the same question/source — they remain
+ distinct cache_keys."""
+ from aborist.qa.keys import governance_policy_hash
+
+ p_with = dict(DEFAULT_POLICY)
+ p_with["base_version"] = "wikitext-base-v1"
+ p_without = dict(DEFAULT_POLICY)
+ p_without["base_version"] = None
+
+ h_with = governance_policy_hash(p_with)
+ h_without = governance_policy_hash(p_without)
+ assert h_with != h_without
]