qa: pipe wikitext-base-v1 into LLM context (was verifier-only)

Both ask() (single-doc) and query() (multi-source) now run the
assembled context through aborist.wikitext.to_base() before message
construction. Hermes sees clean prose instead of raw wikitext, which:

- lets the model quote verbatim (raw [[Cloud Strife]] was unquotable
  in fluent prose; the model paraphrased to bare "Cloud Strife" and
  the verifier flagged that as UNGROUNDED)
- shrinks token bills ~43% on Wikipedia chunks (measured: 125,152 →
  71,771 chars on the FF7 main-character query)
- aligns LLM-input with what the verifier already strips, so the
  prose-vs-prose substring check is end-to-end consistent

Gated on policy["base_version"] (= wikitext-base-v1) so:
- governance_policy_hash flows the choice — prior answers cached
  under raw-wikitext policy retain distinct cache_keys, no migration
- operators can opt out (set base_version=None) for debugging
- environments without mwparserfromhell soft-fall to raw wikitext
  with no other behavior change

Tests (4): default policy carries base_version (tripwire), ask() user
messages have wikitext markers stripped, opt-out reverts to raw
wikitext, governance_hash differs with vs without base_version.

aborist/qa/query.py

@@ -60,6 +60,13 @@ from aborist.qa.keys import (
     question_hash,
 )
 from aborist.qa.verify import verify_quotes
+
+try:
+    from aborist.wikitext import BASE_VERSION as _WIKITEXT_BASE_VERSION
+    from aborist.wikitext import to_base as _wikitext_to_base
+except ImportError:  # pragma: no cover
+    _WIKITEXT_BASE_VERSION = None
+    _wikitext_to_base = None
 from aborist.search import FTS5Backend
 from aborist.store import (
     append_audit,
@@ -224,6 +231,12 @@ DEFAULT_QUERY_POLICY = {
     "entity_policy": "proximity",
     "entity_proximity_n": 3,
     "entity_proximity_window": 300,
+    # Strip wikitext markup before the LLM ever sees the context. Lets
+    # Hermes quote prose verbatim and shrinks token bills (~43% on
+    # Wikipedia chunks). Bumps governance_policy_hash so prior cached
+    # answers under raw-wikitext policy stay distinct on lookup. No-op
+    # if mwparserfromhell isn't installed.
+    "base_version": _WIKITEXT_BASE_VERSION,
 }
 
 
@@ -664,6 +677,13 @@ def query(
             },
         }
     context = "\n\n".join(context_parts)
+
+    # Wikitext → prose before the LLM sees it. The model can then quote
+    # verbatim against the prose form; the verifier compares like-against-
+    # like. Idempotent if context is already plain prose. Gated on
+    # policy["base_version"] so this is part of governance_policy_hash.
+    if policy.get("base_version") and _wikitext_to_base is not None:
+        context = _wikitext_to_base(context)
     context_ms = _ms_since(t_phase)
 
     # 3. Build messages + hashes.

aborist/qa/runner.py

@@ -32,6 +32,13 @@ from aborist.qa.keys import (
 from aborist.qa.verify import verify_quotes
 from aborist.store import append_audit, transaction
 
+try:
+    from aborist.wikitext import BASE_VERSION as _WIKITEXT_BASE_VERSION
+    from aborist.wikitext import to_base as _wikitext_to_base
+except ImportError:  # pragma: no cover
+    _WIKITEXT_BASE_VERSION = None
+    _wikitext_to_base = None
+
 
 DEFAULT_POLICY = {
     "system_prompt": (
@@ -59,6 +66,12 @@ DEFAULT_POLICY = {
     "entity_policy": "proximity",
     "entity_proximity_n": 3,
     "entity_proximity_window": 300,
+    # Strip wikitext markup before the LLM ever sees the context. Lets
+    # Hermes quote prose verbatim and shrinks token bills (~43% on
+    # Wikipedia chunks). Bumps governance_policy_hash so prior cached
+    # answers under raw-wikitext policy stay distinct on lookup. Set
+    # via the wikitext extras; no-op if mwparserfromhell isn't installed.
+    "base_version": _WIKITEXT_BASE_VERSION,
 }
 
 
@@ -102,6 +115,13 @@ def ask(
 
     document_text = "\n\n".join(unpack_chunk(r["content"]) for r in chunk_rows)
 
+    # Wikitext → prose before the LLM sees it. The model can then quote
+    # verbatim against the prose form; the verifier compares like-against-
+    # like. Idempotent if context is already plain prose. Gated on
+    # policy["base_version"] so this is part of governance_policy_hash.
+    if policy.get("base_version") and _wikitext_to_base is not None:
+        document_text = _wikitext_to_base(document_text)
+
     # System sets the policy; a user-turn reminder restates the verbatim-quote
     # rule one message before the document arrives. Document + question land
     # last as the most-recent tokens before generation.

tests/test_llm_context_base.py

@@ -0,0 +1,168 @@
+"""LLM context is wikitext-stripped before send.
+
+Before this lands, runner.ask() and query.query() handed Hermes raw
+[[wikitext]] markup. The model paraphrased it into clean prose then
+labeled the result as a verbatim quote — verifier correctly flagged
+those as UNGROUNDED. With base_version in policy, both paths run
+to_base() on the assembled context before message construction; Hermes
+sees prose so it CAN quote verbatim, and the verifier compares prose-
+to-prose end-to-end (idempotent — verify_quotes also runs to_base).
+
+These tests use StubClient to capture exactly what messages would be
+sent to the LLM, asserting the wikitext markup is gone before the
+model would see it.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from aborist.document import Document
+from aborist.ingest import ingest_source
+from aborist.qa.client import StubClient
+from aborist.qa.runner import DEFAULT_POLICY, ask
+from aborist.store import connect
+from aborist.wikitext import BASE_VERSION
+
+
+WIKITEXT_DOC = (
+    "{{Refimprove|date=October 2009}}\n"
+    "[[File:foo.jpg|thumb|caption]]\n"
+    "''[[Final Fantasy VII]]'' follows [[protagonist]] [[Cloud Strife]], "
+    "a troubled mercenary.<ref>Smith 2010</ref> "
+    "He joins [[AVALANCHE]] to stop [[Shinra Electric Power Company|Shinra]]."
+)
+
+
+class _OneDocSource:
+    source_type = "llm_ctx_test"
+
+    def __init__(self, content: str):
+        self._doc = Document(
+            uri="https://example.com/ff7",
+            content=content,
+            source_type=self.source_type,
+            title="Test",
+        )
+
+    def iter_documents(self):
+        yield self._doc
+
+
+def _ingest_one(db_path: Path, content: str) -> str:
+    conn = connect(db_path)
+    try:
+        ingest_source(conn, _OneDocSource(content))
+        row = conn.execute(
+            "SELECT document_root FROM documents WHERE document_uri=?",
+            ("https://example.com/ff7",),
+        ).fetchone()
+    finally:
+        conn.close()
+    return row["document_root"]
+
+
+# ---------------------------------------------------------------------------
+# DEFAULT_POLICY carries base_version
+# ---------------------------------------------------------------------------
+
+
+def test_default_policy_includes_base_version():
+    """If base_version disappears from default policy, the LLM goes back
+    to seeing raw wikitext silently. Tripwire."""
+    assert DEFAULT_POLICY["base_version"] == BASE_VERSION
+
+
+# ---------------------------------------------------------------------------
+# Stripped context flows to the LLM (ask path)
+# ---------------------------------------------------------------------------
+
+
+def _user_messages_text(client: StubClient) -> str:
+    """Concatenate every user-role message from the LAST captured call."""
+    last = client.calls[-1]
+    return "\n".join(m["content"] for m in last["messages"] if m["role"] == "user")
+
+
+def test_ask_passes_base_form_to_llm(tmp_path):
+    db = tmp_path / "qa.db"
+    root = _ingest_one(db, WIKITEXT_DOC)
+
+    client = StubClient(answer="I don't know based on the provided document.")
+    conn = connect(db)
+    try:
+        ask(
+            conn,
+            document_root=root,
+            question="who is the protagonist?",
+            client=client,
+            model_id="stub-model",
+        )
+    finally:
+        conn.close()
+
+    sent = _user_messages_text(client)
+    # Wikitext markers MUST be gone.
+    assert "[[" not in sent
+    assert "]]" not in sent
+    assert "{{Refimprove" not in sent
+    assert "<ref>" not in sent
+    assert "File:" not in sent
+    # Topical content survives.
+    assert "Cloud Strife" in sent
+    assert "Final Fantasy VII" in sent
+    assert "AVALANCHE" in sent
+
+
+def test_ask_with_base_version_disabled_passes_raw_wikitext(tmp_path):
+    """Operator escape hatch: setting base_version=None reverts to raw
+    wikitext context. Confirms the gate is effective and reversible."""
+    db = tmp_path / "qa.db"
+    root = _ingest_one(db, WIKITEXT_DOC)
+
+    custom_policy = dict(DEFAULT_POLICY)
+    custom_policy["base_version"] = None
+
+    client = StubClient(answer="stub")
+    conn = connect(db)
+    try:
+        ask(
+            conn,
+            document_root=root,
+            question="who is the protagonist?",
+            client=client,
+            model_id="stub-model",
+            policy=custom_policy,
+        )
+    finally:
+        conn.close()
+
+    sent = _user_messages_text(client)
+    # Raw wikitext markers should now BE present in what the LLM saw.
+    assert "[[" in sent
+    assert "]]" in sent
+
+
+# ---------------------------------------------------------------------------
+# governance_policy_hash flows base_version (cache_key separation)
+# ---------------------------------------------------------------------------
+
+
+def test_base_version_changes_governance_hash(tmp_path):
+    """Two policies that differ only in base_version produce different
+    governance_policy_hash values. This is what keeps prior cached
+    answers (under raw-wikitext policy) from satisfying lookups under
+    the new (stripped) policy on the same question/source — they remain
+    distinct cache_keys."""
+    from aborist.qa.keys import governance_policy_hash
+
+    p_with = dict(DEFAULT_POLICY)
+    p_with["base_version"] = "wikitext-base-v1"
+    p_without = dict(DEFAULT_POLICY)
+    p_without["base_version"] = None
+
+    h_with = governance_policy_hash(p_with)
+    h_without = governance_policy_hash(p_without)
+    assert h_with != h_without