tests/qa/prompts: 20 tests pinning load-bearing system prompts (was zero coverage)

arborist/qa/prompts.py — 4 string constants that ARE the contract
with the LLM. Silent edits drop bench STRICT-rate by tens of
points; per CLAUDE.md "bench-maxing" discipline, prompt
regressions need a per-PR guard, not just bench surfacing after
the fact.

This test file pins the load-bearing phrases as regression
guards for both claim_lattice modes:

  CLAIM_LATTICE_SYSTEM_PROMPT (pointer mode)
  - non-empty + substantial
  - both worked examples present (Apple founders + Mars descriptive)
  - two-pointer cap rule (folds into claim_lattice_max_pointers_per_claim)
  - pointer ID shape teaching (E1, E2, E3, [E#] / [E#,E#])
  - one-claim-per-line rule (parser splits on newlines)
  - synthetic-elision-by-construction-impossible: NO instruction
    to wrap claims in double quotes (that's legacy quote-mode)

  CLAIM_LATTICE_GROUNDING_REMINDER (pointer mode user-turn)
  - "REMINDER" prefix; "pointer-line" format restated
  - two-pointer cap restated

  CLAIM_LATTICE_JSON_SYSTEM_PROMPT (JSON mode)
  - schema shape: claims/text/evidence_ids
  - first-char-`{` / last-char-`}` discipline
  - two-evidence-id cap

  CLAIM_LATTICE_JSON_GROUNDING_REMINDER (JSON mode user-turn)
  - schema restated; cap restated

  Cross-prompt parity
  - both modes reference the two-pointer cap
  - both reminders end with "next message" (handoff to question)
  - all 4 constants importable + non-empty strings

Full suite: 1872 passed, 45 skipped.

tests/test_qa_prompts.py

@@ -0,0 +1,207 @@
+"""Tests for ``arborist.qa.prompts`` — load-bearing system prompts
++ user-turn grounding reminders for claim_lattice modes.
+
+The prompts ARE the contract with the LLM. A silent edit that
+removes a critical instruction (e.g. drops the "at most two
+pointers per claim" rule) can drop STRICT-rate by tens of points
+on bench fixtures. These tests pin the contract phrases as
+regression guards.
+
+Per CLAUDE.md "bench-maxing — measure deltas, not opinions":
+prompt regressions are silent in unit tests but loud in bench;
+this file moves the regression guard into a per-PR check.
+"""
+
+from __future__ import annotations
+
+from arborist.qa.prompts import (
+    CLAIM_LATTICE_GROUNDING_REMINDER,
+    CLAIM_LATTICE_JSON_GROUNDING_REMINDER,
+    CLAIM_LATTICE_JSON_SYSTEM_PROMPT,
+    CLAIM_LATTICE_SYSTEM_PROMPT,
+)
+
+
+# --- pointer-mode system prompt ------------------------------------
+
+
+def test_pointer_system_prompt_nonempty():
+    assert CLAIM_LATTICE_SYSTEM_PROMPT
+    assert len(CLAIM_LATTICE_SYSTEM_PROMPT) > 100  # substantial
+
+
+def test_pointer_system_prompt_contains_worked_examples():
+    """Two worked examples (Apple founders + Mars descriptive)
+    are documented as load-bearing for prompt scaffolding. If a
+    PR drops them, this test fires."""
+    assert "WORKED EXAMPLE 1" in CLAIM_LATTICE_SYSTEM_PROMPT
+    assert "WORKED EXAMPLE 2" in CLAIM_LATTICE_SYSTEM_PROMPT
+    assert "END OF EXAMPLES" in CLAIM_LATTICE_SYSTEM_PROMPT
+
+
+def test_pointer_system_prompt_pins_two_pointer_cap():
+    """'At most two pointers per claim' is the cache-key-affecting
+    rule that maps to claim_lattice_max_pointers_per_claim policy
+    field. Removing the prompt instruction without bumping the
+    governance_policy_hash field would silently disagree."""
+    p = CLAIM_LATTICE_SYSTEM_PROMPT.lower()
+    assert "two pointer" in p or "two pointers" in p
+
+
+def test_pointer_system_prompt_documents_pointer_id_shape():
+    """Pointer IDs are E1, E2, E3, etc. Prompt must teach the
+    bracket-tag shape `[E#]` or `[E#,E#]` so the parser's regex
+    matches model output."""
+    p = CLAIM_LATTICE_SYSTEM_PROMPT
+    assert "E1" in p
+    assert "[E" in p  # bracket-tag form
+    assert "E2" in p
+    assert "E3" in p
+
+
+def test_pointer_system_prompt_includes_one_claim_per_line_rule():
+    """Parser splits on newlines; prompt must teach the model to
+    emit one claim per line."""
+    p = CLAIM_LATTICE_SYSTEM_PROMPT.lower()
+    assert "one claim per" in p or "one plain-prose sentence" in p
+
+
+def test_pointer_system_prompt_no_quote_instruction():
+    """Pointer mode is the synthetic-elision-by-construction-
+    impossible path: the model NEVER types the quote string;
+    the runtime interpolates literal spans. The prompt must NOT
+    instruct the model to wrap claims in double quotes (that's
+    the legacy quote-mode prompt)."""
+    p = CLAIM_LATTICE_SYSTEM_PROMPT.lower()
+    # Looking for an instruction like "wrap each claim in
+    # double quotes". The phrase 'verbatim quote' / 'enclosed in
+    # double quotes' is the legacy quote-mode pattern; ensure
+    # absent.
+    assert 'wrap every factual claim in double quotes' not in p
+    assert 'wrap each claim in double quotes' not in p
+
+
+# --- pointer-mode grounding reminder -------------------------------
+
+
+def test_pointer_reminder_nonempty():
+    assert CLAIM_LATTICE_GROUNDING_REMINDER
+    assert "REMINDER" in CLAIM_LATTICE_GROUNDING_REMINDER
+
+
+def test_pointer_reminder_pins_pointer_line_format():
+    """The reminder is the user-turn restatement that fires
+    immediately before the LLM generates. It must restate the
+    pointer-line format."""
+    r = CLAIM_LATTICE_GROUNDING_REMINDER.lower()
+    assert "pointer-line" in r or "pointer line" in r
+    assert "[e1]" in r or "[e#]" in r or "[e1,e#]" in r
+
+
+def test_pointer_reminder_restates_two_pointer_cap():
+    r = CLAIM_LATTICE_GROUNDING_REMINDER.lower()
+    assert "two pointer" in r or "at most two" in r
+
+
+# --- JSON-mode system prompt ---------------------------------------
+
+
+def test_json_system_prompt_nonempty():
+    assert CLAIM_LATTICE_JSON_SYSTEM_PROMPT
+    assert len(CLAIM_LATTICE_JSON_SYSTEM_PROMPT) > 100
+
+
+def test_json_system_prompt_pins_schema_shape():
+    """The verifier accepts the JSON shape `{"claims":[{"text":...,
+    "evidence_ids":[...]}]}`. The prompt must teach exactly this
+    schema or the lenient-parser will trip."""
+    p = CLAIM_LATTICE_JSON_SYSTEM_PROMPT
+    assert '"claims"' in p
+    assert '"text"' in p
+    assert '"evidence_ids"' in p
+
+
+def test_json_system_prompt_pins_first_last_char_rule():
+    """The grammar-constrained-inference contract: output must
+    start with `{` and end with `}` (no preamble, no markdown
+    fence). The pre-parser handles common drift, but the prompt
+    discipline is the first line of defense."""
+    p = CLAIM_LATTICE_JSON_SYSTEM_PROMPT
+    assert "first character is `{`" in p or "begins with `{`" in p
+    assert "last character is `}`" in p or "ends with `}`" in p
+
+
+def test_json_system_prompt_pins_two_evidence_id_cap():
+    """Same two-pointer cap as the pointer-mode prompt — folded
+    into governance_policy_hash via claim_lattice_max_pointers_per_claim."""
+    p = CLAIM_LATTICE_JSON_SYSTEM_PROMPT.lower()
+    assert "two ids per claim" in p or "at most two" in p
+
+
+# --- JSON-mode grounding reminder ----------------------------------
+
+
+def test_json_reminder_nonempty():
+    assert CLAIM_LATTICE_JSON_GROUNDING_REMINDER
+    assert "REMINDER" in CLAIM_LATTICE_JSON_GROUNDING_REMINDER
+
+
+def test_json_reminder_pins_schema():
+    r = CLAIM_LATTICE_JSON_GROUNDING_REMINDER
+    assert '"claims"' in r
+    assert '"text"' in r
+    assert '"evidence_ids"' in r
+
+
+def test_json_reminder_pins_two_per_claim():
+    r = CLAIM_LATTICE_JSON_GROUNDING_REMINDER.lower()
+    assert "two per claim" in r or "at most two" in r
+
+
+# --- cross-prompt parity -------------------------------------------
+
+
+def test_pointer_and_json_prompts_share_cap():
+    """Both modes enforce the two-pointer cap. Failure of either
+    test above is the load-bearing signal; this is just a
+    cross-check that the two prompts reference the same number."""
+    pointer = CLAIM_LATTICE_SYSTEM_PROMPT.lower()
+    json_p = CLAIM_LATTICE_JSON_SYSTEM_PROMPT.lower()
+    # Both mention "two" in the cap context.
+    assert "two" in pointer
+    assert "two" in json_p
+
+
+def test_grounding_reminders_end_with_now_answer():
+    """Both reminders are user-turn messages that fire immediately
+    before the LLM generates. They must end by handing off to the
+    question on the next message."""
+    for rem in (CLAIM_LATTICE_GROUNDING_REMINDER,
+                CLAIM_LATTICE_JSON_GROUNDING_REMINDER):
+        assert "next message" in rem.lower()
+
+
+# --- import surface -----------------------------------------------
+
+
+def test_module_exports_four_constants():
+    """Hand-written check that the 4 documented constants are
+    importable. Import-error here means the module was renamed or
+    a constant got dropped."""
+    from arborist.qa import prompts
+
+    assert hasattr(prompts, "CLAIM_LATTICE_SYSTEM_PROMPT")
+    assert hasattr(prompts, "CLAIM_LATTICE_GROUNDING_REMINDER")
+    assert hasattr(prompts, "CLAIM_LATTICE_JSON_SYSTEM_PROMPT")
+    assert hasattr(prompts, "CLAIM_LATTICE_JSON_GROUNDING_REMINDER")
+
+
+def test_constants_are_strings():
+    for c in (
+        CLAIM_LATTICE_SYSTEM_PROMPT,
+        CLAIM_LATTICE_GROUNDING_REMINDER,
+        CLAIM_LATTICE_JSON_SYSTEM_PROMPT,
+        CLAIM_LATTICE_JSON_GROUNDING_REMINDER,
+    ):
+        assert isinstance(c, str)
+        assert len(c) > 0