arborist/tests/test_qa_prompts.py

"""Tests for ``arborist.qa.prompts`` — load-bearing system prompts
+ user-turn grounding reminders for claim_lattice modes.

The prompts ARE the contract with the LLM. A silent edit that
removes a critical instruction (e.g. drops the "at most two
pointers per claim" rule) can drop STRICT-rate by tens of points
on bench fixtures. These tests pin the contract phrases as
regression guards.

Per CLAUDE.md "bench-maxing — measure deltas, not opinions":
prompt regressions are silent in unit tests but loud in bench;
this file moves the regression guard into a per-PR check.
"""

from __future__ import annotations

from arborist.qa.prompts import (
    CLAIM_LATTICE_GROUNDING_REMINDER,
    CLAIM_LATTICE_JSON_GROUNDING_REMINDER,
    CLAIM_LATTICE_JSON_SYSTEM_PROMPT,
    CLAIM_LATTICE_SYSTEM_PROMPT,
)


# --- pointer-mode system prompt ------------------------------------


def test_pointer_system_prompt_nonempty():
    assert CLAIM_LATTICE_SYSTEM_PROMPT
    assert len(CLAIM_LATTICE_SYSTEM_PROMPT) > 100  # substantial


def test_pointer_system_prompt_contains_worked_examples():
    """Two worked examples (Apple founders + Mars descriptive)
    are documented as load-bearing for prompt scaffolding. If a
    PR drops them, this test fires."""
    assert "WORKED EXAMPLE 1" in CLAIM_LATTICE_SYSTEM_PROMPT
    assert "WORKED EXAMPLE 2" in CLAIM_LATTICE_SYSTEM_PROMPT
    assert "END OF EXAMPLES" in CLAIM_LATTICE_SYSTEM_PROMPT


def test_pointer_system_prompt_pins_two_pointer_cap():
    """'At most two pointers per claim' is the cache-key-affecting
    rule that maps to claim_lattice_max_pointers_per_claim policy
    field. Removing the prompt instruction without bumping the
    governance_policy_hash field would silently disagree."""
    p = CLAIM_LATTICE_SYSTEM_PROMPT.lower()
    assert "two pointer" in p or "two pointers" in p


def test_pointer_system_prompt_documents_pointer_id_shape():
    """Pointer IDs are E1, E2, E3, etc. Prompt must teach the
    bracket-tag shape `[E#]` or `[E#,E#]` so the parser's regex
    matches model output."""
    p = CLAIM_LATTICE_SYSTEM_PROMPT
    assert "E1" in p
    assert "[E" in p  # bracket-tag form
    assert "E2" in p
    assert "E3" in p


def test_pointer_system_prompt_includes_one_claim_per_line_rule():
    """Parser splits on newlines; prompt must teach the model to
    emit one claim per line."""
    p = CLAIM_LATTICE_SYSTEM_PROMPT.lower()
    assert "one claim per" in p or "one plain-prose sentence" in p


def test_pointer_system_prompt_no_quote_instruction():
    """Pointer mode is the synthetic-elision-by-construction-
    impossible path: the model NEVER types the quote string;
    the runtime interpolates literal spans. The prompt must NOT
    instruct the model to wrap claims in double quotes (that's
    the legacy quote-mode prompt)."""
    p = CLAIM_LATTICE_SYSTEM_PROMPT.lower()
    # Looking for an instruction like "wrap each claim in
    # double quotes". The phrase 'verbatim quote' / 'enclosed in
    # double quotes' is the legacy quote-mode pattern; ensure
    # absent.
    assert 'wrap every factual claim in double quotes' not in p
    assert 'wrap each claim in double quotes' not in p


# --- pointer-mode grounding reminder -------------------------------


def test_pointer_reminder_nonempty():
    assert CLAIM_LATTICE_GROUNDING_REMINDER
    assert "REMINDER" in CLAIM_LATTICE_GROUNDING_REMINDER


def test_pointer_reminder_pins_pointer_line_format():
    """The reminder is the user-turn restatement that fires
    immediately before the LLM generates. It must restate the
    pointer-line format."""
    r = CLAIM_LATTICE_GROUNDING_REMINDER.lower()
    assert "pointer-line" in r or "pointer line" in r
    assert "[e1]" in r or "[e#]" in r or "[e1,e#]" in r


def test_pointer_reminder_restates_two_pointer_cap():
    r = CLAIM_LATTICE_GROUNDING_REMINDER.lower()
    assert "two pointer" in r or "at most two" in r


# --- JSON-mode system prompt ---------------------------------------


def test_json_system_prompt_nonempty():
    assert CLAIM_LATTICE_JSON_SYSTEM_PROMPT
    assert len(CLAIM_LATTICE_JSON_SYSTEM_PROMPT) > 100


def test_json_system_prompt_pins_schema_shape():
    """The verifier accepts the JSON shape `{"claims":[{"text":...,
    "evidence_ids":[...]}]}`. The prompt must teach exactly this
    schema or the lenient-parser will trip."""
    p = CLAIM_LATTICE_JSON_SYSTEM_PROMPT
    assert '"claims"' in p
    assert '"text"' in p
    assert '"evidence_ids"' in p


def test_json_system_prompt_pins_first_last_char_rule():
    """The grammar-constrained-inference contract: output must
    start with `{` and end with `}` (no preamble, no markdown
    fence). The pre-parser handles common drift, but the prompt
    discipline is the first line of defense."""
    p = CLAIM_LATTICE_JSON_SYSTEM_PROMPT
    assert "first character is `{`" in p or "begins with `{`" in p
    assert "last character is `}`" in p or "ends with `}`" in p


def test_json_system_prompt_pins_two_evidence_id_cap():
    """Same two-pointer cap as the pointer-mode prompt — folded
    into governance_policy_hash via claim_lattice_max_pointers_per_claim."""
    p = CLAIM_LATTICE_JSON_SYSTEM_PROMPT.lower()
    assert "two ids per claim" in p or "at most two" in p


# --- JSON-mode grounding reminder ----------------------------------


def test_json_reminder_nonempty():
    assert CLAIM_LATTICE_JSON_GROUNDING_REMINDER
    assert "REMINDER" in CLAIM_LATTICE_JSON_GROUNDING_REMINDER


def test_json_reminder_pins_schema():
    r = CLAIM_LATTICE_JSON_GROUNDING_REMINDER
    assert '"claims"' in r
    assert '"text"' in r
    assert '"evidence_ids"' in r


def test_json_reminder_pins_two_per_claim():
    r = CLAIM_LATTICE_JSON_GROUNDING_REMINDER.lower()
    assert "two per claim" in r or "at most two" in r


# --- cross-prompt parity -------------------------------------------


def test_pointer_and_json_prompts_share_cap():
    """Both modes enforce the two-pointer cap. Failure of either
    test above is the load-bearing signal; this is just a
    cross-check that the two prompts reference the same number."""
    pointer = CLAIM_LATTICE_SYSTEM_PROMPT.lower()
    json_p = CLAIM_LATTICE_JSON_SYSTEM_PROMPT.lower()
    # Both mention "two" in the cap context.
    assert "two" in pointer
    assert "two" in json_p


def test_grounding_reminders_end_with_now_answer():
    """Both reminders are user-turn messages that fire immediately
    before the LLM generates. They must end by handing off to the
    question on the next message."""
    for rem in (CLAIM_LATTICE_GROUNDING_REMINDER,
                CLAIM_LATTICE_JSON_GROUNDING_REMINDER):
        assert "next message" in rem.lower()


# --- import surface -----------------------------------------------


def test_module_exports_four_constants():
    """Hand-written check that the 4 documented constants are
    importable. Import-error here means the module was renamed or
    a constant got dropped."""
    from arborist.qa import prompts

    assert hasattr(prompts, "CLAIM_LATTICE_SYSTEM_PROMPT")
    assert hasattr(prompts, "CLAIM_LATTICE_GROUNDING_REMINDER")
    assert hasattr(prompts, "CLAIM_LATTICE_JSON_SYSTEM_PROMPT")
    assert hasattr(prompts, "CLAIM_LATTICE_JSON_GROUNDING_REMINDER")


def test_constants_are_strings():
    for c in (
        CLAIM_LATTICE_SYSTEM_PROMPT,
        CLAIM_LATTICE_GROUNDING_REMINDER,
        CLAIM_LATTICE_JSON_SYSTEM_PROMPT,
        CLAIM_LATTICE_JSON_GROUNDING_REMINDER,
    ):
        assert isinstance(c, str)
        assert len(c) > 0