arborist/tests/test_directives.py

"""Anti-regression tests for the seven-point program.

Each test pins a directive at the substrate level. If a future PR
silently weakens a directive, the test fails by name.

See `docs/seven-point-program.md` for the directive catalogue. Per-row
bench-side compliance lives in `bench/qa_sweep.py`; this file pins
the invariants in the test layer where they can't drift unnoticed.
"""
from __future__ import annotations

import inspect
import re

import pytest


# ---------------------------------------------------------------------------
# D1 — Stop making Hermes prove things
# ---------------------------------------------------------------------------


def test_d1_verify_quotes_signature_has_no_chat_client():
    """The hard verifier never accepts a chat-client / LLM judge.
    A future PR that adds a chat_client parameter to verify_quotes
    would silently introduce a soft-signal-as-hard-check leak."""
    from arborist.qa.verify import verify_quotes

    sig = inspect.signature(verify_quotes)
    forbidden = {"chat_client", "client", "llm", "judge", "model"}
    found = forbidden & set(sig.parameters.keys())
    assert not found, (
        f"verify_quotes accepts forbidden parameter(s) {found} — "
        f"directive D1 violation: hard verifier must never receive an "
        f"LLM judge."
    )


def test_d1_verify_claim_lattice_signature_has_no_chat_client():
    from arborist.qa.verify import verify_claim_lattice

    sig = inspect.signature(verify_claim_lattice)
    forbidden = {"chat_client", "client", "llm", "judge", "model"}
    found = forbidden & set(sig.parameters.keys())
    assert not found, (
        f"verify_claim_lattice accepts forbidden parameter(s) {found} "
        f"— directive D1 violation."
    )


def test_d1_verify_claim_lattice_json_signature_has_no_chat_client():
    from arborist.qa.verify import verify_claim_lattice_json

    sig = inspect.signature(verify_claim_lattice_json)
    forbidden = {"chat_client", "client", "llm", "judge", "model"}
    found = forbidden & set(sig.parameters.keys())
    assert not found, (
        f"verify_claim_lattice_json accepts forbidden parameter(s) "
        f"{found} — directive D1 violation."
    )


def test_d1_verifier_method_enum_excludes_llm_judges():
    """The schema CHECK constraint on `verifier_method` must not
    include any LLM-as-judge token. If a future migration adds
    `llm`, `model`, `judge`, or `nli` to the enum, the verifier
    started accepting soft signals into the hard chain."""
    from arborist.store import SCHEMA_SQL

    # Find the verifier_method CHECK constraint in SCHEMA_SQL.
    match = re.search(
        r"verifier_method\s+TEXT[^,]+CHECK\s*\(\s*verifier_method\s+IN\s*\(([^)]+)\)\)",
        SCHEMA_SQL,
        re.IGNORECASE,
    )
    assert match is not None, (
        "verifier_method CHECK constraint not found in SCHEMA_SQL — "
        "schema reshape that obscures the enum is itself a directive "
        "regression risk."
    )
    enum_values = [
        s.strip().strip("'\"") for s in match.group(1).split(",")
    ]
    forbidden = {"llm", "model", "judge", "nli", "hermes", "ai"}
    leaked = forbidden & set(enum_values)
    assert not leaked, (
        f"verifier_method enum leaked LLM-judge tokens {leaked} — "
        f"directive D1 violation."
    )


# ---------------------------------------------------------------------------
# D2 — Hermes emits pointer clauses
# ---------------------------------------------------------------------------


def test_d2_answer_modes_include_lattice_variants():
    """ANSWER_MODES must contain both lattice variants so an agent
    can pick a pointer-clause-emitting mode."""
    from arborist.qa.verify import ANSWER_MODES

    assert "claim_lattice_pointer" in ANSWER_MODES
    assert "claim_lattice" in ANSWER_MODES


def test_d2_default_answer_mode_is_a_known_mode():
    from arborist.qa.verify import ANSWER_MODES, DEFAULT_ANSWER_MODE

    assert DEFAULT_ANSWER_MODE in ANSWER_MODES


def test_d2_pointer_parser_exists_and_returns_claim_nodes():
    """The pointer-line parser is the surface that turns model
    output into internal claim nodes. Its absence would mean
    Hermes has nowhere to emit pointer clauses to."""
    from arborist.qa.parse_claims import parse_pointer_claims

    sample = "Steve Jobs co-founded Apple. [E1]\n"
    out = parse_pointer_claims(sample)
    assert len(out) == 1
    # Each parsed claim carries text + evidence pointer ids.
    assert hasattr(out[0], "claim_text")
    assert hasattr(out[0], "pointer_ids")
    assert "Steve Jobs" in out[0].claim_text
    assert out[0].pointer_ids == ["E1"]


# ---------------------------------------------------------------------------
# D3 — Build CTI internally
# ---------------------------------------------------------------------------


def test_d3_runtime_owns_claim_lattice_construction():
    """The runtime parses pointer-line prose into structured claim
    nodes. The MODEL never produces structured output directly in
    pointer mode — it produces prose that the runtime structures.
    A regression that asks the model for structured JSON in pointer
    mode would violate this directive."""
    from arborist.qa.parse_claims import parse_pointer_claims

    multi_line = (
        "Steve Jobs co-founded Apple. [E1]\n"
        "Steve Wozniak co-founded Apple. [E1,E2]\n"
        "Ronald Wayne co-founded Apple. [E1]\n"
    )
    out = parse_pointer_claims(multi_line)
    assert len(out) == 3
    # Each node is a tuple-shaped record, not a free-form dict.
    # The runtime owns the schema, not the model.
    for node in out:
        assert isinstance(node.claim_text, str)
        assert isinstance(node.pointer_ids, list)
        assert all(isinstance(p, str) for p in node.pointer_ids)


def test_d3_evidence_map_built_from_runtime_chunks_not_model_input():
    """The evidence map is constructed by the runtime from retrieved
    chunks, not threaded in from the model's prompt-side
    invention. Two chunks → two evidence objects with deterministic
    pointer ids."""
    from arborist.qa.evidence import build_evidence_map

    chunks = [
        {
            "source_root": "a" * 64,
            "document_uri": "u://a",
            "title": "A",
            "chunk_idx": 0,
            "chunk_root": "11" * 32,
            "span": "alpha",
            "source_role": "primary_answer_source",
        },
        {
            "source_root": "b" * 64,
            "document_uri": "u://b",
            "title": "B",
            "chunk_idx": 0,
            "chunk_root": "22" * 32,
            "span": "beta",
            "source_role": "primary_answer_source",
        },
    ]
    em = build_evidence_map(chunks)
    assert len(em) == 2
    # Pointer ids are minted by the runtime, sequentially.
    assert em[0].pointer_id == "E1"
    assert em[1].pointer_id == "E2"
    # Evidence ids are content-addressed, not model-supplied.
    assert em[0].evidence_id != em[1].evidence_id


# ---------------------------------------------------------------------------
# D4 — Bind retrieval map AND evidence map
# ---------------------------------------------------------------------------


def test_d4_run_dag_carries_evidence_map_root():
    """The 9-stage CTI run-DAG must commit the evidence_map_root.
    Without this, two runs whose retrieval surfaced different chunks
    would produce indistinguishable run_dag_roots."""
    from arborist.qa.dag import build_run_dag

    sig = inspect.signature(build_run_dag)
    assert "evidence_map_root" in sig.parameters, (
        "build_run_dag missing evidence_map_root parameter — "
        "directive D4 (evidence side) violation."
    )


def test_d4_retrieval_plan_binding_landed():
    """Retrieval-plan binding landed via ticket #000001. The
    `build_run_dag` retrieval stage now hashes both the plan
    (operator-influenceable inputs: keywords, top_k, over_fetch,
    max_context_chars, shard set) and the result (sources_summary).

    Two runs with identical sources but different retrieval keywords
    produce different `run_dag_root` values — provenance closes the
    'how did retrieval choose these sources' gap."""
    from arborist.qa.dag import build_run_dag

    sig = inspect.signature(build_run_dag)
    assert "retrieval_plan_hash" in sig.parameters, (
        "retrieval_plan_hash parameter expected on build_run_dag — "
        "ticket #000001 should have landed it."
    )

    # Confirm the retrieval-stage hash diverges when only the plan
    # differs. Hex-only stub values (q, x → not hex; use a-f, 0-9).
    base_kwargs = dict(
        question_hash="aa" * 32,
        sources=[{
            "document_root": "dd" * 32,
            "source_role": "primary_answer_source",
            "score": 1.0,
            "chunk_idx": 0,
        }],
        context_root="cc" * 32,
        conversation_hash="ee" * 32,
        answer_text="A.",
        audit_mode="STRICT",
        verifier_method="quote",
        n_quotes=1,
        n_verified=1,
    )
    dag_no_plan = build_run_dag(**base_kwargs)
    dag_with_plan_a = build_run_dag(**base_kwargs, retrieval_plan_hash="aa" * 32)
    dag_with_plan_b = build_run_dag(**base_kwargs, retrieval_plan_hash="bb" * 32)
    assert dag_with_plan_a["root"] != dag_no_plan["root"]
    assert dag_with_plan_a["root"] != dag_with_plan_b["root"]


def test_d4_retrieval_plan_hash_module_exists():
    """RetrievalPlan dataclass + retrieval_plan_hash function landed
    in `arborist.qa.retrieval_plan` per ticket #000001."""
    from arborist.qa.retrieval_plan import RetrievalPlan, retrieval_plan_hash

    plan = RetrievalPlan(
        retrieval_keywords="orwell 1984",
        top_k=8,
        over_fetch=32,
        max_context_chars=60000,
    )
    h = retrieval_plan_hash(plan)
    # 64-char hex string.
    assert len(h) == 64
    assert all(c in "0123456789abcdef" for c in h)
    # Deterministic: same plan → same hash.
    assert retrieval_plan_hash(plan) == h
    # Differs when input changes.
    plan_b = RetrievalPlan(
        retrieval_keywords="different",
        top_k=8,
        over_fetch=32,
        max_context_chars=60000,
    )
    assert retrieval_plan_hash(plan_b) != h


# ---------------------------------------------------------------------------
# D5 — Verify pointers deterministically
# ---------------------------------------------------------------------------


def test_d5_verify_claim_lattice_is_deterministic():
    """Same inputs → same verdict, byte-for-byte. No randomness, no
    time-dependent state, no hidden global. Regression here means
    the verifier started consulting some non-deterministic source."""
    from arborist.qa.evidence import build_evidence_map
    from arborist.qa.verify import verify_claim_lattice

    chunks = [{
        "source_root": "f" * 64,
        "document_uri": "u://f",
        "title": "F",
        "chunk_idx": 0,
        "chunk_root": "33" * 32,
        "span": "Brachiosaurus appears in the climactic scene.",
        "source_role": "primary_answer_source",
    }]
    em = build_evidence_map(chunks)
    answer = "Brachiosaurus appears in the film. [E1]\n"
    v1 = verify_claim_lattice(answer, em)
    v2 = verify_claim_lattice(answer, em)
    assert v1["audit_mode"] == v2["audit_mode"]
    assert v1["n_quotes"] == v2["n_quotes"]
    assert v1["n_verified"] == v2["n_verified"]


def test_d5_seven_hard_checks_are_pure_functions():
    """Spot check: the per-check helper for citation overlap
    (`_claim_textually_overlaps_evidence`) is a pure function — no
    network, no chat client, no global state."""
    from arborist.qa.verify import _claim_textually_overlaps_evidence

    sig = inspect.signature(_claim_textually_overlaps_evidence)
    forbidden = {"chat_client", "client", "llm", "judge", "model"}
    assert not (forbidden & set(sig.parameters.keys()))


# ---------------------------------------------------------------------------
# D6 — Anchor-class warrant before semantic NLI
# ---------------------------------------------------------------------------


def test_d6_warrant_check_exists_and_is_lexical():
    """warrant_check must exist as a deterministic lexical check.
    A future replacement that imports an NLI model into the hard
    path would violate D6's 'before NLI' constraint."""
    from arborist.qa.warrant import warrant_check

    sig = inspect.signature(warrant_check)
    forbidden = {"chat_client", "client", "llm", "judge", "model", "nli"}
    found = forbidden & set(sig.parameters.keys())
    assert not found, (
        f"warrant_check accepts forbidden parameter(s) {found} — "
        f"directive D6 mandates lexical-only warrant check."
    )


def test_d6_warrant_fires_on_date_anchor_mismatch():
    """Existing warrant-lite catches the date-anchor lazy-anchor
    class. This case is the deterministic anchor (year strings
    are case-trivial; ALL-match semantics). Regression here means
    the date-anchor extractor or check silently broke."""
    from arborist.qa.warrant import warrant_check

    # Claim asserts "1985"; cited span has no 1985 anywhere.
    ok, missing = warrant_check(
        claim_text="Back to the Future was released in theaters in 1985.",
        cited_spans=[
            "An unrelated chunk about a film released in 1955 "
            "discussing trilogy mechanics and pinball adaptations."
        ],
        question="what year was back to the future released?",
    )
    assert not ok, (
        "warrant should fail when claim asserts year 1985 but no "
        "cited span contains 1985"
    )
    assert "1985" in " ".join(missing)


def test_d6_warrant_relation_shape_fires_when_all_anchors_missing():
    """Proper-noun anchor uses ANY-match semantics: at least one
    extracted anchor must appear in some cited span. This test
    exercises a true failure where the claim's only proper-noun
    anchor is missing from every cited span."""
    from arborist.qa.warrant import warrant_check

    ok, missing = warrant_check(
        claim_text="Mr. Burns is the boss.",
        cited_spans=[
            "Springfield Nuclear Power Plant employs many workers in "
            "the maintenance and reactor sections of the facility."
        ],
        question="who is the boss?",
    )
    assert not ok, (
        "warrant should fail when claim's only proper-noun anchor "
        "is missing from every cited span"
    )
    assert any("Burns" in m for m in missing)


def test_d6_warrant_generalization_landed():
    """Per-shape warrant detectors landed via ticket #000003.
    Entity-list / count / why-cause shapes now dispatch through
    warrant_check alongside the original relation + date classes."""
    import arborist.qa.warrant as warrant_mod

    expected = (
        "_question_is_entity_list_shape",
        "_question_is_count_shape",
        "_question_is_why_shape",
        "extract_count_anchors",
        "extract_cause_anchors",
    )
    missing = [
        name for name in expected if not hasattr(warrant_mod, name)
    ]
    assert missing == [], (
        f"warrant module missing expected detectors / extractors "
        f"{missing} — ticket #000003 marker should have all five."
    )


# ---------------------------------------------------------------------------
# D7 — Rename labels honestly
# ---------------------------------------------------------------------------


def test_d7_renderer_relabels_strict_for_claim_lattice():
    """Renderer must map STRICT → ladder rung for claim-lattice
    methods. A regression here would put STRICT on synthesis-heavy
    claims, overclaiming semantic truth. The four-rung ladder
    (POINTER-LINKED / ANCHOR-WARRANTED / EVIDENCE-WARRANTED) replaces
    the previous two-rung EVIDENCE-LINKED/EVIDENCE-LINKED-PARTIAL
    surface as of #000005."""
    from arborist.cli import _render_audit_label

    # No violations + STRICT → top rung.
    assert "EVIDENCE-WARRANTED" in _render_audit_label("STRICT", "claim_lattice", [])
    assert "EVIDENCE-WARRANTED" in _render_audit_label("STRICT", "claim_lattice_pointer", [])
    # WARRANT_MISSING → POINTER-LINKED rung.
    assert "POINTER-LINKED" in _render_audit_label(
        "STRICT", "claim_lattice", [{"kind": "WARRANT_MISSING"}]
    )


def test_d7_renderer_keeps_strict_for_pinned_span_methods():
    """Quote / span / entity / paraphrase verify against pinned
    spans, not synthesis. STRICT is honest there; renderer must
    NOT apply the ladder relabel."""
    from arborist.cli import _render_audit_label

    for method in ("quote", "span", "entity", "paraphrase"):
        label = _render_audit_label("STRICT", method, [])
        assert "STRICT" in label, (
            f"verifier_method={method} STRICT relabeled — D7 says "
            f"only claim-lattice methods get the ladder rename."
        )
        for rung in ("POINTER-LINKED", "ANCHOR-WARRANTED", "EVIDENCE-WARRANTED", "EVIDENCE-LINKED"):
            assert rung not in label


def test_d7_audit_mode_enum_canonical_set():
    """Schema column must keep the canonical 3-value enum so v9.8
    cache_key invariants hold. Renderer-level relabel (above)
    doesn't touch this."""
    from arborist.store import SCHEMA_SQL

    match = re.search(
        r"audit_mode\s+TEXT[^,]+CHECK\s*\(\s*audit_mode\s+IN\s*\(([^)]+)\)\)",
        SCHEMA_SQL,
        re.IGNORECASE,
    )
    assert match is not None
    enum_values = sorted(
        s.strip().strip("'\"") for s in match.group(1).split(",")
    )
    assert enum_values == ["HYBRID", "STRICT", "UNGROUNDED"], (
        f"audit_mode enum drifted to {enum_values} — schema column "
        f"must stay {{STRICT, HYBRID, UNGROUNDED}}; rendered labels "
        f"are display-layer only."
    )


# ---------------------------------------------------------------------------
# D8 — Automate only after the invariants are test-pinned
# ---------------------------------------------------------------------------


def test_d8_seven_point_program_doc_exists():
    """The directive catalogue must exist and be discoverable from
    CLAUDE.md. If someone deletes the program doc but leaves the
    bench coverage column referencing it, the substrate stops
    documenting what it enforces."""
    from pathlib import Path

    repo_root = Path(__file__).parent.parent
    program_doc = repo_root / "docs" / "seven-point-program.md"
    assert program_doc.exists(), (
        f"docs/seven-point-program.md missing — directive D8 "
        f"discipline relies on the program doc as the audit lens. "
        f"Restore from git or open a ticket explaining the new "
        f"governance scheme."
    )


def test_d8_bench_directive_compliance_helper_exists():
    """The bench harness must carry the directive_compliance helper.
    Removing it would silently delete the per-run governance signal."""
    import importlib.util
    from pathlib import Path

    repo_root = Path(__file__).parent.parent
    spec = importlib.util.spec_from_file_location(
        "qa_sweep_dir_check", repo_root / "bench" / "qa_sweep.py"
    )
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    assert hasattr(mod, "_directive_compliance"), (
        "bench/qa_sweep.py missing _directive_compliance helper — "
        "D8 enforcement substrate gone."
    )


def test_d8_tickets_index_exists_and_pins_open_directives():
    """TICKETS.md must list every open ticket that maps to a
    partial directive. If a partial directive has no open ticket,
    the design log is lying about the work-in-progress state."""
    from pathlib import Path

    repo_root = Path(__file__).parent.parent
    tickets = (repo_root / "docs" / "TICKETS.md").read_text()
    program = (repo_root / "docs" / "seven-point-program.md").read_text()

    # Every partial directive (½) in the program doc must have at
    # least one open ticket referenced. Today's partials: D3, D4, D6.
    # Each of those should appear in the tickets index Directive
    # column, mapped to an open ticket.
    for directive_id in ("D3", "D4", "D6"):
        # Program doc lists the directive as ½.
        assert f"| {directive_id[1]} |" in program or directive_id in program
        # Tickets index references that directive.
        assert directive_id in tickets, (
            f"partial directive {directive_id} missing from TICKETS.md "
            f"index — open ticket should map to it."
        )