arborist/tests/test_directives.py

"""Anti-regression tests for the seven-point program.

Each test pins a directive at the substrate level. If a future PR
silently weakens a directive, the test fails by name.

See `docs/seven-point-program.md` for the directive catalogue. Per-row
bench-side compliance lives in `bench/qa_sweep.py`; this file pins
the invariants in the test layer where they can't drift unnoticed.
"""
from __future__ import annotations

import inspect
import re

import pytest


# ---------------------------------------------------------------------------
# D1 — Stop making Hermes prove things
# ---------------------------------------------------------------------------


def test_d1_verify_quotes_signature_has_no_chat_client():
    """The hard verifier never accepts a chat-client / LLM judge.
    A future PR that adds a chat_client parameter to verify_quotes
    would silently introduce a soft-signal-as-hard-check leak."""
    from aborist.qa.verify import verify_quotes

    sig = inspect.signature(verify_quotes)
    forbidden = {"chat_client", "client", "llm", "judge", "model"}
    found = forbidden & set(sig.parameters.keys())
    assert not found, (
        f"verify_quotes accepts forbidden parameter(s) {found} — "
        f"directive D1 violation: hard verifier must never receive an "
        f"LLM judge."
    )


def test_d1_verify_claim_lattice_signature_has_no_chat_client():
    from aborist.qa.verify import verify_claim_lattice

    sig = inspect.signature(verify_claim_lattice)
    forbidden = {"chat_client", "client", "llm", "judge", "model"}
    found = forbidden & set(sig.parameters.keys())
    assert not found, (
        f"verify_claim_lattice accepts forbidden parameter(s) {found} "
        f"— directive D1 violation."
    )


def test_d1_verify_claim_lattice_json_signature_has_no_chat_client():
    from aborist.qa.verify import verify_claim_lattice_json

    sig = inspect.signature(verify_claim_lattice_json)
    forbidden = {"chat_client", "client", "llm", "judge", "model"}
    found = forbidden & set(sig.parameters.keys())
    assert not found, (
        f"verify_claim_lattice_json accepts forbidden parameter(s) "
        f"{found} — directive D1 violation."
    )


def test_d1_verifier_method_enum_excludes_llm_judges():
    """The schema CHECK constraint on `verifier_method` must not
    include any LLM-as-judge token. If a future migration adds
    `llm`, `model`, `judge`, or `nli` to the enum, the verifier
    started accepting soft signals into the hard chain."""
    from aborist.store import SCHEMA_SQL

    # Find the verifier_method CHECK constraint in SCHEMA_SQL.
    match = re.search(
        r"verifier_method\s+TEXT[^,]+CHECK\s*\(\s*verifier_method\s+IN\s*\(([^)]+)\)\)",
        SCHEMA_SQL,
        re.IGNORECASE,
    )
    assert match is not None, (
        "verifier_method CHECK constraint not found in SCHEMA_SQL — "
        "schema reshape that obscures the enum is itself a directive "
        "regression risk."
    )
    enum_values = [
        s.strip().strip("'\"") for s in match.group(1).split(",")
    ]
    forbidden = {"llm", "model", "judge", "nli", "hermes", "ai"}
    leaked = forbidden & set(enum_values)
    assert not leaked, (
        f"verifier_method enum leaked LLM-judge tokens {leaked} — "
        f"directive D1 violation."
    )


# ---------------------------------------------------------------------------
# D2 — Hermes emits pointer clauses
# ---------------------------------------------------------------------------


def test_d2_answer_modes_include_lattice_variants():
    """ANSWER_MODES must contain both lattice variants so an agent
    can pick a pointer-clause-emitting mode."""
    from aborist.qa.verify import ANSWER_MODES

    assert "claim_lattice_pointer" in ANSWER_MODES
    assert "claim_lattice" in ANSWER_MODES


def test_d2_default_answer_mode_is_a_known_mode():
    from aborist.qa.verify import ANSWER_MODES, DEFAULT_ANSWER_MODE

    assert DEFAULT_ANSWER_MODE in ANSWER_MODES


def test_d2_pointer_parser_exists_and_returns_claim_nodes():
    """The pointer-line parser is the surface that turns model
    output into internal claim nodes. Its absence would mean
    Hermes has nowhere to emit pointer clauses to."""
    from aborist.qa.parse_claims import parse_pointer_claims

    sample = "Steve Jobs co-founded Apple. [E1]\n"
    out = parse_pointer_claims(sample)
    assert len(out) == 1
    # Each parsed claim carries text + evidence pointer ids.
    assert hasattr(out[0], "claim_text")
    assert hasattr(out[0], "pointer_ids")
    assert "Steve Jobs" in out[0].claim_text
    assert out[0].pointer_ids == ["E1"]


# ---------------------------------------------------------------------------
# D3 — Build CTI internally
# ---------------------------------------------------------------------------


def test_d3_runtime_owns_claim_lattice_construction():
    """The runtime parses pointer-line prose into structured claim
    nodes. The MODEL never produces structured output directly in
    pointer mode — it produces prose that the runtime structures.
    A regression that asks the model for structured JSON in pointer
    mode would violate this directive."""
    from aborist.qa.parse_claims import parse_pointer_claims

    multi_line = (
        "Steve Jobs co-founded Apple. [E1]\n"
        "Steve Wozniak co-founded Apple. [E1,E2]\n"
        "Ronald Wayne co-founded Apple. [E1]\n"
    )
    out = parse_pointer_claims(multi_line)
    assert len(out) == 3
    # Each node is a tuple-shaped record, not a free-form dict.
    # The runtime owns the schema, not the model.
    for node in out:
        assert isinstance(node.claim_text, str)
        assert isinstance(node.pointer_ids, list)
        assert all(isinstance(p, str) for p in node.pointer_ids)


def test_d3_evidence_map_built_from_runtime_chunks_not_model_input():
    """The evidence map is constructed by the runtime from retrieved
    chunks, not threaded in from the model's prompt-side
    invention. Two chunks → two evidence objects with deterministic
    pointer ids."""
    from aborist.qa.evidence import build_evidence_map

    chunks = [
        {
            "source_root": "a" * 64,
            "document_uri": "u://a",
            "title": "A",
            "chunk_idx": 0,
            "chunk_root": "11" * 32,
            "span": "alpha",
            "source_role": "primary_answer_source",
        },
        {
            "source_root": "b" * 64,
            "document_uri": "u://b",
            "title": "B",
            "chunk_idx": 0,
            "chunk_root": "22" * 32,
            "span": "beta",
            "source_role": "primary_answer_source",
        },
    ]
    em = build_evidence_map(chunks)
    assert len(em) == 2
    # Pointer ids are minted by the runtime, sequentially.
    assert em[0].pointer_id == "E1"
    assert em[1].pointer_id == "E2"
    # Evidence ids are content-addressed, not model-supplied.
    assert em[0].evidence_id != em[1].evidence_id


# ---------------------------------------------------------------------------
# D4 — Bind retrieval map AND evidence map
# ---------------------------------------------------------------------------


def test_d4_run_dag_carries_evidence_map_root():
    """The 9-stage CTI run-DAG must commit the evidence_map_root.
    Without this, two runs whose retrieval surfaced different chunks
    would produce indistinguishable run_dag_roots."""
    from aborist.qa.dag import build_run_dag

    sig = inspect.signature(build_run_dag)
    assert "evidence_map_root" in sig.parameters, (
        "build_run_dag missing evidence_map_root parameter — "
        "directive D4 (evidence side) violation."
    )


def test_d4_retrieval_plan_binding_status():
    """Marker: retrieval-plan binding (the input side of D4) is
    pending via ticket #000001. This test documents the open gap;
    flip the assertion when ticket #000001 lands.

    Today's `build_run_dag` does NOT accept a `retrieval_plan_hash`
    parameter. When it does, this test should INVERT (assert the
    parameter exists)."""
    from aborist.qa.dag import build_run_dag

    sig = inspect.signature(build_run_dag)
    # As of 2026-05-01: parameter not present. When ticket #000001
    # lands, the lines below flip from `not in` to `in`.
    assert "retrieval_plan_hash" not in sig.parameters, (
        "retrieval_plan_hash parameter detected — ticket #000001 has "
        "landed; flip this assertion to the positive form and remove "
        "the marker."
    )


# ---------------------------------------------------------------------------
# D5 — Verify pointers deterministically
# ---------------------------------------------------------------------------


def test_d5_verify_claim_lattice_is_deterministic():
    """Same inputs → same verdict, byte-for-byte. No randomness, no
    time-dependent state, no hidden global. Regression here means
    the verifier started consulting some non-deterministic source."""
    from aborist.qa.evidence import build_evidence_map
    from aborist.qa.verify import verify_claim_lattice

    chunks = [{
        "source_root": "f" * 64,
        "document_uri": "u://f",
        "title": "F",
        "chunk_idx": 0,
        "chunk_root": "33" * 32,
        "span": "Brachiosaurus appears in the climactic scene.",
        "source_role": "primary_answer_source",
    }]
    em = build_evidence_map(chunks)
    answer = "Brachiosaurus appears in the film. [E1]\n"
    v1 = verify_claim_lattice(answer, em)
    v2 = verify_claim_lattice(answer, em)
    assert v1["audit_mode"] == v2["audit_mode"]
    assert v1["n_quotes"] == v2["n_quotes"]
    assert v1["n_verified"] == v2["n_verified"]


def test_d5_seven_hard_checks_are_pure_functions():
    """Spot check: the per-check helper for citation overlap
    (`_claim_textually_overlaps_evidence`) is a pure function — no
    network, no chat client, no global state."""
    from aborist.qa.verify import _claim_textually_overlaps_evidence

    sig = inspect.signature(_claim_textually_overlaps_evidence)
    forbidden = {"chat_client", "client", "llm", "judge", "model"}
    assert not (forbidden & set(sig.parameters.keys()))


# ---------------------------------------------------------------------------
# D6 — Anchor-class warrant before semantic NLI
# ---------------------------------------------------------------------------


def test_d6_warrant_check_exists_and_is_lexical():
    """warrant_check must exist as a deterministic lexical check.
    A future replacement that imports an NLI model into the hard
    path would violate D6's 'before NLI' constraint."""
    from aborist.qa.warrant import warrant_check

    sig = inspect.signature(warrant_check)
    forbidden = {"chat_client", "client", "llm", "judge", "model", "nli"}
    found = forbidden & set(sig.parameters.keys())
    assert not found, (
        f"warrant_check accepts forbidden parameter(s) {found} — "
        f"directive D6 mandates lexical-only warrant check."
    )


def test_d6_warrant_fires_on_date_anchor_mismatch():
    """Existing warrant-lite catches the date-anchor lazy-anchor
    class. This case is the deterministic anchor (year strings
    are case-trivial; ALL-match semantics). Regression here means
    the date-anchor extractor or check silently broke."""
    from aborist.qa.warrant import warrant_check

    # Claim asserts "1985"; cited span has no 1985 anywhere.
    ok, missing = warrant_check(
        claim_text="Back to the Future was released in theaters in 1985.",
        cited_spans=[
            "An unrelated chunk about a film released in 1955 "
            "discussing trilogy mechanics and pinball adaptations."
        ],
        question="what year was back to the future released?",
    )
    assert not ok, (
        "warrant should fail when claim asserts year 1985 but no "
        "cited span contains 1985"
    )
    assert "1985" in " ".join(missing)


def test_d6_warrant_relation_shape_fires_when_all_anchors_missing():
    """Proper-noun anchor uses ANY-match semantics: at least one
    extracted anchor must appear in some cited span. This test
    exercises a true failure where the claim's only proper-noun
    anchor is missing from every cited span."""
    from aborist.qa.warrant import warrant_check

    ok, missing = warrant_check(
        claim_text="Mr. Burns is the boss.",
        cited_spans=[
            "Springfield Nuclear Power Plant employs many workers in "
            "the maintenance and reactor sections of the facility."
        ],
        question="who is the boss?",
    )
    assert not ok, (
        "warrant should fail when claim's only proper-noun anchor "
        "is missing from every cited span"
    )
    assert any("Burns" in m for m in missing)


def test_d6_warrant_generalization_landed():
    """Per-shape warrant detectors landed via ticket #000003.
    Entity-list / count / why-cause shapes now dispatch through
    warrant_check alongside the original relation + date classes."""
    import aborist.qa.warrant as warrant_mod

    expected = (
        "_question_is_entity_list_shape",
        "_question_is_count_shape",
        "_question_is_why_shape",
        "extract_count_anchors",
        "extract_cause_anchors",
    )
    missing = [
        name for name in expected if not hasattr(warrant_mod, name)
    ]
    assert missing == [], (
        f"warrant module missing expected detectors / extractors "
        f"{missing} — ticket #000003 marker should have all five."
    )


# ---------------------------------------------------------------------------
# D7 — Rename labels honestly
# ---------------------------------------------------------------------------


def test_d7_renderer_relabels_strict_for_claim_lattice():
    """Renderer must map STRICT → EVIDENCE-LINKED for claim-lattice
    methods. A regression here would put STRICT on synthesis-heavy
    claims, overclaiming semantic truth."""
    from aborist.cli import _render_audit_label

    assert "EVIDENCE-LINKED" in _render_audit_label("STRICT", "claim_lattice")
    assert "EVIDENCE-LINKED" in _render_audit_label("STRICT", "claim_lattice_pointer")


def test_d7_renderer_keeps_strict_for_pinned_span_methods():
    """Quote / span / entity / paraphrase verify against pinned
    spans, not synthesis. STRICT is honest there; renderer must
    NOT relabel."""
    from aborist.cli import _render_audit_label

    for method in ("quote", "span", "entity", "paraphrase"):
        label = _render_audit_label("STRICT", method)
        assert "STRICT" in label, (
            f"verifier_method={method} STRICT relabeled — D7 says "
            f"only claim-lattice methods get the EVIDENCE-LINKED rename."
        )
        assert "EVIDENCE-LINKED" not in label


def test_d7_audit_mode_enum_canonical_set():
    """Schema column must keep the canonical 3-value enum so v9.8
    cache_key invariants hold. Renderer-level relabel (above)
    doesn't touch this."""
    from aborist.store import SCHEMA_SQL

    match = re.search(
        r"audit_mode\s+TEXT[^,]+CHECK\s*\(\s*audit_mode\s+IN\s*\(([^)]+)\)\)",
        SCHEMA_SQL,
        re.IGNORECASE,
    )
    assert match is not None
    enum_values = sorted(
        s.strip().strip("'\"") for s in match.group(1).split(",")
    )
    assert enum_values == ["HYBRID", "STRICT", "UNGROUNDED"], (
        f"audit_mode enum drifted to {enum_values} — schema column "
        f"must stay {{STRICT, HYBRID, UNGROUNDED}}; rendered labels "
        f"are display-layer only."
    )


# ---------------------------------------------------------------------------
# D8 — Automate only after the invariants are test-pinned
# ---------------------------------------------------------------------------


def test_d8_seven_point_program_doc_exists():
    """The directive catalogue must exist and be discoverable from
    CLAUDE.md. If someone deletes the program doc but leaves the
    bench coverage column referencing it, the substrate stops
    documenting what it enforces."""
    from pathlib import Path

    repo_root = Path(__file__).parent.parent
    program_doc = repo_root / "docs" / "seven-point-program.md"
    assert program_doc.exists(), (
        f"docs/seven-point-program.md missing — directive D8 "
        f"discipline relies on the program doc as the audit lens. "
        f"Restore from git or open a ticket explaining the new "
        f"governance scheme."
    )


def test_d8_bench_directive_compliance_helper_exists():
    """The bench harness must carry the directive_compliance helper.
    Removing it would silently delete the per-run governance signal."""
    import importlib.util
    from pathlib import Path

    repo_root = Path(__file__).parent.parent
    spec = importlib.util.spec_from_file_location(
        "qa_sweep_dir_check", repo_root / "bench" / "qa_sweep.py"
    )
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    assert hasattr(mod, "_directive_compliance"), (
        "bench/qa_sweep.py missing _directive_compliance helper — "
        "D8 enforcement substrate gone."
    )


def test_d8_tickets_index_exists_and_pins_open_directives():
    """TICKETS.md must list every open ticket that maps to a
    partial directive. If a partial directive has no open ticket,
    the design log is lying about the work-in-progress state."""
    from pathlib import Path

    repo_root = Path(__file__).parent.parent
    tickets = (repo_root / "docs" / "TICKETS.md").read_text()
    program = (repo_root / "docs" / "seven-point-program.md").read_text()

    # Every partial directive (½) in the program doc must have at
    # least one open ticket referenced. Today's partials: D3, D4, D6.
    # Each of those should appear in the tickets index Directive
    # column, mapped to an open ticket.
    for directive_id in ("D3", "D4", "D6"):
        # Program doc lists the directive as ½.
        assert f"| {directive_id[1]} |" in program or directive_id in program
        # Tickets index references that directive.
        assert directive_id in tickets, (
            f"partial directive {directive_id} missing from TICKETS.md "
            f"index — open ticket should map to it."
        )