arborist/tests/test_metacognition.py

"""Meta-Cognition Preflight Guard — Ticket #000010 Phase 1 tests.

Pin the seven test cases from the source doc §14 plus per-detector
unit tests. All checks are deterministic; no LLM, no I/O.

Source: ``~/Downloads/meta-cognition_for_hermes(1).txt`` 2026-05-03.
"""

from __future__ import annotations

import pytest

from arborist.qa.metacognition import (
    PREFLIGHT_VERSION,
    QuestionState,
    detect_contradiction,
    detect_false_premise,
    detect_out_of_corpus,
    detect_temporal_sensitivity,
    preflight_question,
)


# ---------------------------------------------------------------- temporal

@pytest.mark.parametrize("q", [
    "Who is the current CEO of OpenAI?",
    "What is the latest version of Python?",
    "Who is the President of France today?",
    "What is the current price of Bitcoin?",
    "Who won the championship this year?",
    "As of right now, who holds the world record?",
])
def test_temporal_high_sensitivity(q):
    sens, matched = detect_temporal_sensitivity(q)
    assert sens == "high", f"{q} → {sens}"
    assert len(matched) >= 1


@pytest.mark.parametrize("q", [
    "what is the capital of france?",
    "who painted the mona lisa?",
    "what is the speed of light?",
    "who wrote the play hamlet?",
])
def test_temporal_low_sensitivity_for_factoids(q):
    sens, _ = detect_temporal_sensitivity(q)
    assert sens == "low"


# ---------------------------------------------------------------- contradiction

def test_contradiction_unmarried_spouse():
    pairs = detect_contradiction("Which unmarried spouse is Homer married to?")
    # `unmarried` + `spouse` AND `unmarried` + `married` both fire.
    pair_set = set(pairs)
    assert ("unmarried", "spouse") in pair_set
    assert ("unmarried", "married") in pair_set


def test_contradiction_alive_dead():
    pairs = detect_contradiction("Which character is alive and dead simultaneously?")
    assert ("alive", "dead") in set(pairs)


def test_contradiction_no_false_positives_on_singletons():
    """`unmarried` alone shouldn't fire — needs both halves of a pair."""
    pairs = detect_contradiction("Is George Washington still unmarried?")
    assert pairs == ()


def test_contradiction_clean_questions_have_no_pairs():
    pairs = detect_contradiction("who painted the mona lisa?")
    assert pairs == ()


# ---------------------------------------------------------------- false premise

def test_false_premise_when_did_x_stop_y():
    hints = detect_false_premise("when did Mr. Burns stop being Homer's father?")
    assert len(hints) >= 1
    h = hints[0]
    assert h["kind"] == "stopped_doing"
    assert "Mr. Burns" in h["subject"]


def test_false_premise_why_did_x_cause_y():
    hints = detect_false_premise("why did the moon cause the tides?")
    assert len(hints) >= 1
    assert hints[0]["kind"] == "caused"


def test_false_premise_how_did_x_become_y():
    hints = detect_false_premise("how did Mr. Burns become Homer's biological father?")
    assert len(hints) >= 1
    assert hints[0]["kind"] == "became"
    assert "Mr. Burns" in hints[0]["subject"]


def test_false_premise_no_hint_on_neutral_question():
    hints = detect_false_premise("who is bilbo baggins?")
    assert hints == ()


# ---------------------------------------------------------------- out of corpus

@pytest.mark.parametrize("q", [
    "What does my uploaded contract say about clause 9?",
    "What is in my private notes from yesterday?",
    "What does the file I sent you say about X?",
    "In my email inbox, what did Alice write?",
])
def test_out_of_corpus_detected(q):
    assert detect_out_of_corpus(q) is True


@pytest.mark.parametrize("q", [
    "What is the capital of France?",
    "Who wrote Hamlet?",
    "What is the speed of light?",
])
def test_out_of_corpus_not_detected_on_encyclopedic(q):
    assert detect_out_of_corpus(q) is False


# ---------------------------------------------------------------- preflight (full integration)

def test_preflight_factoid_is_well_formed():
    state = preflight_question("what is the capital of france?")
    assert "well_formed" in state.logical_statuses
    assert state.preflight_result == "PREFLIGHT_OK"
    assert state.question_shape in ("single_fact", "negation")


def test_preflight_broad_quantifier_unbounded():
    """Source doc §5.3 + #000008 §10.1: `winners of all major sports?`
    classifies as broad-quantifier-unbounded."""
    state = preflight_question("Winners of all major sports?")
    assert "broad_quantifier_unbounded" in state.logical_statuses
    assert state.preflight_result == "PREFLIGHT_PARTIAL"
    assert state.quantifier_intensity == "ALL"
    assert state.scope_bound_hint == "unbounded"


def test_preflight_broad_quantifier_bounded_does_not_fire_unbounded():
    """`name all members of the beatles` is a bounded universal.
    Should classify as `under_specified` (broad but bounded), NOT
    `broad_quantifier_unbounded`."""
    state = preflight_question("name all members of the beatles")
    assert "broad_quantifier_unbounded" not in state.logical_statuses
    assert state.scope_bound_hint == "bounded"


def test_preflight_contradictory_question():
    """Source doc §5.2: `Which unmarried spouse is Homer married to?`
    classifies as contradictory."""
    state = preflight_question("Which unmarried spouse is Homer married to?")
    assert "contradictory_question" in state.logical_statuses
    assert state.contradiction_pairs  # at least one pair


def test_preflight_false_premise_suspected():
    """Source doc §5.1: `When did Mr. Burns become Homer's biological
    father?` classifies as false_premise_suspected."""
    state = preflight_question(
        "When did Mr. Burns become Homer's biological father?"
    )
    assert "false_premise_suspected" in state.logical_statuses
    assert state.false_premise_hints


def test_preflight_time_sensitive_stale_risk():
    """Source doc §5.4: `Who is the current CEO of X?` classifies
    as stale_risk."""
    state = preflight_question("Who is the current CEO of OpenAI?")
    assert "stale_risk" in state.logical_statuses
    assert state.temporal_sensitivity == "high"
    assert state.answer_constraints.get("requires_current_source") is True


def test_preflight_out_of_corpus_blocked():
    """Source doc §5.5: out-of-corpus references should BLOCK."""
    state = preflight_question("What does my uploaded contract say?")
    assert "out_of_corpus_risk" in state.logical_statuses
    assert state.preflight_result == "PREFLIGHT_BLOCKED"


def test_preflight_reference_frame_ambiguous():
    """Source doc §5.6: when caller passes multiple reference frames,
    classifier marks reference_frame_ambiguous."""
    state = preflight_question(
        "Has Oceania always been at war with East Asia?",
        reference_frames=("literal_geography", "orwell_1984"),
    )
    assert "reference_frame_ambiguous" in state.logical_statuses


# ---------------------------------------------------------------- gating

def test_master_kill_disables_preflight():
    """policy={'metacognition_enabled': False} short-circuits with
    a stub QuestionState. Logical statuses tuple is empty;
    preflight_result is PREFLIGHT_OK so downstream isn't blocked."""
    state = preflight_question(
        "Winners of all major sports?",
        policy={"metacognition_enabled": False},
    )
    assert state.logical_statuses == ()
    assert state.preflight_result == "PREFLIGHT_OK"
    assert state.question_shape == "metacognition_disabled"


def test_per_detector_disable():
    """Each detector can be turned off independently. Verify the
    contradiction check obeys its switch."""
    state = preflight_question(
        "Which unmarried spouse is Homer married to?",
        policy={"metacognition_contradiction_check": False},
    )
    # Contradiction detector skipped → no contradictory_question status.
    assert "contradictory_question" not in state.logical_statuses
    assert state.contradiction_pairs == ()


def test_block_on_contradiction_opt_in():
    """By default contradictory questions surface as PARTIAL (label
    only). `metacognition_block_on_contradiction=True` flips to
    BLOCKED."""
    state_default = preflight_question(
        "Which unmarried spouse is Homer married to?",
    )
    state_strict = preflight_question(
        "Which unmarried spouse is Homer married to?",
        policy={"metacognition_block_on_contradiction": True},
    )
    assert state_default.preflight_result == "PREFLIGHT_PARTIAL"
    assert state_strict.preflight_result == "PREFLIGHT_BLOCKED"


# ---------------------------------------------------------------- determinism

def test_question_hash_stable():
    """Same question (modulo case + whitespace) → same hash."""
    a = preflight_question("Who painted the Mona Lisa?")
    b = preflight_question("  who painted the mona lisa? ")
    assert a.question_hash == b.question_hash


def test_preflight_policy_hash_changes_with_policy_flip():
    """Flipping a metacognition policy field bumps the policy hash —
    enables governance binding via _VERIFIER_POLICY_FIELDS (Phase 3)."""
    a = preflight_question(
        "Winners of all major sports?",
        policy={"metacognition_temporal_check": True},
    )
    b = preflight_question(
        "Winners of all major sports?",
        policy={"metacognition_temporal_check": False},
    )
    assert a.preflight_policy_hash != b.preflight_policy_hash


# ---------------------------------------------------------------- versioning

def test_preflight_version_pinned():
    state = preflight_question("any question")
    assert state.classifier_version == PREFLIGHT_VERSION
    assert PREFLIGHT_VERSION == "metacognition-v0.1"


# ---------------------------------------------------------------- empty / edge

def test_empty_question_blocked():
    state = preflight_question("")
    assert state.preflight_result == "PREFLIGHT_BLOCKED"
    assert state.question_shape == "empty"


def test_whitespace_only_question_blocked():
    state = preflight_question("   \n\t   ")
    assert state.preflight_result == "PREFLIGHT_BLOCKED"


def test_question_state_is_serializable():
    """to_dict() must produce JSON-friendly output for run-DAG /
    bench JSONL persistence."""
    import json
    state = preflight_question("Winners of all major sports?")
    payload = state.to_dict()
    # Round-trip through json.dumps; failure here means a non-
    # serializable type leaked in.
    json.dumps(payload, ensure_ascii=False)


# ---------------------------------------------------------------- governance binding (Phase 3)

@pytest.mark.parametrize("field", [
    "metacognition_enabled",
    "metacognition_temporal_check",
    "metacognition_contradiction_check",
    "metacognition_false_premise_check",
    "metacognition_out_of_corpus_check",
    "metacognition_block_on_contradiction",
])
def test_metacognition_field_is_in_verifier_policy_fields(field):
    """All six policy fields must be in _VERIFIER_POLICY_FIELDS so
    flipping any of them invalidates prior cache records on lookup."""
    from arborist.qa.keys import _VERIFIER_POLICY_FIELDS
    assert field in _VERIFIER_POLICY_FIELDS


def test_governance_hash_changes_when_metacognition_enabled_flips():
    from arborist.qa.keys import _VERIFIER_POLICY_FIELDS, verifier_policy_hash
    base = dict.fromkeys(_VERIFIER_POLICY_FIELDS, "default")
    base["metacognition_enabled"] = True
    h_on = verifier_policy_hash(base)
    base["metacognition_enabled"] = False
    h_off = verifier_policy_hash(base)
    assert h_on != h_off


def test_governance_hash_changes_when_block_on_contradiction_flips():
    from arborist.qa.keys import _VERIFIER_POLICY_FIELDS, verifier_policy_hash
    base = dict.fromkeys(_VERIFIER_POLICY_FIELDS, "default")
    base["metacognition_block_on_contradiction"] = False
    h_off = verifier_policy_hash(base)
    base["metacognition_block_on_contradiction"] = True
    h_on = verifier_policy_hash(base)
    assert h_off != h_on


def test_default_policy_has_metacognition_enabled():
    """Master switch default-on per ticket #000010 §7.3 — detectors
    are pure-on-question so the cost is negligible."""
    from arborist.qa.runner import DEFAULT_POLICY as RUNNER_POLICY
    from arborist.qa.query import DEFAULT_QUERY_POLICY
    assert RUNNER_POLICY["metacognition_enabled"] is True
    assert DEFAULT_QUERY_POLICY["metacognition_enabled"] is True


def test_default_policy_block_on_contradiction_off():
    """Default to label-only on contradictions. False-positive risk
    not yet bench-validated; opt-in via --block-on-contradiction."""
    from arborist.qa.runner import DEFAULT_POLICY as RUNNER_POLICY
    from arborist.qa.query import DEFAULT_QUERY_POLICY
    assert RUNNER_POLICY["metacognition_block_on_contradiction"] is False
    assert DEFAULT_QUERY_POLICY["metacognition_block_on_contradiction"] is False


# ---------------------------------------------------------------- audit-line tails (Phase 4)

def _result_with_question_state(state: QuestionState) -> dict:
    """Synthetic query() result with the minimum fields needed by
    `_render_warrant_tail`."""
    return {
        "violations": [],
        "claim_cap_applied": None,
        "question_state": state.to_dict(),
    }


def test_tail_renders_false_premise():
    from arborist.cli import _render_warrant_tail
    state = preflight_question(
        "When did Mr. Burns become Homer's biological father?"
    )
    tail = _render_warrant_tail(_result_with_question_state(state))
    assert "false premise" in tail


def test_tail_renders_contradictory():
    from arborist.cli import _render_warrant_tail
    state = preflight_question("Which unmarried spouse is Homer married to?")
    tail = _render_warrant_tail(_result_with_question_state(state))
    assert "contradictory" in tail


def test_tail_renders_stale_risk():
    from arborist.cli import _render_warrant_tail
    state = preflight_question("Who is the current CEO of OpenAI?")
    tail = _render_warrant_tail(_result_with_question_state(state))
    assert "stale risk" in tail


def test_tail_renders_out_of_corpus():
    from arborist.cli import _render_warrant_tail
    state = preflight_question("What does my uploaded contract say?")
    tail = _render_warrant_tail(_result_with_question_state(state))
    assert "out of corpus" in tail


def test_tail_omits_metacog_when_well_formed():
    from arborist.cli import _render_warrant_tail
    state = preflight_question("what is the capital of france?")
    tail = _render_warrant_tail(_result_with_question_state(state))
    # well_formed should NOT produce any metacog tail tokens.
    assert "false premise" not in tail
    assert "contradictory" not in tail
    assert "stale risk" not in tail
    assert "out of corpus" not in tail


def test_tail_combines_metacog_with_existing_kinds():
    """Multiple tails compose: a TITLE_MISMATCH from the verifier
    plus a stale_risk from the preflight should both surface."""
    from arborist.cli import _render_warrant_tail
    state = preflight_question("Who is the current CEO of OpenAI?")
    result = _result_with_question_state(state)
    result["violations"] = [{"kind": "TITLE_MISMATCH"}]
    tail = _render_warrant_tail(result)
    assert "title mismatch" in tail
    assert "stale risk" in tail