"""Meta-Cognition Preflight Guard — Ticket #000010 Phase 1 tests. Pin the seven test cases from the source doc §14 plus per-detector unit tests. All checks are deterministic; no LLM, no I/O. Source: ``~/Downloads/meta-cognition_for_hermes(1).txt`` 2026-05-03. """ from __future__ import annotations import pytest from arborist.qa.metacognition import ( PREFLIGHT_VERSION, QuestionState, detect_contradiction, detect_false_premise, detect_out_of_corpus, detect_temporal_sensitivity, preflight_question, ) # ---------------------------------------------------------------- temporal @pytest.mark.parametrize("q", [ "Who is the current CEO of OpenAI?", "What is the latest version of Python?", "Who is the President of France today?", "What is the current price of Bitcoin?", "Who won the championship this year?", "As of right now, who holds the world record?", ]) def test_temporal_high_sensitivity(q): sens, matched = detect_temporal_sensitivity(q) assert sens == "high", f"{q} → {sens}" assert len(matched) >= 1 @pytest.mark.parametrize("q", [ "what is the capital of france?", "who painted the mona lisa?", "what is the speed of light?", "who wrote the play hamlet?", ]) def test_temporal_low_sensitivity_for_factoids(q): sens, _ = detect_temporal_sensitivity(q) assert sens == "low" # ---------------------------------------------------------------- contradiction def test_contradiction_unmarried_spouse(): pairs = detect_contradiction("Which unmarried spouse is Homer married to?") # `unmarried` + `spouse` AND `unmarried` + `married` both fire. pair_set = set(pairs) assert ("unmarried", "spouse") in pair_set assert ("unmarried", "married") in pair_set def test_contradiction_alive_dead(): pairs = detect_contradiction("Which character is alive and dead simultaneously?") assert ("alive", "dead") in set(pairs) def test_contradiction_no_false_positives_on_singletons(): """`unmarried` alone shouldn't fire — needs both halves of a pair.""" pairs = detect_contradiction("Is George Washington still unmarried?") assert pairs == () def test_contradiction_clean_questions_have_no_pairs(): pairs = detect_contradiction("who painted the mona lisa?") assert pairs == () # ---------------------------------------------------------------- false premise def test_false_premise_when_did_x_stop_y(): hints = detect_false_premise("when did Mr. Burns stop being Homer's father?") assert len(hints) >= 1 h = hints[0] assert h["kind"] == "stopped_doing" assert "Mr. Burns" in h["subject"] def test_false_premise_why_did_x_cause_y(): hints = detect_false_premise("why did the moon cause the tides?") assert len(hints) >= 1 assert hints[0]["kind"] == "caused" def test_false_premise_how_did_x_become_y(): hints = detect_false_premise("how did Mr. Burns become Homer's biological father?") assert len(hints) >= 1 assert hints[0]["kind"] == "became" assert "Mr. Burns" in hints[0]["subject"] def test_false_premise_no_hint_on_neutral_question(): hints = detect_false_premise("who is bilbo baggins?") assert hints == () # ---------------------------------------------------------------- out of corpus @pytest.mark.parametrize("q", [ "What does my uploaded contract say about clause 9?", "What is in my private notes from yesterday?", "What does the file I sent you say about X?", "In my email inbox, what did Alice write?", ]) def test_out_of_corpus_detected(q): assert detect_out_of_corpus(q) is True @pytest.mark.parametrize("q", [ "What is the capital of France?", "Who wrote Hamlet?", "What is the speed of light?", ]) def test_out_of_corpus_not_detected_on_encyclopedic(q): assert detect_out_of_corpus(q) is False # ---------------------------------------------------------------- preflight (full integration) def test_preflight_factoid_is_well_formed(): state = preflight_question("what is the capital of france?") assert "well_formed" in state.logical_statuses assert state.preflight_result == "PREFLIGHT_OK" assert state.question_shape in ("single_fact", "negation") def test_preflight_broad_quantifier_unbounded(): """Source doc §5.3 + #000008 §10.1: `winners of all major sports?` classifies as broad-quantifier-unbounded.""" state = preflight_question("Winners of all major sports?") assert "broad_quantifier_unbounded" in state.logical_statuses assert state.preflight_result == "PREFLIGHT_PARTIAL" assert state.quantifier_intensity == "ALL" assert state.scope_bound_hint == "unbounded" def test_preflight_broad_quantifier_bounded_does_not_fire_unbounded(): """`name all members of the beatles` is a bounded universal. Should classify as `under_specified` (broad but bounded), NOT `broad_quantifier_unbounded`.""" state = preflight_question("name all members of the beatles") assert "broad_quantifier_unbounded" not in state.logical_statuses assert state.scope_bound_hint == "bounded" def test_preflight_contradictory_question(): """Source doc §5.2: `Which unmarried spouse is Homer married to?` classifies as contradictory.""" state = preflight_question("Which unmarried spouse is Homer married to?") assert "contradictory_question" in state.logical_statuses assert state.contradiction_pairs # at least one pair def test_preflight_false_premise_suspected(): """Source doc §5.1: `When did Mr. Burns become Homer's biological father?` classifies as false_premise_suspected.""" state = preflight_question( "When did Mr. Burns become Homer's biological father?" ) assert "false_premise_suspected" in state.logical_statuses assert state.false_premise_hints def test_preflight_time_sensitive_stale_risk(): """Source doc §5.4: `Who is the current CEO of X?` classifies as stale_risk.""" state = preflight_question("Who is the current CEO of OpenAI?") assert "stale_risk" in state.logical_statuses assert state.temporal_sensitivity == "high" assert state.answer_constraints.get("requires_current_source") is True def test_preflight_out_of_corpus_blocked(): """Source doc §5.5: out-of-corpus references should BLOCK.""" state = preflight_question("What does my uploaded contract say?") assert "out_of_corpus_risk" in state.logical_statuses assert state.preflight_result == "PREFLIGHT_BLOCKED" def test_preflight_reference_frame_ambiguous(): """Source doc §5.6: when caller passes multiple reference frames, classifier marks reference_frame_ambiguous.""" state = preflight_question( "Has Oceania always been at war with East Asia?", reference_frames=("literal_geography", "orwell_1984"), ) assert "reference_frame_ambiguous" in state.logical_statuses # ---------------------------------------------------------------- gating def test_master_kill_disables_preflight(): """policy={'metacognition_enabled': False} short-circuits with a stub QuestionState. Logical statuses tuple is empty; preflight_result is PREFLIGHT_OK so downstream isn't blocked.""" state = preflight_question( "Winners of all major sports?", policy={"metacognition_enabled": False}, ) assert state.logical_statuses == () assert state.preflight_result == "PREFLIGHT_OK" assert state.question_shape == "metacognition_disabled" def test_per_detector_disable(): """Each detector can be turned off independently. Verify the contradiction check obeys its switch.""" state = preflight_question( "Which unmarried spouse is Homer married to?", policy={"metacognition_contradiction_check": False}, ) # Contradiction detector skipped → no contradictory_question status. assert "contradictory_question" not in state.logical_statuses assert state.contradiction_pairs == () def test_block_on_contradiction_opt_in(): """By default contradictory questions surface as PARTIAL (label only). `metacognition_block_on_contradiction=True` flips to BLOCKED.""" state_default = preflight_question( "Which unmarried spouse is Homer married to?", ) state_strict = preflight_question( "Which unmarried spouse is Homer married to?", policy={"metacognition_block_on_contradiction": True}, ) assert state_default.preflight_result == "PREFLIGHT_PARTIAL" assert state_strict.preflight_result == "PREFLIGHT_BLOCKED" # ---------------------------------------------------------------- determinism def test_question_hash_stable(): """Same question (modulo case + whitespace) → same hash.""" a = preflight_question("Who painted the Mona Lisa?") b = preflight_question(" who painted the mona lisa? ") assert a.question_hash == b.question_hash def test_preflight_policy_hash_changes_with_policy_flip(): """Flipping a metacognition policy field bumps the policy hash — enables governance binding via _VERIFIER_POLICY_FIELDS (Phase 3).""" a = preflight_question( "Winners of all major sports?", policy={"metacognition_temporal_check": True}, ) b = preflight_question( "Winners of all major sports?", policy={"metacognition_temporal_check": False}, ) assert a.preflight_policy_hash != b.preflight_policy_hash # ---------------------------------------------------------------- versioning def test_preflight_version_pinned(): state = preflight_question("any question") assert state.classifier_version == PREFLIGHT_VERSION assert PREFLIGHT_VERSION == "metacognition-v0.1" # ---------------------------------------------------------------- empty / edge def test_empty_question_blocked(): state = preflight_question("") assert state.preflight_result == "PREFLIGHT_BLOCKED" assert state.question_shape == "empty" def test_whitespace_only_question_blocked(): state = preflight_question(" \n\t ") assert state.preflight_result == "PREFLIGHT_BLOCKED" def test_question_state_is_serializable(): """to_dict() must produce JSON-friendly output for run-DAG / bench JSONL persistence.""" import json state = preflight_question("Winners of all major sports?") payload = state.to_dict() # Round-trip through json.dumps; failure here means a non- # serializable type leaked in. json.dumps(payload, ensure_ascii=False) # ---------------------------------------------------------------- governance binding (Phase 3) @pytest.mark.parametrize("field", [ "metacognition_enabled", "metacognition_temporal_check", "metacognition_contradiction_check", "metacognition_false_premise_check", "metacognition_out_of_corpus_check", "metacognition_block_on_contradiction", ]) def test_metacognition_field_is_in_verifier_policy_fields(field): """All six policy fields must be in _VERIFIER_POLICY_FIELDS so flipping any of them invalidates prior cache records on lookup.""" from arborist.qa.keys import _VERIFIER_POLICY_FIELDS assert field in _VERIFIER_POLICY_FIELDS def test_governance_hash_changes_when_metacognition_enabled_flips(): from arborist.qa.keys import _VERIFIER_POLICY_FIELDS, verifier_policy_hash base = dict.fromkeys(_VERIFIER_POLICY_FIELDS, "default") base["metacognition_enabled"] = True h_on = verifier_policy_hash(base) base["metacognition_enabled"] = False h_off = verifier_policy_hash(base) assert h_on != h_off def test_governance_hash_changes_when_block_on_contradiction_flips(): from arborist.qa.keys import _VERIFIER_POLICY_FIELDS, verifier_policy_hash base = dict.fromkeys(_VERIFIER_POLICY_FIELDS, "default") base["metacognition_block_on_contradiction"] = False h_off = verifier_policy_hash(base) base["metacognition_block_on_contradiction"] = True h_on = verifier_policy_hash(base) assert h_off != h_on def test_default_policy_has_metacognition_enabled(): """Master switch default-on per ticket #000010 §7.3 — detectors are pure-on-question so the cost is negligible.""" from arborist.qa.runner import DEFAULT_POLICY as RUNNER_POLICY from arborist.qa.query import DEFAULT_QUERY_POLICY assert RUNNER_POLICY["metacognition_enabled"] is True assert DEFAULT_QUERY_POLICY["metacognition_enabled"] is True def test_default_policy_block_on_contradiction_off(): """Default to label-only on contradictions. False-positive risk not yet bench-validated; opt-in via --block-on-contradiction.""" from arborist.qa.runner import DEFAULT_POLICY as RUNNER_POLICY from arborist.qa.query import DEFAULT_QUERY_POLICY assert RUNNER_POLICY["metacognition_block_on_contradiction"] is False assert DEFAULT_QUERY_POLICY["metacognition_block_on_contradiction"] is False # ---------------------------------------------------------------- audit-line tails (Phase 4) def _result_with_question_state(state: QuestionState) -> dict: """Synthetic query() result with the minimum fields needed by `_render_warrant_tail`.""" return { "violations": [], "claim_cap_applied": None, "question_state": state.to_dict(), } def test_tail_renders_false_premise(): from arborist.cli import _render_warrant_tail state = preflight_question( "When did Mr. Burns become Homer's biological father?" ) tail = _render_warrant_tail(_result_with_question_state(state)) assert "false premise" in tail def test_tail_renders_contradictory(): from arborist.cli import _render_warrant_tail state = preflight_question("Which unmarried spouse is Homer married to?") tail = _render_warrant_tail(_result_with_question_state(state)) assert "contradictory" in tail def test_tail_renders_stale_risk(): from arborist.cli import _render_warrant_tail state = preflight_question("Who is the current CEO of OpenAI?") tail = _render_warrant_tail(_result_with_question_state(state)) assert "stale risk" in tail def test_tail_renders_out_of_corpus(): from arborist.cli import _render_warrant_tail state = preflight_question("What does my uploaded contract say?") tail = _render_warrant_tail(_result_with_question_state(state)) assert "out of corpus" in tail def test_tail_omits_metacog_when_well_formed(): from arborist.cli import _render_warrant_tail state = preflight_question("what is the capital of france?") tail = _render_warrant_tail(_result_with_question_state(state)) # well_formed should NOT produce any metacog tail tokens. assert "false premise" not in tail assert "contradictory" not in tail assert "stale risk" not in tail assert "out of corpus" not in tail def test_tail_combines_metacog_with_existing_kinds(): """Multiple tails compose: a TITLE_MISMATCH from the verifier plus a stale_risk from the preflight should both surface.""" from arborist.cli import _render_warrant_tail state = preflight_question("Who is the current CEO of OpenAI?") result = _result_with_question_state(state) result["violations"] = [{"kind": "TITLE_MISMATCH"}] tail = _render_warrant_tail(result) assert "title mismatch" in tail assert "stale risk" in tail