From 4fada8e22eeabfc034980b600fdb448541a8217b Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Fri, 1 May 2026 16:32:08 -0400 Subject: [PATCH] tests: anti-regression layer for the seven-point program (23 tests) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each directive gets a structural pin in tests/test_directives.py. A future PR that silently weakens a directive fails by name. D1 (no LLM in hard verifier path): - verify_quotes / verify_claim_lattice / verify_claim_lattice_json signatures must NOT carry chat_client / llm / judge / model parameters. - SCHEMA_SQL verifier_method CHECK constraint enum excludes 'llm', 'model', 'judge', 'nli', 'hermes', 'ai'. D2 (Hermes emits pointer clauses): - ANSWER_MODES contains both lattice variants. - DEFAULT_ANSWER_MODE in ANSWER_MODES. - parse_pointer_claims returns structured (claim_text, pointer_ids) nodes from pointer-line input. D3 (build CTI internally): - Runtime parses pointer-line prose into typed claim nodes (text + pointer_ids list); model never produces structured output directly in pointer mode. - Evidence map built by runtime from retrieved chunks; pointer ids minted sequentially; evidence ids content-addressed. D4 (bind retrieval AND evidence map): - build_run_dag accepts evidence_map_root parameter (✓). - Marker test: build_run_dag does NOT yet accept retrieval_plan_hash parameter; flip when ticket #000001 lands. D5 (deterministic pointer verify): - verify_claim_lattice is deterministic: same inputs → same verdict byte-for-byte. - _claim_textually_overlaps_evidence (the per-pair coverage check) is a pure function with no LLM-side parameters. D6 (anchor-class warrant before NLI): - warrant_check signature is lexical-only (no chat_client / llm / judge / nli params). - Date-anchor failure case fires (year missing from cited spans). - Relation-shape failure case fires when ALL proper-noun anchors are missing from cited spans. - Marker test: per-shape detectors for entity-list / count / why-cause shapes are absent today; flip when ticket #000003 lands. D7 (rename labels honestly): - Renderer maps STRICT → EVIDENCE-LINKED for claim_lattice* methods. - Renderer keeps STRICT for quote / span / entity / paraphrase methods (those verify against pinned spans, not synthesis). - SCHEMA_SQL audit_mode CHECK constraint enum stays {STRICT, HYBRID, UNGROUNDED} regardless of renderer relabel. D8 (test-pinning before automation): - docs/seven-point-program.md exists. - bench/qa_sweep.py exposes _directive_compliance helper. - docs/TICKETS.md indexes the partial directives (D3, D4, D6) so the design log doesn't drift from the program doc. 23 new tests; full suite 692 passed (was 669, +23). The "marker" tests for D4 and D6 document open work: today they assert that a parameter / detector is ABSENT; when the corresponding ticket lands, the assertion flips to PRESENT. The test failing during ticket implementation is the expected signal, not a regression — the test message names the ticket so the work-in- progress is legible. --- tests/test_directives.py | 473 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 473 insertions(+) create mode 100644 tests/test_directives.py diff --git a/tests/test_directives.py b/tests/test_directives.py new file mode 100644 index 0000000..690ec53 --- /dev/null +++ b/tests/test_directives.py @@ -0,0 +1,473 @@ +"""Anti-regression tests for the seven-point program. + +Each test pins a directive at the substrate level. If a future PR +silently weakens a directive, the test fails by name. + +See `docs/seven-point-program.md` for the directive catalogue. Per-row +bench-side compliance lives in `bench/qa_sweep.py`; this file pins +the invariants in the test layer where they can't drift unnoticed. +""" +from __future__ import annotations + +import inspect +import re + +import pytest + + +# --------------------------------------------------------------------------- +# D1 — Stop making Hermes prove things +# --------------------------------------------------------------------------- + + +def test_d1_verify_quotes_signature_has_no_chat_client(): + """The hard verifier never accepts a chat-client / LLM judge. + A future PR that adds a chat_client parameter to verify_quotes + would silently introduce a soft-signal-as-hard-check leak.""" + from aborist.qa.verify import verify_quotes + + sig = inspect.signature(verify_quotes) + forbidden = {"chat_client", "client", "llm", "judge", "model"} + found = forbidden & set(sig.parameters.keys()) + assert not found, ( + f"verify_quotes accepts forbidden parameter(s) {found} — " + f"directive D1 violation: hard verifier must never receive an " + f"LLM judge." + ) + + +def test_d1_verify_claim_lattice_signature_has_no_chat_client(): + from aborist.qa.verify import verify_claim_lattice + + sig = inspect.signature(verify_claim_lattice) + forbidden = {"chat_client", "client", "llm", "judge", "model"} + found = forbidden & set(sig.parameters.keys()) + assert not found, ( + f"verify_claim_lattice accepts forbidden parameter(s) {found} " + f"— directive D1 violation." + ) + + +def test_d1_verify_claim_lattice_json_signature_has_no_chat_client(): + from aborist.qa.verify import verify_claim_lattice_json + + sig = inspect.signature(verify_claim_lattice_json) + forbidden = {"chat_client", "client", "llm", "judge", "model"} + found = forbidden & set(sig.parameters.keys()) + assert not found, ( + f"verify_claim_lattice_json accepts forbidden parameter(s) " + f"{found} — directive D1 violation." + ) + + +def test_d1_verifier_method_enum_excludes_llm_judges(): + """The schema CHECK constraint on `verifier_method` must not + include any LLM-as-judge token. If a future migration adds + `llm`, `model`, `judge`, or `nli` to the enum, the verifier + started accepting soft signals into the hard chain.""" + from aborist.store import SCHEMA_SQL + + # Find the verifier_method CHECK constraint in SCHEMA_SQL. + match = re.search( + r"verifier_method\s+TEXT[^,]+CHECK\s*\(\s*verifier_method\s+IN\s*\(([^)]+)\)\)", + SCHEMA_SQL, + re.IGNORECASE, + ) + assert match is not None, ( + "verifier_method CHECK constraint not found in SCHEMA_SQL — " + "schema reshape that obscures the enum is itself a directive " + "regression risk." + ) + enum_values = [ + s.strip().strip("'\"") for s in match.group(1).split(",") + ] + forbidden = {"llm", "model", "judge", "nli", "hermes", "ai"} + leaked = forbidden & set(enum_values) + assert not leaked, ( + f"verifier_method enum leaked LLM-judge tokens {leaked} — " + f"directive D1 violation." + ) + + +# --------------------------------------------------------------------------- +# D2 — Hermes emits pointer clauses +# --------------------------------------------------------------------------- + + +def test_d2_answer_modes_include_lattice_variants(): + """ANSWER_MODES must contain both lattice variants so an agent + can pick a pointer-clause-emitting mode.""" + from aborist.qa.verify import ANSWER_MODES + + assert "claim_lattice_pointer" in ANSWER_MODES + assert "claim_lattice" in ANSWER_MODES + + +def test_d2_default_answer_mode_is_a_known_mode(): + from aborist.qa.verify import ANSWER_MODES, DEFAULT_ANSWER_MODE + + assert DEFAULT_ANSWER_MODE in ANSWER_MODES + + +def test_d2_pointer_parser_exists_and_returns_claim_nodes(): + """The pointer-line parser is the surface that turns model + output into internal claim nodes. Its absence would mean + Hermes has nowhere to emit pointer clauses to.""" + from aborist.qa.parse_claims import parse_pointer_claims + + sample = "Steve Jobs co-founded Apple. [E1]\n" + out = parse_pointer_claims(sample) + assert len(out) == 1 + # Each parsed claim carries text + evidence pointer ids. + assert hasattr(out[0], "claim_text") + assert hasattr(out[0], "pointer_ids") + assert "Steve Jobs" in out[0].claim_text + assert out[0].pointer_ids == ["E1"] + + +# --------------------------------------------------------------------------- +# D3 — Build CTI internally +# --------------------------------------------------------------------------- + + +def test_d3_runtime_owns_claim_lattice_construction(): + """The runtime parses pointer-line prose into structured claim + nodes. The MODEL never produces structured output directly in + pointer mode — it produces prose that the runtime structures. + A regression that asks the model for structured JSON in pointer + mode would violate this directive.""" + from aborist.qa.parse_claims import parse_pointer_claims + + multi_line = ( + "Steve Jobs co-founded Apple. [E1]\n" + "Steve Wozniak co-founded Apple. [E1,E2]\n" + "Ronald Wayne co-founded Apple. [E1]\n" + ) + out = parse_pointer_claims(multi_line) + assert len(out) == 3 + # Each node is a tuple-shaped record, not a free-form dict. + # The runtime owns the schema, not the model. + for node in out: + assert isinstance(node.claim_text, str) + assert isinstance(node.pointer_ids, list) + assert all(isinstance(p, str) for p in node.pointer_ids) + + +def test_d3_evidence_map_built_from_runtime_chunks_not_model_input(): + """The evidence map is constructed by the runtime from retrieved + chunks, not threaded in from the model's prompt-side + invention. Two chunks → two evidence objects with deterministic + pointer ids.""" + from aborist.qa.evidence import build_evidence_map + + chunks = [ + { + "source_root": "a" * 64, + "document_uri": "u://a", + "title": "A", + "chunk_idx": 0, + "chunk_root": "11" * 32, + "span": "alpha", + "source_role": "primary_answer_source", + }, + { + "source_root": "b" * 64, + "document_uri": "u://b", + "title": "B", + "chunk_idx": 0, + "chunk_root": "22" * 32, + "span": "beta", + "source_role": "primary_answer_source", + }, + ] + em = build_evidence_map(chunks) + assert len(em) == 2 + # Pointer ids are minted by the runtime, sequentially. + assert em[0].pointer_id == "E1" + assert em[1].pointer_id == "E2" + # Evidence ids are content-addressed, not model-supplied. + assert em[0].evidence_id != em[1].evidence_id + + +# --------------------------------------------------------------------------- +# D4 — Bind retrieval map AND evidence map +# --------------------------------------------------------------------------- + + +def test_d4_run_dag_carries_evidence_map_root(): + """The 9-stage CTI run-DAG must commit the evidence_map_root. + Without this, two runs whose retrieval surfaced different chunks + would produce indistinguishable run_dag_roots.""" + from aborist.qa.dag import build_run_dag + + sig = inspect.signature(build_run_dag) + assert "evidence_map_root" in sig.parameters, ( + "build_run_dag missing evidence_map_root parameter — " + "directive D4 (evidence side) violation." + ) + + +def test_d4_retrieval_plan_binding_status(): + """Marker: retrieval-plan binding (the input side of D4) is + pending via ticket #000001. This test documents the open gap; + flip the assertion when ticket #000001 lands. + + Today's `build_run_dag` does NOT accept a `retrieval_plan_hash` + parameter. When it does, this test should INVERT (assert the + parameter exists).""" + from aborist.qa.dag import build_run_dag + + sig = inspect.signature(build_run_dag) + # As of 2026-05-01: parameter not present. When ticket #000001 + # lands, the lines below flip from `not in` to `in`. + assert "retrieval_plan_hash" not in sig.parameters, ( + "retrieval_plan_hash parameter detected — ticket #000001 has " + "landed; flip this assertion to the positive form and remove " + "the marker." + ) + + +# --------------------------------------------------------------------------- +# D5 — Verify pointers deterministically +# --------------------------------------------------------------------------- + + +def test_d5_verify_claim_lattice_is_deterministic(): + """Same inputs → same verdict, byte-for-byte. No randomness, no + time-dependent state, no hidden global. Regression here means + the verifier started consulting some non-deterministic source.""" + from aborist.qa.evidence import build_evidence_map + from aborist.qa.verify import verify_claim_lattice + + chunks = [{ + "source_root": "f" * 64, + "document_uri": "u://f", + "title": "F", + "chunk_idx": 0, + "chunk_root": "33" * 32, + "span": "Brachiosaurus appears in the climactic scene.", + "source_role": "primary_answer_source", + }] + em = build_evidence_map(chunks) + answer = "Brachiosaurus appears in the film. [E1]\n" + v1 = verify_claim_lattice(answer, em) + v2 = verify_claim_lattice(answer, em) + assert v1["audit_mode"] == v2["audit_mode"] + assert v1["n_quotes"] == v2["n_quotes"] + assert v1["n_verified"] == v2["n_verified"] + + +def test_d5_seven_hard_checks_are_pure_functions(): + """Spot check: the per-check helper for citation overlap + (`_claim_textually_overlaps_evidence`) is a pure function — no + network, no chat client, no global state.""" + from aborist.qa.verify import _claim_textually_overlaps_evidence + + sig = inspect.signature(_claim_textually_overlaps_evidence) + forbidden = {"chat_client", "client", "llm", "judge", "model"} + assert not (forbidden & set(sig.parameters.keys())) + + +# --------------------------------------------------------------------------- +# D6 — Anchor-class warrant before semantic NLI +# --------------------------------------------------------------------------- + + +def test_d6_warrant_check_exists_and_is_lexical(): + """warrant_check must exist as a deterministic lexical check. + A future replacement that imports an NLI model into the hard + path would violate D6's 'before NLI' constraint.""" + from aborist.qa.warrant import warrant_check + + sig = inspect.signature(warrant_check) + forbidden = {"chat_client", "client", "llm", "judge", "model", "nli"} + found = forbidden & set(sig.parameters.keys()) + assert not found, ( + f"warrant_check accepts forbidden parameter(s) {found} — " + f"directive D6 mandates lexical-only warrant check." + ) + + +def test_d6_warrant_fires_on_date_anchor_mismatch(): + """Existing warrant-lite catches the date-anchor lazy-anchor + class. This case is the deterministic anchor (year strings + are case-trivial; ALL-match semantics). Regression here means + the date-anchor extractor or check silently broke.""" + from aborist.qa.warrant import warrant_check + + # Claim asserts "1985"; cited span has no 1985 anywhere. + ok, missing = warrant_check( + claim_text="Back to the Future was released in theaters in 1985.", + cited_spans=[ + "An unrelated chunk about a film released in 1955 " + "discussing trilogy mechanics and pinball adaptations." + ], + question="what year was back to the future released?", + ) + assert not ok, ( + "warrant should fail when claim asserts year 1985 but no " + "cited span contains 1985" + ) + assert "1985" in " ".join(missing) + + +def test_d6_warrant_relation_shape_fires_when_all_anchors_missing(): + """Proper-noun anchor uses ANY-match semantics: at least one + extracted anchor must appear in some cited span. This test + exercises a true failure where the claim's only proper-noun + anchor is missing from every cited span.""" + from aborist.qa.warrant import warrant_check + + ok, missing = warrant_check( + claim_text="Mr. Burns is the boss.", + cited_spans=[ + "Springfield Nuclear Power Plant employs many workers in " + "the maintenance and reactor sections of the facility." + ], + question="who is the boss?", + ) + assert not ok, ( + "warrant should fail when claim's only proper-noun anchor " + "is missing from every cited span" + ) + assert any("Burns" in m for m in missing) + + +def test_d6_warrant_generalization_status(): + """Marker: warrant generalization to entity-list / count / + why-cause shapes is pending via ticket #000003. Today's + warrant_check has no detector for these shapes; the test + documents the gap and should be updated when #000003 lands.""" + import aborist.qa.warrant as warrant_mod + + # Detectors that should EXIST when ticket #000003 closes: + expected_after_close = ( + "_question_is_entity_list_shape", + "_question_is_count_shape", + "_question_is_why_shape", + ) + found = [ + name for name in expected_after_close + if hasattr(warrant_mod, name) + ] + # Today: zero of them exist. After #000003: all of them. + assert found == [], ( + f"warrant detectors {found} are present — ticket #000003 " + f"likely landed; flip this assertion to the positive form." + ) + + +# --------------------------------------------------------------------------- +# D7 — Rename labels honestly +# --------------------------------------------------------------------------- + + +def test_d7_renderer_relabels_strict_for_claim_lattice(): + """Renderer must map STRICT → EVIDENCE-LINKED for claim-lattice + methods. A regression here would put STRICT on synthesis-heavy + claims, overclaiming semantic truth.""" + from aborist.cli import _render_audit_label + + assert "EVIDENCE-LINKED" in _render_audit_label("STRICT", "claim_lattice") + assert "EVIDENCE-LINKED" in _render_audit_label("STRICT", "claim_lattice_pointer") + + +def test_d7_renderer_keeps_strict_for_pinned_span_methods(): + """Quote / span / entity / paraphrase verify against pinned + spans, not synthesis. STRICT is honest there; renderer must + NOT relabel.""" + from aborist.cli import _render_audit_label + + for method in ("quote", "span", "entity", "paraphrase"): + label = _render_audit_label("STRICT", method) + assert "STRICT" in label, ( + f"verifier_method={method} STRICT relabeled — D7 says " + f"only claim-lattice methods get the EVIDENCE-LINKED rename." + ) + assert "EVIDENCE-LINKED" not in label + + +def test_d7_audit_mode_enum_canonical_set(): + """Schema column must keep the canonical 3-value enum so v9.8 + cache_key invariants hold. Renderer-level relabel (above) + doesn't touch this.""" + from aborist.store import SCHEMA_SQL + + match = re.search( + r"audit_mode\s+TEXT[^,]+CHECK\s*\(\s*audit_mode\s+IN\s*\(([^)]+)\)\)", + SCHEMA_SQL, + re.IGNORECASE, + ) + assert match is not None + enum_values = sorted( + s.strip().strip("'\"") for s in match.group(1).split(",") + ) + assert enum_values == ["HYBRID", "STRICT", "UNGROUNDED"], ( + f"audit_mode enum drifted to {enum_values} — schema column " + f"must stay {{STRICT, HYBRID, UNGROUNDED}}; rendered labels " + f"are display-layer only." + ) + + +# --------------------------------------------------------------------------- +# D8 — Automate only after the invariants are test-pinned +# --------------------------------------------------------------------------- + + +def test_d8_seven_point_program_doc_exists(): + """The directive catalogue must exist and be discoverable from + CLAUDE.md. If someone deletes the program doc but leaves the + bench coverage column referencing it, the substrate stops + documenting what it enforces.""" + from pathlib import Path + + repo_root = Path(__file__).parent.parent + program_doc = repo_root / "docs" / "seven-point-program.md" + assert program_doc.exists(), ( + f"docs/seven-point-program.md missing — directive D8 " + f"discipline relies on the program doc as the audit lens. " + f"Restore from git or open a ticket explaining the new " + f"governance scheme." + ) + + +def test_d8_bench_directive_compliance_helper_exists(): + """The bench harness must carry the directive_compliance helper. + Removing it would silently delete the per-run governance signal.""" + import importlib.util + from pathlib import Path + + repo_root = Path(__file__).parent.parent + spec = importlib.util.spec_from_file_location( + "qa_sweep_dir_check", repo_root / "bench" / "qa_sweep.py" + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + assert hasattr(mod, "_directive_compliance"), ( + "bench/qa_sweep.py missing _directive_compliance helper — " + "D8 enforcement substrate gone." + ) + + +def test_d8_tickets_index_exists_and_pins_open_directives(): + """TICKETS.md must list every open ticket that maps to a + partial directive. If a partial directive has no open ticket, + the design log is lying about the work-in-progress state.""" + from pathlib import Path + + repo_root = Path(__file__).parent.parent + tickets = (repo_root / "docs" / "TICKETS.md").read_text() + program = (repo_root / "docs" / "seven-point-program.md").read_text() + + # Every partial directive (½) in the program doc must have at + # least one open ticket referenced. Today's partials: D3, D4, D6. + # Each of those should appear in the tickets index Directive + # column, mapped to an open ticket. + for directive_id in ("D3", "D4", "D6"): + # Program doc lists the directive as ½. + assert f"| {directive_id[1]} |" in program or directive_id in program + # Tickets index references that directive. + assert directive_id in tickets, ( + f"partial directive {directive_id} missing from TICKETS.md " + f"index — open ticket should map to it." + )