From 9ec9469c4d51979a35fd129bc6cb547c93a1bc5b Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Fri, 1 May 2026 20:18:13 -0400 Subject: [PATCH] =?UTF-8?q?qa(frame):=20#000002=20land=20=E2=80=94=20refer?= =?UTF-8?q?ence-frame=20polarity=20contract=20(D3=20=E2=86=92=20=E2=9C=93)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The phrase-pattern retrieval route (commit 1b8677d) closed the RETRIEVAL side of reference-frame failure; this ticket closes the ANSWER side. New module aborist/qa/frame.py: FrameDetection dataclass (frame_kind, reference_title, reference_uri, confidence). Sidecar — never enters cache_key or governance_policy_hash. detect_frame(question, sources, phrase_match_roots) — heuristic detector. Reference-frame classification fires when: (a) phrase route surfaced at least one source, AND (b) at least one phrase-matched source is a reference work, determined by: - title parenthetical disambig (`(novel)` / `(film)` / `(play)` / `(franchise)` / etc.), OR - body sample contains ≥3 DISTINCT fiction markers (novel / published / protagonist / plot / ...). Distinct-marker count keeps the heuristic robust against a history article saying "novel approach" twice. aborist/qa/query.py: Calls detect_frame for lattice modes only. Body sample uses the ARTICLE LEAD (chunk_idx=0, post-wikitext-strip) — fiction markers cluster in the lead on Wikipedia, not in plot chunks that may have been query-relevance-ranked higher. New policy field claim_lattice_polarity_preamble injected as a user-role message before the grounding_reminder when frame_kind == "reference". Format-string with {reference_title} placeholder. Result dict carries frame_detection (kind / title / uri / confidence) for renderer + bench consumption. aborist/cli.py: Renderer adds a `reference frame: ` line when frame_detection.kind == "reference". Skipped for literal / no-phrase-route / ambiguous rows. Live verification — Orwell case: PRE : "The text does not directly state that Oceania has always been at war with East Asia." POST : "In George Orwell's dystopian novel Nineteen Eighty-Four, the nation of Oceania is always at war with Eastasia, but this is a result of propaganda and doublethink, not actual historical continuity. The war with Eastasia is a fabricated conflict to maintain control..." Multi-frame compilation: distinguishes propaganda claim from fictional-actual continuity, exactly the polarity contract from the ticket §2 abstraction. Literal queries (capital of france) unchanged — polarity preamble only injects when frame_kind classifies as "reference". 22 new tests (19 in test_frame.py + 3 in test_cli_render.py). Full suite: 734 passed (was 712, +22). Directive D3 status: ½ → ✓. Ticket #000002 closed. All seven structural directives now ✓: D1, D2, D5 (were ✓); D3 #000002, D4 #000001, D6 #000003, D7 #000005 (closed in this series). --- aborist/cli.py | 12 + aborist/qa/frame.py | 196 ++++++++++++ aborist/qa/query.py | 94 ++++++ docs/TICKETS.md | 2 +- docs/seven-point-program.md | 27 +- ...00002-reference-frame-polarity-contract.md | 84 +++++- tests/test_cli_render.py | 53 ++++ tests/test_frame.py | 280 ++++++++++++++++++ 8 files changed, 725 insertions(+), 23 deletions(-) create mode 100644 aborist/qa/frame.py create mode 100644 tests/test_frame.py diff --git a/aborist/cli.py b/aborist/cli.py index 49a93bf..e2e0e76 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -779,6 +779,18 @@ def _render_query_human(result: dict, question: str) -> str: f"answer {result.get('answer_chars', 0):,} chars" ) + # Reference-frame notes (Ticket #000002). When detect_frame + # classified the query as `reference`, surface the named work + # so an operator knows the substrate routed to a fictional + # source. Skipped for literal / no-phrase-route / ambiguous + # rows — the line only appears when there's something to say. + fd = result.get("frame_detection") or {} + if fd.get("kind") == "reference" and fd.get("reference_title"): + lines.append("") + lines.append(f"reference frame: {fd['reference_title']}") + if fd.get("reference_uri"): + lines.append(f" cited as the named work in the answer") + cache_key = (result.get("cache_key") or "")[:8] lines.append(f"cache_key: {cache_key}… <run with --json for full record>") return "\n".join(lines) diff --git a/aborist/qa/frame.py b/aborist/qa/frame.py new file mode 100644 index 0000000..fcdc1d9 --- /dev/null +++ b/aborist/qa/frame.py @@ -0,0 +1,196 @@ +"""Reference-frame detection for claim-lattice queries. + +Closes the answer-side gap left open by the phrase-pattern retrieval +route (commit `1b8677d` / docs/reference-frame-failure-class.md). The +phrase route surfaces the right article for allusion-shape queries +(e.g. Nineteen Eighty-Four for "has oceania always been at war with +east asia"); this module then classifies the answer's intended frame +so the prompt + renderer can produce a multi-frame answer that +distinguishes literal from reference from in-universe-propaganda. + +See `docs/ticket-000002-reference-frame-polarity-contract.md` for the +full scope. This module is the *substrate* — it returns a +`FrameDetection` per call. Prompt augmentation + renderer extension +live in their respective layers and consume this output. + +Hard or soft? **Soft.** This is a sidecar-class signal that informs +prompt augmentation + renderer presentation; it never enters the +proof path or `cache_key`. Per CLAUDE.md "Soft hash vs hard hash": +heuristics, scores, and judgment calls live in the soft channel. +The hard verifier never consults `FrameDetection` to accept or +reject claims. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Literal + + +# Parenthetical disambiguation suffixes Wikipedia uses for fictional / +# reference works. Case-insensitive; matches at the open-paren without +# a leading word-boundary (`\b` doesn't match between space and `(` +# since both are non-word chars). +_REFERENCE_TITLE_SUFFIX_RE = re.compile( + r"\(" + r"(?:novel|novella|short\s+story|story|film|movie|play|" + r"video\s+game|game|comic|comics|graphic\s+novel|book|" + r"franchise|series|tv\s+series|television\s+series|" + r"manga|anime|fictional\s+character|character|episode|" + r"album|song|musical|opera|poem)" + r"\)", + re.IGNORECASE, +) + + +# Body-side fiction markers. When a retrieved source's body sample +# contains a high density of these tokens, the source is a reference +# work even when the title lacks a disambiguation suffix +# (Nineteen_Eighty-Four article being the canonical case — its title +# is the bare work-name with no parenthetical, but the body opens +# with "is a dystopian social science fiction novel by ..."). +_FICTION_BODY_MARKERS = frozenset({ + "novel", "novels", "novella", "novellas", "fiction", "fictional", + "fantasy", "dystopian", "satire", "satirical", "allegory", + "allegorical", "characters", "protagonist", "antagonist", + "plot", "narrator", "narrative", "chapter", "chapters", + "published", "publishing", "publisher", "screenplay", "directed", + "starring", "cast", "filmed", "broadcast", "aired", "premiered", + "written", "writer", "author", "playwright", "screenwriter", + "adaptation", "adapted", "sequel", "prequel", "trilogy", + "saga", "franchise", "manga", "anime", +}) + +# Distinct-marker threshold: a body sample must contain at least +# this many DISTINCT fiction markers to classify as reference work. +# Counting distinct markers (not total occurrences) defends against +# a single common-vocabulary use repeated through the sample (e.g. +# "novel approach" + "novel approach" doesn't count as two markers, +# only one distinct). +_FICTION_DISTINCT_MARKER_THRESHOLD = 3 + + +FrameKind = Literal["literal", "reference", "ambiguous", "no_phrase_route"] + + +@dataclass(frozen=True) +class FrameDetection: + """Reference-frame classification for a claim-lattice query. + + Returned by ``detect_frame``. Consumed by: + - prompt augmentation (when ``frame_kind == "reference"``, + inject a polarity instruction into the system prompt) + - renderer (when ``frame_kind == "reference"``, render a + Frame notes section under the answer) + + Never persisted in `providence_cache`, never enters + `governance_policy_hash`, never folds into the run-DAG. Soft + sidecar signal only. + """ + frame_kind: FrameKind + reference_title: str | None = None # e.g. "Nineteen Eighty-Four" + reference_uri: str | None = None # e.g. wiki URI tail + confidence: float = 0.0 # 0.0-1.0, heuristic-based + + +def _title_indicates_reference_work(title: str | None) -> bool: + """Title carries a Wikipedia-style disambiguation suffix for a + fictional / reference work.""" + if not title: + return False + return bool(_REFERENCE_TITLE_SUFFIX_RE.search(title)) + + +def _body_indicates_reference_work(body_sample: str | None) -> bool: + """Body sample contains at least + ``_FICTION_DISTINCT_MARKER_THRESHOLD`` distinct fiction markers. + `body_sample` should be a short prefix (~1-2 KB) of the source's + first chunk; we don't scan the whole article. + + Counting DISTINCT markers (not total occurrences) keeps the + heuristic robust against single-marker repetition: a history + article saying "novel approach" twice doesn't trip the detector + because only one marker (`novel`) fired. A reference-work lead + typically clusters several distinct markers (`novel` + `published` + + `characters` + `plot` + `protagonist` etc.). + """ + if not body_sample: + return False + text_lower = body_sample.lower() + distinct_hits = 0 + for marker in _FICTION_BODY_MARKERS: + pattern = r"\b" + re.escape(marker) + r"\b" + if re.search(pattern, text_lower): + distinct_hits += 1 + if distinct_hits >= _FICTION_DISTINCT_MARKER_THRESHOLD: + return True + return False + + +def _source_is_reference_work(source: dict) -> bool: + """A source is a reference work when its title carries a + disambiguation suffix OR its body opens with high fiction-marker + density. Either signal alone suffices. + """ + if _title_indicates_reference_work(source.get("title")): + return True + return _body_indicates_reference_work(source.get("body_sample")) + + +def detect_frame( + question: str, + sources: list[dict], + phrase_match_roots: set[str] | None = None, +) -> FrameDetection: + """Classify the query's intended frame. + + Inputs: + - ``question`` — the user's question text (currently unused + directly; kept for future shape-detection extensions). + - ``sources`` — the retrieved sources for the query. Each + dict should carry ``document_root``, ``title``, and + ``body_sample`` (a short prefix of the first chunk). + - ``phrase_match_roots`` — set of document_roots that were + surfaced by the phrase-pattern retrieval route (allusion + / verbatim multi-token sequence match). When non-empty, + the query is allusion-shape and the corresponding sources + are candidate reference works. + + Algorithm: + 1. If no phrase route fired (``phrase_match_roots`` empty), + the query is literal — no allusion signal at retrieval. + 2. If phrase route fired AND any phrase-matched source is a + reference work (title suffix or body fiction-density), the + query is reference-frame; pick the first such source as + the named reference. + 3. If phrase route fired but no matched source is a reference + work, classify ``ambiguous`` — the verbatim phrase appeared + somewhere in the corpus but not in a fictional context. + + Returns: + FrameDetection with the classified ``frame_kind``, + ``reference_title`` / ``reference_uri`` populated for + reference-frame queries, and a confidence score that + consumers can threshold if they want a more conservative + gate. + """ + if not phrase_match_roots: + return FrameDetection(frame_kind="no_phrase_route", confidence=0.0) + candidate: dict | None = None + for src in sources: + if src.get("document_root") not in phrase_match_roots: + continue + if _source_is_reference_work(src): + candidate = src + break + if candidate is not None: + return FrameDetection( + frame_kind="reference", + reference_title=candidate.get("title"), + reference_uri=candidate.get("document_uri"), + confidence=0.8, + ) + # Phrase route fired but no clear reference-work source — the + # verbatim phrase appeared in non-fiction context. Could be a + # quotation in a history article, a proverbs page, etc. + return FrameDetection(frame_kind="ambiguous", confidence=0.4) diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 7c1063c..77570fc 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -66,6 +66,7 @@ from aborist.qa.keys import ( verifier_policy_hash, ) from aborist.qa.dag import build_run_dag +from aborist.qa.frame import FrameDetection, detect_frame as _detect_frame from aborist.qa.retrieval_plan import RetrievalPlan, retrieval_plan_hash from aborist.qa.evidence import ( build_evidence_map, @@ -382,6 +383,25 @@ DEFAULT_QUERY_POLICY = { "Begin each line with the claim text. Now answer the question " "on the next message." ), + # Reference-frame polarity preamble (Ticket #000002 / Module L). + # Injected as an additional user-role message before the + # grounding_reminder when `detect_frame` classifies the query as + # `reference` (allusion-shape query whose phrase route surfaced a + # fictional / reference-work source). The preamble nudges the + # model toward multi-frame answers — distinguish what the cited + # work depicts as actual continuity from what in-universe + # propaganda or characters claim. Pure prompt-side hint; + # verifier still runs the same hard checks. Empty string disables + # the augmentation. Folds into governance_policy_hash so two + # policies with different preamble text produce different + # cache_keys. + "claim_lattice_polarity_preamble": ( + "This question may be a reference to {reference_title}. When " + "you answer, distinguish what the cited work depicts as " + "actual continuity from what in-universe propaganda or " + "characters claim within it. Cite evidence for each " + "substantive claim using the pointer format above." + ), "claim_lattice_allowed_source_roles": [ "primary_answer_source", "secondary_context_source", @@ -1712,7 +1732,67 @@ def query( def _user_payload(q: str) -> str: return f"Sources:\n\n{context}\n\n---\n\nQuestion: {q}" + # Frame detection (Ticket #000002 / Module L). Lattice-mode only. + # Surfaces whether the query is allusion-shape AND the phrase + # route surfaced a reference-work source. When `reference`, the + # polarity preamble below nudges the model toward multi-frame + # answers. + # + # Body sample for the fiction-marker density check uses the + # ARTICLE LEAD (chunk_idx=0) — that's where fiction markers + # cluster on Wikipedia ("is a dystopian science fiction novel + # by..."). Reusing chunks_for_map would give us the query- + # relevant chunks (e.g. the plot section containing 'always been + # at war') which may have fewer fiction markers. + frame_detection: FrameDetection | None = None + if answer_mode in ("claim_lattice_pointer", "claim_lattice"): + sources_for_frame: list[dict] = [] + seen_roots: set[str] = set() + for h in chosen: + if h.document_root in seen_roots: + continue + seen_roots.add(h.document_root) + doc_chunks = _load_doc_chunks(h.shard_path, h.document_root) + # First chunk by idx ASC — the article lead. + body_sample = doc_chunks[0][2] if doc_chunks else "" + # Wikitext-strip so fiction markers buried under + # `[[wikilinks]]` and `{{templates}}` surface in the + # density check. Bench-time policy gates the strip; we + # apply it unconditionally here since a no-op fallback + # leaves raw wikitext (and the markers still match the + # `\bnovel\b` regex even with surrounding markup). + if ( + policy.get("base_version") + and _wikitext_to_base is not None + and body_sample + ): + body_sample = _wikitext_to_base(body_sample) + sources_for_frame.append({ + "document_root": h.document_root, + "document_uri": h.document_uri, + "title": h.title, + "body_sample": body_sample, + }) + frame_detection = _detect_frame( + question, sources_for_frame, phrase_match_roots=phrase_match_roots + ) + messages = [{"role": "system", "content": sys_prompt}] + # Polarity preamble for reference-frame queries (Ticket #000002). + # Injected as a user-role message BEFORE the grounding_reminder + # so the model sees the frame hint first, then the + # always-applicable structural reminder, then the actual + # evidence + question. + polarity_template = policy.get("claim_lattice_polarity_preamble", "") + if ( + frame_detection is not None + and frame_detection.frame_kind == "reference" + and polarity_template + ): + polarity_msg = polarity_template.format( + reference_title=frame_detection.reference_title or "" + ) + messages.append({"role": "user", "content": polarity_msg}) if grounding_reminder: messages.append({"role": "user", "content": grounding_reminder}) messages.append({"role": "user", "content": _user_payload(question)}) @@ -2356,6 +2436,20 @@ def query( "retrieval_purity": proof_obj.get("retrieval_purity"), "prompt_chars": prompt_chars, "answer_chars": len(answer_text or ""), + # Frame detection (Ticket #000002 / Module L). Sidecar signal + # for renderer / bench; never persisted in providence_cache, + # never enters governance_policy_hash. None for quote-mode + # rows; populated for lattice-mode rows. + "frame_detection": ( + { + "kind": frame_detection.frame_kind, + "reference_title": frame_detection.reference_title, + "reference_uri": frame_detection.reference_uri, + "confidence": frame_detection.confidence, + } + if frame_detection is not None + else None + ), "timings": { "search_ms": search_ms, "context_ms": context_ms, diff --git a/docs/TICKETS.md b/docs/TICKETS.md index ee9b03b..8953d30 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -58,7 +58,7 @@ Newest first. Update on every open/close. | #000005 | Label ladder migration (POINTER-LINKED → …) | closed · 2026-05-02 | 2026-05-01 | D7 | | #000004 | Directive coverage in bench summary | closed · `acd1f9c` | 2026-05-01 | D8 | | #000003 | Anchor-class warrant generalization (Module H+)| closed · 2026-05-02 | 2026-05-01 | D6 | -| #000002 | Reference-Frame Polarity Contract (Module L) | open | 2026-05-01 | D3 | +| #000002 | Reference-Frame Polarity Contract (Module L) | closed · 2026-05-02 | 2026-05-01 | D3 | | #000001 | Retrieval-keywords audit gap | closed · 2026-05-02 | 2026-05-01 | D4 | ## Next ID diff --git a/docs/seven-point-program.md b/docs/seven-point-program.md index f767d50..050a70c 100644 --- a/docs/seven-point-program.md +++ b/docs/seven-point-program.md @@ -201,21 +201,30 @@ gets layered on top. |---|----------------------------------------------|------------|-----------------| | 1 | Stop making Hermes prove things | ✓ | | | 2 | Hermes emits pointer clauses | ✓ | | -| 3 | Build CTI internally | ½ | #000002 | +| 3 | Build CTI internally | ✓ | #000002 (closed)| | 4 | Bind retrieval map AND evidence map | ✓ | #000001 (closed)| | 5 | Verify pointers deterministically | ✓ | | | 6 | Anchor-class warrant before NLI | ✓ | #000003 (closed)| | 7 | Rename labels honestly | ✓ | #000005 (closed)| | 8 | Automate only after test-pinning | discipline | #000004 (closed)| -One of seven structural directives is partial (D3 — multi-frame -answer compilation pending #000002). D4 closed 2026-05-02 via -#000001 (run-DAG binding scope; SQL column + audit events -deferred). D6 closed 2026-05-02 via #000003. D7 closed 2026-05-02 -via #000005 (four-rung ladder: POINTER-LINKED → ANCHOR-WARRANTED → -EVIDENCE-WARRANTED → ENTAILMENT-VERIFIED). D8 is the -meta-discipline that gates how all of the above land; the bench -coverage substrate (#000004) is closed. +All seven structural directives now ✓: + + D1, D2, D5 — were ✓ at landing. + D3 closed 2026-05-02 via #000002 (frame detector + polarity + preamble + renderer extension; multi-frame answers on Orwell + and other reference-frame queries). + D4 closed 2026-05-02 via #000001 (retrieval_plan_hash binding + in the run-DAG; SQL column + audit events deferred). + D6 closed 2026-05-02 via #000003 (anchor-class warrant + generalization: entity-list / count / why-cause shapes). + D7 closed 2026-05-02 via #000005 (four-rung ladder: + POINTER-LINKED → ANCHOR-WARRANTED → EVIDENCE-WARRANTED → + ENTAILMENT-VERIFIED). + +D8 is the meta-discipline that gates how all of the above land; +the bench coverage substrate (#000004) is closed. Tests: 734 +passed. ## Anti-regression test layer diff --git a/docs/ticket-000002-reference-frame-polarity-contract.md b/docs/ticket-000002-reference-frame-polarity-contract.md index 8ab43b4..c14d592 100644 --- a/docs/ticket-000002-reference-frame-polarity-contract.md +++ b/docs/ticket-000002-reference-frame-polarity-contract.md @@ -1,7 +1,8 @@ # Ticket #000002 — Reference-Frame Polarity Contract (Module L) -**Status:** open · awaiting go/no-go +**Status:** closed · landed 2026-05-02 **Opened:** 2026-05-01 +**Closed:** 2026-05-02 **Scope:** Multi-frame answer compilation for queries that admit literal vs fictional-actual vs in-universe-propaganda interpretations. Builds on the phrase-pattern retrieval route (commit `1b8677d`) @@ -239,20 +240,77 @@ To add when this ticket lands: ## 7. Status -**Proposal.** Phrase-pattern retrieval route is landed and verified -on the Orwell case (commit `1b8677d`); this ticket addresses the -remaining answer-side gap. +**Closed 2026-05-02.** Landed via: -Forecast cost: ~3-4 hours of focused work (frame.py + prompt -augmentation + renderer extension + ~7 tests). Risk: medium — -prompt augmentation interacts with the existing claim_lattice -prompt; need bench evidence to confirm no regression on non-reference -questions. +- New module `aborist/qa/frame.py` — `FrameDetection` dataclass + + `detect_frame(question, sources, phrase_match_roots)`. Detects + reference-frame queries via the conjunction of (phrase route + fired) AND (phrase-matched source is a reference work — title + parenthetical disambig like `(novel)` / `(film)` OR body sample + has ≥3 distinct fiction markers like `novel` / `published` / + `protagonist` / `plot`). Distinct-marker count (not total) keeps + the heuristic robust against single-marker repetition (e.g. a + history article saying "novel approach" twice doesn't trip). +- `aborist/qa/query.py` calls `detect_frame` for lattice modes, + using the article LEAD (chunk_idx=0, post-wikitext-strip) as the + body sample so fiction markers cluster where they appear on + Wikipedia (lead paragraph). +- New policy field `claim_lattice_polarity_preamble` injected as + a user-role message before the grounding_reminder when + `frame_kind == "reference"`. Format-string with + `{reference_title}` placeholder for the named work. +- Renderer adds a `reference frame: <title>` line to the human + output when frame detection classifies the row as reference. +- Result dict gains a `frame_detection` field (sidecar; never + enters cache_key or governance_policy_hash). -Forecast value: closes the answer-shape gap on reference-frame -queries. The phrase route currently produces "does not directly -state" hedges; polarity contract produces multi-frame answers -that explicitly distinguish propaganda from continuity. +Live verification on the Orwell case: + +``` +make query Q="has oceania always been at war with east asia" + EVIDENCE-WARRANTED · via claim_lattice 1/1 + - In George Orwell's dystopian novel Nineteen Eighty-Four, the + nation of Oceania is always at war with Eastasia, but this is a + result of propaganda and doublethink, not actual historical + continuity. The war with Eastasia is a fabricated conflict to + maintain control and keep the populace in a constant state of + fear and obedience. + [E13 | Nineteen Eighty-Four | 682f0a11: "...To hide such + contradictions, history is re-written to explain that the + (new) alliance always was so..."] + + reference frame: Nineteen Eighty-Four +``` + +Compare to pre-#000002 answer (`The text does not directly state...`): +the polarity preamble produces a real multi-frame answer that +distinguishes propaganda from fictional-actual continuity, exactly +as the polarity-contract abstraction proposed. + +Literal queries (`what is the capital of france?`) keep their +clean single-frame answers; the polarity preamble only injects +when `detect_frame` classifies as `reference`. + +19 new frame tests in `tests/test_frame.py` covering title-suffix +detection, body-density detection, source-level composition, and +end-to-end classification. 3 renderer tests in +`tests/test_cli_render.py` covering the reference-frame-notes +section. Full suite: 734 passed. + +## 8. Deferred — additional scope + +- **Per-frame answer compilation in the runtime** (the §2 + multi-frame answer compiler): today's implementation nudges via + the polarity preamble; the compiler would programmatically + emit per-frame paragraphs. Empirical evidence on the prompt- + side approach first; promote to runtime compilation if bench + shows the prompt nudge is unreliable. +- **Frame-kind threshold tuning.** `_FICTION_DISTINCT_MARKER_THRESHOLD` + defaults to 3. Bench may surface false-positive shapes (history + articles with literary criticism subsection) where the threshold + needs to climb to 4-5. +- **`ENTAILMENT-VERIFIED` rung from #000005**: still reserved for + a future committed entailment engine; out of scope here. --- diff --git a/tests/test_cli_render.py b/tests/test_cli_render.py index bd63336..69d2cd0 100644 --- a/tests/test_cli_render.py +++ b/tests/test_cli_render.py @@ -296,6 +296,59 @@ def test_render_label_ungrounded_in_claim_lattice_stays_ungrounded(): assert "via claim_lattice" in out +def test_render_shows_reference_frame_notes_when_detected(): + """When frame_detection.kind == 'reference', the renderer + surfaces the named reference work so an operator can see the + substrate routed to a fictional / reference source.""" + r = _result( + audit_mode="STRICT", + verifier_method="claim_lattice", + frame_detection={ + "kind": "reference", + "reference_title": "Nineteen Eighty-Four", + "reference_uri": "https://en.wikipedia.org/wiki/Nineteen_Eighty-Four", + "confidence": 0.8, + }, + ) + out = _render_query_human(r, "has oceania always been at war with east asia") + assert "reference frame: Nineteen Eighty-Four" in out + + +def test_render_omits_reference_frame_notes_for_literal_query(): + """Literal-frame queries (no allusion) don't render the frame + notes line — keeps the daily render clean for the common case.""" + r = _result( + audit_mode="STRICT", + verifier_method="claim_lattice", + frame_detection={ + "kind": "no_phrase_route", + "reference_title": None, + "reference_uri": None, + "confidence": 0.0, + }, + ) + out = _render_query_human(r, "what is the capital of france?") + assert "reference frame:" not in out + + +def test_render_omits_reference_frame_notes_for_ambiguous(): + """Ambiguous frame (phrase route fired but no clear reference + work) → no frame notes line. The signal isn't strong enough + to claim a named reference.""" + r = _result( + audit_mode="STRICT", + verifier_method="claim_lattice", + frame_detection={ + "kind": "ambiguous", + "reference_title": None, + "reference_uri": None, + "confidence": 0.4, + }, + ) + out = _render_query_human(r, "some allusion-shape query") + assert "reference frame:" not in out + + def test_render_label_quote_mode_keeps_audit_mode_token(): """Quote / span / entity / paraphrase modes verify against pinned spans, not synthesis. STRICT in quote mode IS a strong diff --git a/tests/test_frame.py b/tests/test_frame.py new file mode 100644 index 0000000..ed15c45 --- /dev/null +++ b/tests/test_frame.py @@ -0,0 +1,280 @@ +"""Frame-detection tests (#000002 / Module L scope). + +Pins the heuristic detector that classifies a claim-lattice query as +``literal`` / ``reference`` / ``ambiguous`` / ``no_phrase_route`` based +on the phrase-pattern retrieval signal + per-source reference-work +indicators. +""" +from __future__ import annotations + +from aborist.qa.frame import ( + FrameDetection, + _body_indicates_reference_work, + _source_is_reference_work, + _title_indicates_reference_work, + detect_frame, +) + + +# --------------------------------------------------------------------------- +# Title-suffix detection +# --------------------------------------------------------------------------- + + +def test_title_suffix_recognises_novel_film_play(): + assert _title_indicates_reference_work("Jurassic Park (film)") + assert _title_indicates_reference_work("Jurassic Park (novel)") + assert _title_indicates_reference_work("Hamlet (play)") + assert _title_indicates_reference_work("The Lord of the Rings (franchise)") + assert _title_indicates_reference_work("Star Wars (video game)") + + +def test_title_suffix_handles_multiword_disambig(): + assert _title_indicates_reference_work("Some Comic (graphic novel)") + assert _title_indicates_reference_work("Some Show (TV series)") + assert _title_indicates_reference_work("Some Plot (short story)") + + +def test_title_suffix_doesnt_fire_on_literal_geography(): + assert not _title_indicates_reference_work("Oceania") + assert not _title_indicates_reference_work("Asia") + assert not _title_indicates_reference_work("New York City") + assert not _title_indicates_reference_work("Springfield, Missouri") + + +def test_title_suffix_doesnt_fire_on_disambig_for_real_things(): + """Geographic / biographical disambig suffixes should NOT trip + the reference-work detector.""" + assert not _title_indicates_reference_work("Cleveland (Ohio)") + assert not _title_indicates_reference_work("Mercury (planet)") + assert not _title_indicates_reference_work("Mercury (element)") + assert not _title_indicates_reference_work( + "John Smith (politician)" + ) + + +def test_title_suffix_handles_none_or_empty(): + assert not _title_indicates_reference_work(None) + assert not _title_indicates_reference_work("") + + +# --------------------------------------------------------------------------- +# Body fiction-marker density +# --------------------------------------------------------------------------- + + +def test_body_density_recognises_orwell_lead(): + """The Nineteen_Eighty-Four article opens with high fiction- + marker density: 'novel by George Orwell' + 'fiction' + 'plot' / + 'characters' / 'published' typically all in the first KB.""" + sample = ( + "Nineteen Eighty-Four is a dystopian social science fiction " + "novel by English novelist George Orwell. The narrative " + "follows the protagonist Winston Smith, a low-ranking " + "member of the ruling Party. The novel was published in " + "1949. The plot explores themes of totalitarianism. " + "Characters include Big Brother and Julia." + ) + assert _body_indicates_reference_work(sample) + + +def test_body_density_doesnt_fire_on_geography(): + sample = ( + "Oceania is a continent located in the Pacific Ocean. It " + "comprises Australia, New Zealand, and various smaller " + "Pacific island nations. The continent has a diverse " + "geography with mountains, deserts, and coral reefs. The " + "population is concentrated along the coastlines." + ) + assert not _body_indicates_reference_work(sample) + + +def test_body_density_doesnt_fire_on_one_incidental_novel_use(): + """A history article saying 'this was a novel approach' shouldn't + trip the detector — single use vs marker density.""" + sample = ( + "The reform of 1850 introduced a novel approach to " + "regional administration. Local governors retained " + "significant autonomy under the new framework. The " + "policy persisted for several decades before being " + "replaced. " * 2 + ) + assert not _body_indicates_reference_work(sample) + + +def test_body_density_handles_none_or_empty(): + assert not _body_indicates_reference_work(None) + assert not _body_indicates_reference_work("") + + +# --------------------------------------------------------------------------- +# _source_is_reference_work — composition +# --------------------------------------------------------------------------- + + +def test_source_is_reference_work_via_title(): + src = { + "title": "Hamlet (play)", + "body_sample": "Hamlet is a tragedy.", + } + assert _source_is_reference_work(src) + + +def test_source_is_reference_work_via_body_only(): + src = { + "title": "Nineteen Eighty-Four", + "body_sample": ( + "Nineteen Eighty-Four is a dystopian science fiction " + "novel by George Orwell, published in 1949. The plot " + "follows Winston Smith. The novel explores themes of " + "totalitarianism. Characters include Big Brother." + ), + } + assert _source_is_reference_work(src) + + +def test_source_is_not_reference_work_for_geography(): + src = { + "title": "Oceania", + "body_sample": ( + "Oceania is a geographic region comprising Australia, " + "New Zealand, and Pacific Islands." + ), + } + assert not _source_is_reference_work(src) + + +# --------------------------------------------------------------------------- +# detect_frame — end-to-end classification +# --------------------------------------------------------------------------- + + +def test_detect_frame_no_phrase_route(): + """No phrase route fired → no_phrase_route classification.""" + sources = [{"document_root": "abc", "title": "Oceania"}] + fd = detect_frame("oceania population", sources, phrase_match_roots=set()) + assert fd.frame_kind == "no_phrase_route" + assert fd.reference_title is None + + +def test_detect_frame_no_phrase_route_when_arg_is_none(): + sources = [{"document_root": "abc", "title": "Oceania"}] + fd = detect_frame("oceania population", sources, phrase_match_roots=None) + assert fd.frame_kind == "no_phrase_route" + + +def test_detect_frame_reference_via_title_suffix(): + """Phrase route fired AND matched source has a (film) suffix → + reference-frame classification.""" + sources = [ + { + "document_root": "aaa", + "title": "Jurassic Park (film)", + "document_uri": "https://en.wikipedia.org/wiki/Jurassic_Park_(film)", + "body_sample": "Jurassic Park is a 1993 American science fiction action film.", + }, + ] + fd = detect_frame( + "what dinosaurs were in jurassic park?", + sources, + phrase_match_roots={"aaa"}, + ) + assert fd.frame_kind == "reference" + assert fd.reference_title == "Jurassic Park (film)" + assert fd.confidence > 0.5 + + +def test_detect_frame_reference_via_body_density_orwell(): + """Phrase route fired AND matched source has fiction body markers + → reference-frame even without a title suffix (Nineteen + Eighty-Four canonical case).""" + sources = [ + { + "document_root": "bbb", + "title": "Nineteen Eighty-Four", + "document_uri": "https://en.wikipedia.org/wiki/Nineteen_Eighty-Four", + "body_sample": ( + "Nineteen Eighty-Four is a dystopian science fiction " + "novel by George Orwell, published in 1949. The plot " + "follows Winston Smith, a Party member. Characters " + "include Big Brother and Julia. The novel explores " + "totalitarianism." + ), + }, + ] + fd = detect_frame( + "has oceania always been at war with east asia", + sources, + phrase_match_roots={"bbb"}, + ) + assert fd.frame_kind == "reference" + assert fd.reference_title == "Nineteen Eighty-Four" + + +def test_detect_frame_ambiguous_when_phrase_matched_but_not_reference(): + """Phrase route fired but the matched source is a non-fiction + article (e.g. a verbatim quotation that happens to appear in a + history page). Classification: ambiguous.""" + sources = [ + { + "document_root": "ccc", + "title": "List of common phrases", + "document_uri": "https://example.org/list", + "body_sample": ( + "This is a list page. It contains various phrases " + "and their meanings. Geographic regions include " + "Asia, Africa, and Europe." + ), + }, + ] + fd = detect_frame( + "always been at war with east asia", + sources, + phrase_match_roots={"ccc"}, + ) + assert fd.frame_kind == "ambiguous" + assert fd.reference_title is None + + +def test_detect_frame_picks_first_reference_source_when_multiple(): + """When phrase route surfaced multiple sources and ≥2 are + reference works, pick the first by retrieval order.""" + sources = [ + { + "document_root": "aaa", + "title": "Jurassic Park (film)", + "body_sample": "An action film.", + }, + { + "document_root": "bbb", + "title": "Jurassic Park (novel)", + "body_sample": "A novel by Michael Crichton.", + }, + ] + fd = detect_frame( + "dinosaurs in jurassic park", + sources, + phrase_match_roots={"aaa", "bbb"}, + ) + assert fd.frame_kind == "reference" + # First-by-order is the film. + assert fd.reference_title == "Jurassic Park (film)" + + +def test_detect_frame_skips_non_phrase_matched_sources(): + """A reference-work source that wasn't phrase-matched doesn't + trigger the reference frame (otherwise any retrieval that + happens to include a (film)-suffixed source would over-fire).""" + sources = [ + { + "document_root": "aaa", + "title": "Hamlet (play)", # reference work but not phrase-matched + "body_sample": "A tragedy.", + }, + ] + fd = detect_frame( + "hamlet plot summary", + sources, + phrase_match_roots=set(), # no phrase route + ) + assert fd.frame_kind == "no_phrase_route"