diff --git a/aborist/qa/inspect.py b/aborist/qa/inspect.py index 8385ef0..2983132 100644 --- a/aborist/qa/inspect.py +++ b/aborist/qa/inspect.py @@ -484,6 +484,147 @@ def diagnose_title_relevance( } +# Cue tokens that flag a question as carrying poetic / metaphorical +# framing. Empirically motivated by the 2026-05-02 emergent log entry +# 'How can a swallowtail butterfly, gracefully fluttering amidst the +# rockiest terrain, remain undeterred by the upbraiding winds...' — +# the model traded the metaphor for literal Macleay's-Swallowtail +# taxonomic facts, the warrant passed (the literal anchor IS in cited +# spans), but the user's metaphorical question was never answered. +# Sidecar only — flags candidates for human review; the verifier's +# binary output stays authoritative. +# +# Cue rules (each adds one to the metaphor-cue count): +# - adverbs ending in -ly that aren't in the common stopword pool +# (gracefully, defiantly, undeterredly, etc.) +# - present-participle adjectives ending in -ing that aren't common +# verb forms (upbraiding, fluttering, brooding) — heuristic uses +# the cue list below to avoid bare verb -ing +# - superlative adjectives ending in -est at length >= 5 +# (rockiest, gentlest, fastest, harshest) +# - preposition phrases that almost always carry a metaphorical +# setting (amidst, amid, despite, against, beneath) +_METAPHOR_LY_STOPWORDS = frozenset({ + # Common -ly adverbs that don't carry poetic framing + "early", "only", "really", "actually", "usually", "probably", + "possibly", "lately", "rapidly", "slowly", "quickly", "directly", + "fully", "exactly", "simply", "finally", "originally", "currently", + "immediately", "specifically", "generally", "normally", "essentially", + # Common -ly NOUNS that the suffix heuristic would otherwise pick + # up as adverbs. The naive .endswith('ly') match misfires on + # animal names, place names, and common nouns ending in -ly. + "butterfly", "dragonfly", "mayfly", "firefly", "barfly", "horsefly", + "july", "italy", "sicily", "family", "holly", "jelly", "lily", + "rally", "folly", "billy", "sally", "tally", "wally", "willy", + "dolly", "molly", "polly", "supply", "apply", "imply", "comply", + "reply", "rely", "rely", "ally", "alley", "bully", "ugly", + "homely", "lovely", "lonely", "lively", "lowly", # adj, not adv +}) + +_METAPHOR_PREPOSITION_CUES = frozenset({ + "amidst", "amid", "despite", "against", "beneath", "amongst", + "throughout", "alongside", "betwixt", "within", +}) + +_METAPHOR_ING_BLACKLIST = frozenset({ + # Common verb -ing forms that shouldn't count as poetic adjectives + "doing", "being", "having", "saying", "going", "making", + "thinking", "feeling", "looking", "getting", "taking", "giving", + "knowing", "seeing", "coming", "running", "working", "playing", + "talking", "asking", "telling", "showing", "needing", "trying", + "understanding", "happening", "starting", "stopping", "moving", + "during", "according", "regarding", "concerning", +}) + + +def _extract_metaphor_cues(text: str) -> list[str]: + """Extract poetic / metaphorical cue tokens from ``text``. + + Returns the cue tokens in lowercase, sorted unique. Sidecar only — + the heuristic is deliberately loose; over-flagging is acceptable + since the signal is read by humans, not fed back into the chain. + """ + tokens = _content_tokens_in_order(text or "") + cues: set[str] = set() + for t in tokens: + if t in _METAPHOR_PREPOSITION_CUES: + cues.add(t) + continue + if t.endswith("ly") and t not in _METAPHOR_LY_STOPWORDS and len(t) >= 5: + cues.add(t) + continue + if t.endswith("est") and len(t) >= 6: + # superlatives — rockiest, harshest, gentlest + cues.add(t) + continue + if t.endswith("ing") and len(t) >= 6 and t not in _METAPHOR_ING_BLACKLIST: + # poetic present-participle adjectives — upbraiding, + # fluttering, brooding. The blacklist filters bare verb + # -ing forms. + cues.add(t) + continue + return sorted(cues) + + +def diagnose_metaphor_deflection( + question_text: str, answer_text: str +) -> dict[str, Any]: + """Soft signal: did the question carry metaphorical framing the + answer ignored? + + Empirically observed 2026-05-02 on 'How can a swallowtail butterfly, + gracefully fluttering amidst the rockiest terrain, remain + undeterred by the upbraiding winds that seem to challenge its + delicate flight?' — model traded the metaphor for literal Macleay's + Swallowtail taxonomic facts. Warrant passed (the literal noun + anchor IS in cited spans), DEFLECTION_DETECTED didn't fire (the + last content token did echo in the answer), but the user's + metaphorical question was never answered. Honest gap; structural + catch requires NLI-grade semantics. + + Detection rule (purely lexical): + + 1. Extract metaphor cue tokens from the question (adverbs, + poetic -ing adjectives, superlatives, prepositional cues — + see ``_extract_metaphor_cues``). + 2. Extract content tokens from the answer. + 3. ``metaphor_deflection`` when: + question has >= 3 cue tokens, AND + answer's content tokens overlap zero of them. + + Otherwise: ``no_signal`` (either too few cues to decide, or + answer engaged with at least one cue token). + + Returns: + { + "kind": "metaphor_deflection" | "no_signal", + "cue_tokens": [...], # cues found in question + "answer_overlap": [...], # cues that appeared in answer + "answer_overlap_count": int, + "cue_count": int, + } + + Sidecar only — never enters the binary verifier output. The + threshold (3 cues + 0 overlap) is deliberately conservative; the + smell triggers only when the question is *strongly* poetic and + the answer is *purely* literal. + """ + cues = _extract_metaphor_cues(question_text) + answer_tokens = set(_content_tokens_in_order(answer_text or "")) + overlap = sorted(t for t in cues if t in answer_tokens) + if len(cues) >= 3 and not overlap: + kind = "metaphor_deflection" + else: + kind = "no_signal" + return { + "kind": kind, + "cue_tokens": cues, + "answer_overlap": overlap, + "answer_overlap_count": len(overlap), + "cue_count": len(cues), + } + + def diagnose_deflection(question_text: str, answer_text: str) -> dict[str, Any]: """Soft signal: did the answer change topic? diff --git a/bench/qa_sweep.py b/bench/qa_sweep.py index a293845..15c3e58 100644 --- a/bench/qa_sweep.py +++ b/bench/qa_sweep.py @@ -109,8 +109,11 @@ def _run_one( # never mentions the question's subject) at bench-aggregate # scale so a creeping "model deflects rather than refuses" # regression is legible across runs. - from aborist.qa.inspect import diagnose_deflection + from aborist.qa.inspect import diagnose_deflection, diagnose_metaphor_deflection deflection = diagnose_deflection(question, result.get("answer_text") or "") + metaphor = diagnose_metaphor_deflection( + question, result.get("answer_text") or "" + ) audit_mode = result.get("audit_mode") return { @@ -132,6 +135,9 @@ def _run_one( "deflection_kind": deflection["kind"], "subject_anchor": deflection["subject_anchor"], "subject_in_answer": deflection["subject_in_answer"], + "metaphor_deflection_kind": metaphor["kind"], + "metaphor_cue_count": metaphor["cue_count"], + "metaphor_overlap_count": metaphor["answer_overlap_count"], # Capacity metrics — char-level proxy for prompt-token budget. # Surfaces "did STRICT come from a tight 5KB prompt or a 50KB # context-stuffed one?" at aggregate scale. Lets the bench diff --git a/scripts/bench_emergent.py b/scripts/bench_emergent.py index a434b94..b3ddfe6 100644 --- a/scripts/bench_emergent.py +++ b/scripts/bench_emergent.py @@ -155,6 +155,15 @@ def run_one_cycle( t_answer = time.time() + # Sidecar smell signal: metaphorical question framing the answer + # ignored. Surfaced 2026-05-02 by the swallowtail/upbraided/rockiest + # emergent log entry. Off-the-binary-chain — gives the teacher + # reviewer a flag for poetic-question-vs-literal-answer mismatches. + from aborist.qa.inspect import diagnose_metaphor_deflection + metaphor = diagnose_metaphor_deflection( + question or "", result.get("answer_text") or "" + ) + return { "ts": int(t_pick), "iso_ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(t_pick)), @@ -169,6 +178,9 @@ def run_one_cycle( "violation_kinds": sorted( {v.get("kind") for v in (result.get("violations") or []) if v.get("kind")} ), + "metaphor_deflection_kind": metaphor["kind"], + "metaphor_cue_count": metaphor["cue_count"], + "metaphor_overlap_count": metaphor["answer_overlap_count"], "sources": [ {"title": s.get("title"), "uri": s.get("document_uri"), "used": s.get("used")} for s in (result.get("sources") or []) diff --git a/tests/test_inspect.py b/tests/test_inspect.py index 52170f3..c0390ba 100644 --- a/tests/test_inspect.py +++ b/tests/test_inspect.py @@ -22,6 +22,7 @@ from aborist.qa.inspect import ( _classify_span, _normalize, diagnose_deflection, + diagnose_metaphor_deflection, diagnose_title_relevance, inspect_cache_key, ) @@ -604,3 +605,82 @@ def test_title_mismatch_no_claim_tokens(): """Defensive: empty/stopword-only claim returns no_claim_tokens.""" d = diagnose_title_relevance("the a an", ["Anything"]) assert d["kind"] == "no_claim_tokens" + + +# ───────────────────────────────────────────────────────────────────── +# Metaphor-deflection sidecar +# ───────────────────────────────────────────────────────────────────── + + +def test_metaphor_deflection_swallowtail_canary(): + """The original 2026-05-02 emergent log case: swallowtail butterfly + question framed metaphorically (gracefully fluttering amidst the + rockiest terrain undeterred by upbraiding winds), answer was + purely literal taxonomic (Macleay's Swallowtail found in Eastern + Australia ...). Sidecar should fire.""" + q = ( + "How can a swallowtail butterfly, gracefully fluttering amidst " + "the rockiest terrain, remain undeterred by the upbraiding " + "winds that seem to challenge its delicate flight?" + ) + a = ( + "The Macleay's Swallowtail butterfly is found in Eastern " + "Australia including the ACT, New South Wales, Queensland, " + "Victoria and Tasmania." + ) + d = diagnose_metaphor_deflection(q, a) + assert d["kind"] == "metaphor_deflection" + # cue_count should pick up at least: amidst, fluttering, gracefully, + # rockiest, upbraiding (≥5 real cues; butterfly filtered). + assert d["cue_count"] >= 4 + assert "amidst" in d["cue_tokens"] + assert "rockiest" in d["cue_tokens"] + assert "upbraiding" in d["cue_tokens"] + assert "gracefully" in d["cue_tokens"] + assert "butterfly" not in d["cue_tokens"] # filtered noun + assert d["answer_overlap_count"] == 0 + + +def test_metaphor_deflection_no_signal_on_literal_question(): + """Plain factual questions don't have metaphor cues; sidecar + returns no_signal.""" + d = diagnose_metaphor_deflection( + "who painted the mona lisa?", + "Leonardo da Vinci painted the Mona Lisa around 1503.", + ) + assert d["kind"] == "no_signal" + assert d["cue_count"] == 0 + + +def test_metaphor_deflection_no_signal_when_answer_engages_cues(): + """If the answer echoes any of the question's metaphor cues, the + sidecar does NOT fire — the model engaged with the framing.""" + q = "gracefully amidst rockiest terrain undeterred by upbraiding winds" + a = "The bird flies gracefully amidst the rockiest terrain undeterred." + d = diagnose_metaphor_deflection(q, a) + assert d["kind"] == "no_signal" + assert d["answer_overlap_count"] >= 1 + + +def test_metaphor_deflection_filters_common_ly_nouns(): + """Naive .endswith('ly') would pick up 'butterfly', 'family', + 'italy', 'july' etc. as adverbs. The block-list filters them so + they don't inflate cue count.""" + from aborist.qa.inspect import _extract_metaphor_cues + cues = _extract_metaphor_cues( + "the butterfly flew over italy in july with the family" + ) + # None of the -ly-suffix nouns should appear as cues. + for noun in ("butterfly", "italy", "july", "family"): + assert noun not in cues, f"{noun} leaked through as a cue" + + +def test_metaphor_deflection_under_threshold_returns_no_signal(): + """Sidecar requires >=3 cue tokens to fire — a single -ly word + isn't enough signal to flag metaphor framing.""" + d = diagnose_metaphor_deflection( + "What gracefully describes a circle?", + "A circle is the set of points equidistant from a center.", + ) + assert d["kind"] == "no_signal" + assert d["cue_count"] < 3