qa(inspect): metaphor-deflection sidecar (METAPHORICAL_DEFLECTION smell)
Empirically motivated by the 2026-05-02 emergent log:
Q: 'How can a swallowtail butterfly, gracefully fluttering amidst
the rockiest terrain, remain undeterred by the upbraiding
winds...'
A: 'The Macleay's Swallowtail butterfly is found in Eastern
Australia including the ACT, New South Wales, Queensland...'
The model traded the metaphor for literal Macleay's-Swallowtail
taxonomic facts. Warrant passed (the literal anchor IS in cited
spans), DEFLECTION_DETECTED didn't fire (the last content token
'flight' did echo somewhere), the bench landed HYBRID 3/3 — but
the user's metaphorical question was never engaged.
Honest gap: catching this structurally requires NLI-grade
semantics, which is the verifier-semantic-gap design proposal.
Until that lands, ship a SMELL SIDECAR — purely lexical, sidecar
only, never enters the binary verifier output.
Detection rule:
1. Extract metaphor cues from the question:
- -ly adverbs (gracefully, defiantly), excluding common
-ly nouns (butterfly, italy, july) via blocklist
- -ing present participles >=6 chars (upbraiding,
fluttering, brooding), excluding common verb -ing forms
- -est superlatives >=6 chars (rockiest, harshest)
- prepositional cues (amidst, despite, against, beneath)
2. Count overlap with answer's content tokens.
3. Fire metaphor_deflection when:
cue_count >= 3 AND answer_overlap_count == 0
The threshold is conservative; the smell only triggers on
STRONGLY poetic questions with PURELY literal answers.
Wire-up:
- aborist/qa/inspect.py:diagnose_metaphor_deflection
- bench/qa_sweep.py: rows gain metaphor_deflection_kind +
metaphor_cue_count + metaphor_overlap_count
- scripts/bench_emergent.py: same fields on emergent log rows
5 new tests in tests/test_inspect.py:
- swallowtail canary case fires metaphor_deflection
- literal questions (mona lisa) return no_signal
- questions whose answer engages cues return no_signal
- common -ly nouns (butterfly, italy, july, family) filtered
- sub-threshold cue counts return no_signal
756/34 tests pass (5 new + 751 prior).
This commit is contained in:
parent
a1dd330455
commit
8fec3a5d56
4 changed files with 240 additions and 1 deletions
|
|
@ -484,6 +484,147 @@ def diagnose_title_relevance(
|
|||
}
|
||||
|
||||
|
||||
# Cue tokens that flag a question as carrying poetic / metaphorical
|
||||
# framing. Empirically motivated by the 2026-05-02 emergent log entry
|
||||
# 'How can a swallowtail butterfly, gracefully fluttering amidst the
|
||||
# rockiest terrain, remain undeterred by the upbraiding winds...' —
|
||||
# the model traded the metaphor for literal Macleay's-Swallowtail
|
||||
# taxonomic facts, the warrant passed (the literal anchor IS in cited
|
||||
# spans), but the user's metaphorical question was never answered.
|
||||
# Sidecar only — flags candidates for human review; the verifier's
|
||||
# binary output stays authoritative.
|
||||
#
|
||||
# Cue rules (each adds one to the metaphor-cue count):
|
||||
# - adverbs ending in -ly that aren't in the common stopword pool
|
||||
# (gracefully, defiantly, undeterredly, etc.)
|
||||
# - present-participle adjectives ending in -ing that aren't common
|
||||
# verb forms (upbraiding, fluttering, brooding) — heuristic uses
|
||||
# the cue list below to avoid bare verb -ing
|
||||
# - superlative adjectives ending in -est at length >= 5
|
||||
# (rockiest, gentlest, fastest, harshest)
|
||||
# - preposition phrases that almost always carry a metaphorical
|
||||
# setting (amidst, amid, despite, against, beneath)
|
||||
_METAPHOR_LY_STOPWORDS = frozenset({
|
||||
# Common -ly adverbs that don't carry poetic framing
|
||||
"early", "only", "really", "actually", "usually", "probably",
|
||||
"possibly", "lately", "rapidly", "slowly", "quickly", "directly",
|
||||
"fully", "exactly", "simply", "finally", "originally", "currently",
|
||||
"immediately", "specifically", "generally", "normally", "essentially",
|
||||
# Common -ly NOUNS that the suffix heuristic would otherwise pick
|
||||
# up as adverbs. The naive .endswith('ly') match misfires on
|
||||
# animal names, place names, and common nouns ending in -ly.
|
||||
"butterfly", "dragonfly", "mayfly", "firefly", "barfly", "horsefly",
|
||||
"july", "italy", "sicily", "family", "holly", "jelly", "lily",
|
||||
"rally", "folly", "billy", "sally", "tally", "wally", "willy",
|
||||
"dolly", "molly", "polly", "supply", "apply", "imply", "comply",
|
||||
"reply", "rely", "rely", "ally", "alley", "bully", "ugly",
|
||||
"homely", "lovely", "lonely", "lively", "lowly", # adj, not adv
|
||||
})
|
||||
|
||||
_METAPHOR_PREPOSITION_CUES = frozenset({
|
||||
"amidst", "amid", "despite", "against", "beneath", "amongst",
|
||||
"throughout", "alongside", "betwixt", "within",
|
||||
})
|
||||
|
||||
_METAPHOR_ING_BLACKLIST = frozenset({
|
||||
# Common verb -ing forms that shouldn't count as poetic adjectives
|
||||
"doing", "being", "having", "saying", "going", "making",
|
||||
"thinking", "feeling", "looking", "getting", "taking", "giving",
|
||||
"knowing", "seeing", "coming", "running", "working", "playing",
|
||||
"talking", "asking", "telling", "showing", "needing", "trying",
|
||||
"understanding", "happening", "starting", "stopping", "moving",
|
||||
"during", "according", "regarding", "concerning",
|
||||
})
|
||||
|
||||
|
||||
def _extract_metaphor_cues(text: str) -> list[str]:
|
||||
"""Extract poetic / metaphorical cue tokens from ``text``.
|
||||
|
||||
Returns the cue tokens in lowercase, sorted unique. Sidecar only —
|
||||
the heuristic is deliberately loose; over-flagging is acceptable
|
||||
since the signal is read by humans, not fed back into the chain.
|
||||
"""
|
||||
tokens = _content_tokens_in_order(text or "")
|
||||
cues: set[str] = set()
|
||||
for t in tokens:
|
||||
if t in _METAPHOR_PREPOSITION_CUES:
|
||||
cues.add(t)
|
||||
continue
|
||||
if t.endswith("ly") and t not in _METAPHOR_LY_STOPWORDS and len(t) >= 5:
|
||||
cues.add(t)
|
||||
continue
|
||||
if t.endswith("est") and len(t) >= 6:
|
||||
# superlatives — rockiest, harshest, gentlest
|
||||
cues.add(t)
|
||||
continue
|
||||
if t.endswith("ing") and len(t) >= 6 and t not in _METAPHOR_ING_BLACKLIST:
|
||||
# poetic present-participle adjectives — upbraiding,
|
||||
# fluttering, brooding. The blacklist filters bare verb
|
||||
# -ing forms.
|
||||
cues.add(t)
|
||||
continue
|
||||
return sorted(cues)
|
||||
|
||||
|
||||
def diagnose_metaphor_deflection(
|
||||
question_text: str, answer_text: str
|
||||
) -> dict[str, Any]:
|
||||
"""Soft signal: did the question carry metaphorical framing the
|
||||
answer ignored?
|
||||
|
||||
Empirically observed 2026-05-02 on 'How can a swallowtail butterfly,
|
||||
gracefully fluttering amidst the rockiest terrain, remain
|
||||
undeterred by the upbraiding winds that seem to challenge its
|
||||
delicate flight?' — model traded the metaphor for literal Macleay's
|
||||
Swallowtail taxonomic facts. Warrant passed (the literal noun
|
||||
anchor IS in cited spans), DEFLECTION_DETECTED didn't fire (the
|
||||
last content token did echo in the answer), but the user's
|
||||
metaphorical question was never answered. Honest gap; structural
|
||||
catch requires NLI-grade semantics.
|
||||
|
||||
Detection rule (purely lexical):
|
||||
|
||||
1. Extract metaphor cue tokens from the question (adverbs,
|
||||
poetic -ing adjectives, superlatives, prepositional cues —
|
||||
see ``_extract_metaphor_cues``).
|
||||
2. Extract content tokens from the answer.
|
||||
3. ``metaphor_deflection`` when:
|
||||
question has >= 3 cue tokens, AND
|
||||
answer's content tokens overlap zero of them.
|
||||
|
||||
Otherwise: ``no_signal`` (either too few cues to decide, or
|
||||
answer engaged with at least one cue token).
|
||||
|
||||
Returns:
|
||||
{
|
||||
"kind": "metaphor_deflection" | "no_signal",
|
||||
"cue_tokens": [...], # cues found in question
|
||||
"answer_overlap": [...], # cues that appeared in answer
|
||||
"answer_overlap_count": int,
|
||||
"cue_count": int,
|
||||
}
|
||||
|
||||
Sidecar only — never enters the binary verifier output. The
|
||||
threshold (3 cues + 0 overlap) is deliberately conservative; the
|
||||
smell triggers only when the question is *strongly* poetic and
|
||||
the answer is *purely* literal.
|
||||
"""
|
||||
cues = _extract_metaphor_cues(question_text)
|
||||
answer_tokens = set(_content_tokens_in_order(answer_text or ""))
|
||||
overlap = sorted(t for t in cues if t in answer_tokens)
|
||||
if len(cues) >= 3 and not overlap:
|
||||
kind = "metaphor_deflection"
|
||||
else:
|
||||
kind = "no_signal"
|
||||
return {
|
||||
"kind": kind,
|
||||
"cue_tokens": cues,
|
||||
"answer_overlap": overlap,
|
||||
"answer_overlap_count": len(overlap),
|
||||
"cue_count": len(cues),
|
||||
}
|
||||
|
||||
|
||||
def diagnose_deflection(question_text: str, answer_text: str) -> dict[str, Any]:
|
||||
"""Soft signal: did the answer change topic?
|
||||
|
||||
|
|
|
|||
|
|
@ -109,8 +109,11 @@ def _run_one(
|
|||
# never mentions the question's subject) at bench-aggregate
|
||||
# scale so a creeping "model deflects rather than refuses"
|
||||
# regression is legible across runs.
|
||||
from aborist.qa.inspect import diagnose_deflection
|
||||
from aborist.qa.inspect import diagnose_deflection, diagnose_metaphor_deflection
|
||||
deflection = diagnose_deflection(question, result.get("answer_text") or "")
|
||||
metaphor = diagnose_metaphor_deflection(
|
||||
question, result.get("answer_text") or ""
|
||||
)
|
||||
|
||||
audit_mode = result.get("audit_mode")
|
||||
return {
|
||||
|
|
@ -132,6 +135,9 @@ def _run_one(
|
|||
"deflection_kind": deflection["kind"],
|
||||
"subject_anchor": deflection["subject_anchor"],
|
||||
"subject_in_answer": deflection["subject_in_answer"],
|
||||
"metaphor_deflection_kind": metaphor["kind"],
|
||||
"metaphor_cue_count": metaphor["cue_count"],
|
||||
"metaphor_overlap_count": metaphor["answer_overlap_count"],
|
||||
# Capacity metrics — char-level proxy for prompt-token budget.
|
||||
# Surfaces "did STRICT come from a tight 5KB prompt or a 50KB
|
||||
# context-stuffed one?" at aggregate scale. Lets the bench
|
||||
|
|
|
|||
|
|
@ -155,6 +155,15 @@ def run_one_cycle(
|
|||
|
||||
t_answer = time.time()
|
||||
|
||||
# Sidecar smell signal: metaphorical question framing the answer
|
||||
# ignored. Surfaced 2026-05-02 by the swallowtail/upbraided/rockiest
|
||||
# emergent log entry. Off-the-binary-chain — gives the teacher
|
||||
# reviewer a flag for poetic-question-vs-literal-answer mismatches.
|
||||
from aborist.qa.inspect import diagnose_metaphor_deflection
|
||||
metaphor = diagnose_metaphor_deflection(
|
||||
question or "", result.get("answer_text") or ""
|
||||
)
|
||||
|
||||
return {
|
||||
"ts": int(t_pick),
|
||||
"iso_ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(t_pick)),
|
||||
|
|
@ -169,6 +178,9 @@ def run_one_cycle(
|
|||
"violation_kinds": sorted(
|
||||
{v.get("kind") for v in (result.get("violations") or []) if v.get("kind")}
|
||||
),
|
||||
"metaphor_deflection_kind": metaphor["kind"],
|
||||
"metaphor_cue_count": metaphor["cue_count"],
|
||||
"metaphor_overlap_count": metaphor["answer_overlap_count"],
|
||||
"sources": [
|
||||
{"title": s.get("title"), "uri": s.get("document_uri"), "used": s.get("used")}
|
||||
for s in (result.get("sources") or [])
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from aborist.qa.inspect import (
|
|||
_classify_span,
|
||||
_normalize,
|
||||
diagnose_deflection,
|
||||
diagnose_metaphor_deflection,
|
||||
diagnose_title_relevance,
|
||||
inspect_cache_key,
|
||||
)
|
||||
|
|
@ -604,3 +605,82 @@ def test_title_mismatch_no_claim_tokens():
|
|||
"""Defensive: empty/stopword-only claim returns no_claim_tokens."""
|
||||
d = diagnose_title_relevance("the a an", ["Anything"])
|
||||
assert d["kind"] == "no_claim_tokens"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# Metaphor-deflection sidecar
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_metaphor_deflection_swallowtail_canary():
|
||||
"""The original 2026-05-02 emergent log case: swallowtail butterfly
|
||||
question framed metaphorically (gracefully fluttering amidst the
|
||||
rockiest terrain undeterred by upbraiding winds), answer was
|
||||
purely literal taxonomic (Macleay's Swallowtail found in Eastern
|
||||
Australia ...). Sidecar should fire."""
|
||||
q = (
|
||||
"How can a swallowtail butterfly, gracefully fluttering amidst "
|
||||
"the rockiest terrain, remain undeterred by the upbraiding "
|
||||
"winds that seem to challenge its delicate flight?"
|
||||
)
|
||||
a = (
|
||||
"The Macleay's Swallowtail butterfly is found in Eastern "
|
||||
"Australia including the ACT, New South Wales, Queensland, "
|
||||
"Victoria and Tasmania."
|
||||
)
|
||||
d = diagnose_metaphor_deflection(q, a)
|
||||
assert d["kind"] == "metaphor_deflection"
|
||||
# cue_count should pick up at least: amidst, fluttering, gracefully,
|
||||
# rockiest, upbraiding (≥5 real cues; butterfly filtered).
|
||||
assert d["cue_count"] >= 4
|
||||
assert "amidst" in d["cue_tokens"]
|
||||
assert "rockiest" in d["cue_tokens"]
|
||||
assert "upbraiding" in d["cue_tokens"]
|
||||
assert "gracefully" in d["cue_tokens"]
|
||||
assert "butterfly" not in d["cue_tokens"] # filtered noun
|
||||
assert d["answer_overlap_count"] == 0
|
||||
|
||||
|
||||
def test_metaphor_deflection_no_signal_on_literal_question():
|
||||
"""Plain factual questions don't have metaphor cues; sidecar
|
||||
returns no_signal."""
|
||||
d = diagnose_metaphor_deflection(
|
||||
"who painted the mona lisa?",
|
||||
"Leonardo da Vinci painted the Mona Lisa around 1503.",
|
||||
)
|
||||
assert d["kind"] == "no_signal"
|
||||
assert d["cue_count"] == 0
|
||||
|
||||
|
||||
def test_metaphor_deflection_no_signal_when_answer_engages_cues():
|
||||
"""If the answer echoes any of the question's metaphor cues, the
|
||||
sidecar does NOT fire — the model engaged with the framing."""
|
||||
q = "gracefully amidst rockiest terrain undeterred by upbraiding winds"
|
||||
a = "The bird flies gracefully amidst the rockiest terrain undeterred."
|
||||
d = diagnose_metaphor_deflection(q, a)
|
||||
assert d["kind"] == "no_signal"
|
||||
assert d["answer_overlap_count"] >= 1
|
||||
|
||||
|
||||
def test_metaphor_deflection_filters_common_ly_nouns():
|
||||
"""Naive .endswith('ly') would pick up 'butterfly', 'family',
|
||||
'italy', 'july' etc. as adverbs. The block-list filters them so
|
||||
they don't inflate cue count."""
|
||||
from aborist.qa.inspect import _extract_metaphor_cues
|
||||
cues = _extract_metaphor_cues(
|
||||
"the butterfly flew over italy in july with the family"
|
||||
)
|
||||
# None of the -ly-suffix nouns should appear as cues.
|
||||
for noun in ("butterfly", "italy", "july", "family"):
|
||||
assert noun not in cues, f"{noun} leaked through as a cue"
|
||||
|
||||
|
||||
def test_metaphor_deflection_under_threshold_returns_no_signal():
|
||||
"""Sidecar requires >=3 cue tokens to fire — a single -ly word
|
||||
isn't enough signal to flag metaphor framing."""
|
||||
d = diagnose_metaphor_deflection(
|
||||
"What gracefully describes a circle?",
|
||||
"A circle is the set of points equidistant from a center.",
|
||||
)
|
||||
assert d["kind"] == "no_signal"
|
||||
assert d["cue_count"] < 3
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue