From b87966cc89e622f503cc053fc74d9512be21f8e9 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sat, 2 May 2026 13:52:34 -0400 Subject: [PATCH] qa(inspect): derive metaphor cues from /usr/share/dict/words MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase-2 lesson applied. The previous metaphor-deflection sidecar shipped two hand-curated frozensets (_METAPHOR_LY_STOPWORDS, _METAPHOR_ING_BLACKLIST) listing English -ly nouns and common -ing verb forms. Same trap as concept_relations Phase 1: hand-curating exception lists for English suffix patterns has an infinite tail (every newly-encountered -ly noun earns one more git commit). Replace with derivation. The system English wordlist /usr/share/dict/words is already on the bench-emergent picker's critical path; reuse it as the morphological substrate. Cue rule (purely lexical, derived): - -ly word is adverbial iff stem (or stem+e, or stem-i+y) is in the wordlist: gracefully → graceful ✓ truly → true ✓ happily → happy ✓ butterfly → butterf ✗ (noun, filtered automatically) italy → ital ✗ (proper noun) family → famil ✗ (noun) - -ing word is participle iff stem (or stem+e, or de-doubled consonant stem) is a verb in the wordlist: fluttering → flutter ✓ running → run ✓ (consonant de-doubled) making → make ✓ sterling → sterl ✗ (noun, filtered) during → dur ✗ (preposition, filtered) - -est word is superlative iff stem (or stem+e, or stem-i+y) is an adjective in the wordlist: rockiest → rocky ✓ longest → long ✓ safest → safe ✓ The closed-class prepositional cue set stays hand-listed (~10 words) — these are a finite grammatical class, not a vocabulary tail. _METAPHOR_LY_STOPWORDS and _METAPHOR_ING_BLACKLIST deleted entirely. Graceful degradation: if /usr/share/dict/words doesn't exist (BSD, some containers, Windows), the wordlist function returns empty; suffix tests all return False; the sidecar quietly returns no_signal. The closed prepositional set still works. Verified on swallowtail canary: 5 cues found (amidst, fluttering, gracefully, rockiest, upbraiding), threshold met, fires metaphor_deflection. Minor residual noise (holly → 'hol' in dict; interest → 'inter' in dict) accepted as soft-signal cost. The signal is read by human reviewers; over-flagging at the +1-cue level rarely crosses the 3-cue threshold for a real question. 756/34 tests pass. --- aborist/qa/inspect.py | 159 +++++++++++++++++++++++++++++------------- 1 file changed, 109 insertions(+), 50 deletions(-) diff --git a/aborist/qa/inspect.py b/aborist/qa/inspect.py index 2983132..aaf982e 100644 --- a/aborist/qa/inspect.py +++ b/aborist/qa/inspect.py @@ -494,75 +494,134 @@ def diagnose_title_relevance( # Sidecar only — flags candidates for human review; the verifier's # binary output stays authoritative. # -# Cue rules (each adds one to the metaphor-cue count): -# - adverbs ending in -ly that aren't in the common stopword pool -# (gracefully, defiantly, undeterredly, etc.) -# - present-participle adjectives ending in -ing that aren't common -# verb forms (upbraiding, fluttering, brooding) — heuristic uses -# the cue list below to avoid bare verb -ing -# - superlative adjectives ending in -est at length >= 5 -# (rockiest, gentlest, fastest, harshest) -# - preposition phrases that almost always carry a metaphorical -# setting (amidst, amid, despite, against, beneath) -_METAPHOR_LY_STOPWORDS = frozenset({ - # Common -ly adverbs that don't carry poetic framing - "early", "only", "really", "actually", "usually", "probably", - "possibly", "lately", "rapidly", "slowly", "quickly", "directly", - "fully", "exactly", "simply", "finally", "originally", "currently", - "immediately", "specifically", "generally", "normally", "essentially", - # Common -ly NOUNS that the suffix heuristic would otherwise pick - # up as adverbs. The naive .endswith('ly') match misfires on - # animal names, place names, and common nouns ending in -ly. - "butterfly", "dragonfly", "mayfly", "firefly", "barfly", "horsefly", - "july", "italy", "sicily", "family", "holly", "jelly", "lily", - "rally", "folly", "billy", "sally", "tally", "wally", "willy", - "dolly", "molly", "polly", "supply", "apply", "imply", "comply", - "reply", "rely", "rely", "ally", "alley", "bully", "ugly", - "homely", "lovely", "lonely", "lively", "lowly", # adj, not adv -}) +# Cues are DERIVED from the system English wordlist +# (/usr/share/dict/words), not hand-curated. Same Phase-2 lesson the +# concept_relations layer learned: hand-typed exception lists scale +# poorly. The wordlist already encodes "is this a real English word"; +# adverbiality / participle-ness / superlativity comes from the +# morphological-stem-test against the wordlist. Pure derivation. +# +# Cue families: +# - Closed-class prepositions that almost always carry metaphorical +# setting (amidst, despite, beneath, ...). Tiny fixed list — these +# are a closed grammatical class, not a vocabulary tail. +# - -ly adverbs whose stem (or stem+e, or stem-i+y) is in the +# wordlist (gracefully → graceful, truly → true, happily → happy). +# The stem test filters -ly NOUNS like butterfly (stem 'butterf' +# not in wordlist) without a hand-curated noun list. +# - -ing present-participle adjectives whose stem (or stem+e, or +# consonant-de-doubled stem) is a verb in the wordlist +# (fluttering → flutter, running → run, upbraiding → upbraid). +# - -est superlatives whose stem (or stem+e, or stem-i+y) is an +# adjective in the wordlist (rockiest → rocky, safest → safe, +# longest → long). _METAPHOR_PREPOSITION_CUES = frozenset({ "amidst", "amid", "despite", "against", "beneath", "amongst", - "throughout", "alongside", "betwixt", "within", + "throughout", "alongside", "betwixt", "atop", }) -_METAPHOR_ING_BLACKLIST = frozenset({ - # Common verb -ing forms that shouldn't count as poetic adjectives - "doing", "being", "having", "saying", "going", "making", - "thinking", "feeling", "looking", "getting", "taking", "giving", - "knowing", "seeing", "coming", "running", "working", "playing", - "talking", "asking", "telling", "showing", "needing", "trying", - "understanding", "happening", "starting", "stopping", "moving", - "during", "according", "regarding", "concerning", -}) +_DICT_WORDS_PATH = Path("/usr/share/dict/words") + +_english_wordlist_cache: frozenset[str] | None = None + + +def _english_wordlist() -> frozenset[str]: + """Load /usr/share/dict/words once; return as a lowercase frozenset. + + Graceful degradation: if the wordlist doesn't exist (some + containers, BSDs, Windows), return an empty set — every cue + returns False, the sidecar quietly returns ``no_signal``. + """ + global _english_wordlist_cache + if _english_wordlist_cache is None: + if not _DICT_WORDS_PATH.exists(): + _english_wordlist_cache = frozenset() + else: + _english_wordlist_cache = frozenset( + w.strip().lower() + for w in _DICT_WORDS_PATH.read_text(encoding="utf-8", errors="replace").splitlines() + if w.strip() + ) + return _english_wordlist_cache + + +def _is_adverbial_ly(word: str) -> bool: + """True if ``word`` is an -ly adverb whose stem is a real word.""" + if len(word) < 5 or not word.endswith("ly"): + return False + dictionary = _english_wordlist() + if not dictionary: + return False + stem = word[:-2] + if len(stem) < 3: + return False + if stem in dictionary: # gracefully → graceful + return True + if stem + "e" in dictionary: # truly → true + return True + if stem.endswith("i") and stem[:-1] + "y" in dictionary: # happily → happy + return True + return False + + +def _is_present_participle(word: str) -> bool: + """True if ``word`` is an -ing form of a verb in the wordlist.""" + if len(word) < 6 or not word.endswith("ing"): + return False + dictionary = _english_wordlist() + if not dictionary: + return False + stem = word[:-3] + if len(stem) < 3: + return False + if stem in dictionary: # fluttering → flutter + return True + if stem + "e" in dictionary: # making → make + return True + if len(stem) >= 3 and stem[-1] == stem[-2] and stem[:-1] in dictionary: + return True # running → run + return False + + +def _is_superlative_est(word: str) -> bool: + """True if ``word`` is an -est superlative of an adjective in the wordlist.""" + if len(word) < 6 or not word.endswith("est"): + return False + dictionary = _english_wordlist() + if not dictionary: + return False + stem = word[:-3] + if len(stem) < 3: + return False + if stem in dictionary: # longest → long + return True + if stem + "e" in dictionary: # safest → safe + return True + if stem.endswith("i") and stem[:-1] + "y" in dictionary: # rockiest → rocky + return True + return False def _extract_metaphor_cues(text: str) -> list[str]: """Extract poetic / metaphorical cue tokens from ``text``. - Returns the cue tokens in lowercase, sorted unique. Sidecar only — - the heuristic is deliberately loose; over-flagging is acceptable - since the signal is read by humans, not fed back into the chain. + Returns the cue tokens in lowercase, sorted unique. Three suffix + classes (-ly / -ing / -est) plus a tiny fixed prepositional set. + Suffix tests derive from the system wordlist via stem-existence + rules — no hand-curated noun-exception lists. """ tokens = _content_tokens_in_order(text or "") cues: set[str] = set() for t in tokens: if t in _METAPHOR_PREPOSITION_CUES: cues.add(t) - continue - if t.endswith("ly") and t not in _METAPHOR_LY_STOPWORDS and len(t) >= 5: + elif _is_adverbial_ly(t): cues.add(t) - continue - if t.endswith("est") and len(t) >= 6: - # superlatives — rockiest, harshest, gentlest + elif _is_superlative_est(t): cues.add(t) - continue - if t.endswith("ing") and len(t) >= 6 and t not in _METAPHOR_ING_BLACKLIST: - # poetic present-participle adjectives — upbraiding, - # fluttering, brooding. The blacklist filters bare verb - # -ing forms. + elif _is_present_participle(t): cues.add(t) - continue return sorted(cues)