diff --git a/aborist/qa/inspect.py b/aborist/qa/inspect.py index 2983132..aaf982e 100644 --- a/aborist/qa/inspect.py +++ b/aborist/qa/inspect.py @@ -494,75 +494,134 @@ def diagnose_title_relevance( # Sidecar only — flags candidates for human review; the verifier's # binary output stays authoritative. # -# Cue rules (each adds one to the metaphor-cue count): -# - adverbs ending in -ly that aren't in the common stopword pool -# (gracefully, defiantly, undeterredly, etc.) -# - present-participle adjectives ending in -ing that aren't common -# verb forms (upbraiding, fluttering, brooding) — heuristic uses -# the cue list below to avoid bare verb -ing -# - superlative adjectives ending in -est at length >= 5 -# (rockiest, gentlest, fastest, harshest) -# - preposition phrases that almost always carry a metaphorical -# setting (amidst, amid, despite, against, beneath) -_METAPHOR_LY_STOPWORDS = frozenset({ - # Common -ly adverbs that don't carry poetic framing - "early", "only", "really", "actually", "usually", "probably", - "possibly", "lately", "rapidly", "slowly", "quickly", "directly", - "fully", "exactly", "simply", "finally", "originally", "currently", - "immediately", "specifically", "generally", "normally", "essentially", - # Common -ly NOUNS that the suffix heuristic would otherwise pick - # up as adverbs. The naive .endswith('ly') match misfires on - # animal names, place names, and common nouns ending in -ly. - "butterfly", "dragonfly", "mayfly", "firefly", "barfly", "horsefly", - "july", "italy", "sicily", "family", "holly", "jelly", "lily", - "rally", "folly", "billy", "sally", "tally", "wally", "willy", - "dolly", "molly", "polly", "supply", "apply", "imply", "comply", - "reply", "rely", "rely", "ally", "alley", "bully", "ugly", - "homely", "lovely", "lonely", "lively", "lowly", # adj, not adv -}) +# Cues are DERIVED from the system English wordlist +# (/usr/share/dict/words), not hand-curated. Same Phase-2 lesson the +# concept_relations layer learned: hand-typed exception lists scale +# poorly. The wordlist already encodes "is this a real English word"; +# adverbiality / participle-ness / superlativity comes from the +# morphological-stem-test against the wordlist. Pure derivation. +# +# Cue families: +# - Closed-class prepositions that almost always carry metaphorical +# setting (amidst, despite, beneath, ...). Tiny fixed list — these +# are a closed grammatical class, not a vocabulary tail. +# - -ly adverbs whose stem (or stem+e, or stem-i+y) is in the +# wordlist (gracefully → graceful, truly → true, happily → happy). +# The stem test filters -ly NOUNS like butterfly (stem 'butterf' +# not in wordlist) without a hand-curated noun list. +# - -ing present-participle adjectives whose stem (or stem+e, or +# consonant-de-doubled stem) is a verb in the wordlist +# (fluttering → flutter, running → run, upbraiding → upbraid). +# - -est superlatives whose stem (or stem+e, or stem-i+y) is an +# adjective in the wordlist (rockiest → rocky, safest → safe, +# longest → long). _METAPHOR_PREPOSITION_CUES = frozenset({ "amidst", "amid", "despite", "against", "beneath", "amongst", - "throughout", "alongside", "betwixt", "within", + "throughout", "alongside", "betwixt", "atop", }) -_METAPHOR_ING_BLACKLIST = frozenset({ - # Common verb -ing forms that shouldn't count as poetic adjectives - "doing", "being", "having", "saying", "going", "making", - "thinking", "feeling", "looking", "getting", "taking", "giving", - "knowing", "seeing", "coming", "running", "working", "playing", - "talking", "asking", "telling", "showing", "needing", "trying", - "understanding", "happening", "starting", "stopping", "moving", - "during", "according", "regarding", "concerning", -}) +_DICT_WORDS_PATH = Path("/usr/share/dict/words") + +_english_wordlist_cache: frozenset[str] | None = None + + +def _english_wordlist() -> frozenset[str]: + """Load /usr/share/dict/words once; return as a lowercase frozenset. + + Graceful degradation: if the wordlist doesn't exist (some + containers, BSDs, Windows), return an empty set — every cue + returns False, the sidecar quietly returns ``no_signal``. + """ + global _english_wordlist_cache + if _english_wordlist_cache is None: + if not _DICT_WORDS_PATH.exists(): + _english_wordlist_cache = frozenset() + else: + _english_wordlist_cache = frozenset( + w.strip().lower() + for w in _DICT_WORDS_PATH.read_text(encoding="utf-8", errors="replace").splitlines() + if w.strip() + ) + return _english_wordlist_cache + + +def _is_adverbial_ly(word: str) -> bool: + """True if ``word`` is an -ly adverb whose stem is a real word.""" + if len(word) < 5 or not word.endswith("ly"): + return False + dictionary = _english_wordlist() + if not dictionary: + return False + stem = word[:-2] + if len(stem) < 3: + return False + if stem in dictionary: # gracefully → graceful + return True + if stem + "e" in dictionary: # truly → true + return True + if stem.endswith("i") and stem[:-1] + "y" in dictionary: # happily → happy + return True + return False + + +def _is_present_participle(word: str) -> bool: + """True if ``word`` is an -ing form of a verb in the wordlist.""" + if len(word) < 6 or not word.endswith("ing"): + return False + dictionary = _english_wordlist() + if not dictionary: + return False + stem = word[:-3] + if len(stem) < 3: + return False + if stem in dictionary: # fluttering → flutter + return True + if stem + "e" in dictionary: # making → make + return True + if len(stem) >= 3 and stem[-1] == stem[-2] and stem[:-1] in dictionary: + return True # running → run + return False + + +def _is_superlative_est(word: str) -> bool: + """True if ``word`` is an -est superlative of an adjective in the wordlist.""" + if len(word) < 6 or not word.endswith("est"): + return False + dictionary = _english_wordlist() + if not dictionary: + return False + stem = word[:-3] + if len(stem) < 3: + return False + if stem in dictionary: # longest → long + return True + if stem + "e" in dictionary: # safest → safe + return True + if stem.endswith("i") and stem[:-1] + "y" in dictionary: # rockiest → rocky + return True + return False def _extract_metaphor_cues(text: str) -> list[str]: """Extract poetic / metaphorical cue tokens from ``text``. - Returns the cue tokens in lowercase, sorted unique. Sidecar only — - the heuristic is deliberately loose; over-flagging is acceptable - since the signal is read by humans, not fed back into the chain. + Returns the cue tokens in lowercase, sorted unique. Three suffix + classes (-ly / -ing / -est) plus a tiny fixed prepositional set. + Suffix tests derive from the system wordlist via stem-existence + rules — no hand-curated noun-exception lists. """ tokens = _content_tokens_in_order(text or "") cues: set[str] = set() for t in tokens: if t in _METAPHOR_PREPOSITION_CUES: cues.add(t) - continue - if t.endswith("ly") and t not in _METAPHOR_LY_STOPWORDS and len(t) >= 5: + elif _is_adverbial_ly(t): cues.add(t) - continue - if t.endswith("est") and len(t) >= 6: - # superlatives — rockiest, harshest, gentlest + elif _is_superlative_est(t): cues.add(t) - continue - if t.endswith("ing") and len(t) >= 6 and t not in _METAPHOR_ING_BLACKLIST: - # poetic present-participle adjectives — upbraiding, - # fluttering, brooding. The blacklist filters bare verb - # -ing forms. + elif _is_present_participle(t): cues.add(t) - continue return sorted(cues)