diff --git a/arborist/qa/_text_norm.py b/arborist/qa/_text_norm.py index 9ec56a2..829648f 100644 --- a/arborist/qa/_text_norm.py +++ b/arborist/qa/_text_norm.py @@ -82,3 +82,23 @@ def tokenize_text(text: str) -> list[str]: continue out.append(t) return out + + +def stem_for_match(t: str) -> str: + """Light suffix-strip for query-token vs title/body matching. + + Two normalizations rolled into one trailing-s strip: + - possessive "superman's" → "supermans" → "superman" + (apostrophe stripped first; ASCII and curly quotes + handled — fold safely on tokens that pre-strippers + don't reach, e.g. raw title text) + - plural "powers" → "power", "girlfriends" → "girlfriend" + + Conservative: only fires on tokens > 4 chars (preserves "is", "as", + "us") and skips ``ss``-enders ("class", "moss"). Idempotent — + stemming an already-stemmed token is a no-op. + """ + t = t.replace("'", "").replace("’", "") + if len(t) > 4 and t.endswith("s") and not t.endswith("ss"): + return t[:-1] + return t diff --git a/arborist/qa/corpus.py b/arborist/qa/corpus.py index 26e3d61..f60ab13 100644 --- a/arborist/qa/corpus.py +++ b/arborist/qa/corpus.py @@ -89,15 +89,9 @@ def apply_title_boost( return hits from arborist.qa._text_norm import ( _WORD_RE, STOPWORDS, - fold_accents, numeral_expand, tokenize_text, + fold_accents, numeral_expand, stem_for_match as _stem, tokenize_text, ) - def _stem(t: str) -> str: - t = t.replace("'", "").replace("’", "") - if len(t) > 4 and t.endswith("s") and not t.endswith("ss"): - return t[:-1] - return t - query_stems = numeral_expand({_stem(t) for t in tokenize_text(query)}) if not query_stems: return hits diff --git a/arborist/qa/query.py b/arborist/qa/query.py index d4d16da..4f88c0b 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -1048,23 +1048,10 @@ def _docs_with_core_keyword_match( return rows -def _stem_token_for_match(t: str) -> str: - """Light suffix-strip for query-token vs body matching. - - Two normalizations: - possessive ``"superman's" -> "supermans" -> "superman"`` (the apostrophe - is already gone via _TITLE_TOKEN_RE; we drop the trailing - ``s`` here so the lookup matches plain ``superman`` in body). - plural ``"powers" -> "power"``, ``"girlfriends" -> "girlfriend"`` - so plural questions match singular source mentions. - - Both are the same operation: strip trailing ``s`` for tokens > 4 chars. - Conservative on short tokens (``"is"``, ``"as"``, ``"us"`` would lose - meaning) and on tokens that don't end in ``s`` (no-op). - """ - if len(t) > 4 and t.endswith("s") and not t.endswith("ss"): - return t[:-1] - return t +# Canonical stemmer lives in _text_norm.py so query.py + corpus.py + +# source_roles.py share one implementation. Re-exported under the old +# name so existing call sites (and tests) keep working. +from arborist.qa._text_norm import stem_for_match as _stem_token_for_match # noqa: E402 def _body_count_with_stem(body: str, t: str) -> int: diff --git a/arborist/qa/source_roles.py b/arborist/qa/source_roles.py index 568b8c4..aac5ce8 100644 --- a/arborist/qa/source_roles.py +++ b/arborist/qa/source_roles.py @@ -100,12 +100,13 @@ def classify_source_role( return "sequel_background_source" if any(k in t for k in _SECONDARY_TITLE_MARKERS): return "secondary_context_source" - # Lazy import — `_title_query_tokens` + `_stem_token_for_match` - # live in query.py today; they move to _text_norm.py in a later - # Phase 1 step. Import-cycle-safe by deferring to call time. - from arborist.qa.query import _title_query_tokens, _stem_token_for_match + # _title_query_tokens still lives in query.py (full fold stack is + # not lifted yet); the stemmer is now canonical in _text_norm. + # Lazy import keeps the import path acyclic. + from arborist.qa.query import _title_query_tokens + from arborist.qa._text_norm import stem_for_match title_tokens = _title_query_tokens(t.replace("_", " ")) - title_stems = {_stem_token_for_match(tok) for tok in title_tokens} + title_stems = {stem_for_match(tok) for tok in title_tokens} if qtokens_stem and len(title_stems & qtokens_stem) >= max( 1, len(qtokens_stem) - 1 ):