From a83e47b1ce3fd50226c05d17484dde5d3254fa8d Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sun, 31 May 2026 12:17:32 -0400 Subject: [PATCH] qa/_text_norm: canonical stem_for_match (dedup query.py + corpus.py) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 step 2 of #53. Hoist the trailing-s stemmer to a single home in arborist.qa._text_norm so query.py, corpus.py, and source_roles.py all use the same implementation. Before: - query.py:_stem_token_for_match — `len>4 and endswith('s') and not endswith('ss')` (apostrophes assumed pre-stripped by _TITLE_TOKEN_RE) - corpus.py:apply_title_boost._stem (inline) — same length/suffix check PLUS apostrophe strip ("'", "’") Behaviorally compatible when inputs are pre-stripped, but the dual implementations were a drift risk waiting to bite. The apostrophe-safe version (corpus.py's) is the canonical now — handles raw title text without an upstream sanitizer, no behavior change for the pre-stripped call sites. Re-exports in query.py + import-update in corpus.py + source_roles.py keep every existing caller working. 257 query/corpus/sidecar/wallet/ bucket/claim_lattice tests pass. Defers lifting _title_query_tokens (and its 5-fold variant helpers — hyphen, numeral, accent, honorific, brit) for later: those carry years of bench-tuned hot-loop optimization and a TITLE_TOKEN_POLICY slug threaded into run-DAG provenance. source_roles still lazy-imports _title_query_tokens from query.py; no change there. --- arborist/qa/_text_norm.py | 20 ++++++++++++++++++++ arborist/qa/corpus.py | 8 +------- arborist/qa/query.py | 21 ++++----------------- arborist/qa/source_roles.py | 11 ++++++----- 4 files changed, 31 insertions(+), 29 deletions(-) diff --git a/arborist/qa/_text_norm.py b/arborist/qa/_text_norm.py index 9ec56a2..829648f 100644 --- a/arborist/qa/_text_norm.py +++ b/arborist/qa/_text_norm.py @@ -82,3 +82,23 @@ def tokenize_text(text: str) -> list[str]: continue out.append(t) return out + + +def stem_for_match(t: str) -> str: + """Light suffix-strip for query-token vs title/body matching. + + Two normalizations rolled into one trailing-s strip: + - possessive "superman's" → "supermans" → "superman" + (apostrophe stripped first; ASCII and curly quotes + handled — fold safely on tokens that pre-strippers + don't reach, e.g. raw title text) + - plural "powers" → "power", "girlfriends" → "girlfriend" + + Conservative: only fires on tokens > 4 chars (preserves "is", "as", + "us") and skips ``ss``-enders ("class", "moss"). Idempotent — + stemming an already-stemmed token is a no-op. + """ + t = t.replace("'", "").replace("’", "") + if len(t) > 4 and t.endswith("s") and not t.endswith("ss"): + return t[:-1] + return t diff --git a/arborist/qa/corpus.py b/arborist/qa/corpus.py index 26e3d61..f60ab13 100644 --- a/arborist/qa/corpus.py +++ b/arborist/qa/corpus.py @@ -89,15 +89,9 @@ def apply_title_boost( return hits from arborist.qa._text_norm import ( _WORD_RE, STOPWORDS, - fold_accents, numeral_expand, tokenize_text, + fold_accents, numeral_expand, stem_for_match as _stem, tokenize_text, ) - def _stem(t: str) -> str: - t = t.replace("'", "").replace("’", "") - if len(t) > 4 and t.endswith("s") and not t.endswith("ss"): - return t[:-1] - return t - query_stems = numeral_expand({_stem(t) for t in tokenize_text(query)}) if not query_stems: return hits diff --git a/arborist/qa/query.py b/arborist/qa/query.py index d4d16da..4f88c0b 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -1048,23 +1048,10 @@ def _docs_with_core_keyword_match( return rows -def _stem_token_for_match(t: str) -> str: - """Light suffix-strip for query-token vs body matching. - - Two normalizations: - possessive ``"superman's" -> "supermans" -> "superman"`` (the apostrophe - is already gone via _TITLE_TOKEN_RE; we drop the trailing - ``s`` here so the lookup matches plain ``superman`` in body). - plural ``"powers" -> "power"``, ``"girlfriends" -> "girlfriend"`` - so plural questions match singular source mentions. - - Both are the same operation: strip trailing ``s`` for tokens > 4 chars. - Conservative on short tokens (``"is"``, ``"as"``, ``"us"`` would lose - meaning) and on tokens that don't end in ``s`` (no-op). - """ - if len(t) > 4 and t.endswith("s") and not t.endswith("ss"): - return t[:-1] - return t +# Canonical stemmer lives in _text_norm.py so query.py + corpus.py + +# source_roles.py share one implementation. Re-exported under the old +# name so existing call sites (and tests) keep working. +from arborist.qa._text_norm import stem_for_match as _stem_token_for_match # noqa: E402 def _body_count_with_stem(body: str, t: str) -> int: diff --git a/arborist/qa/source_roles.py b/arborist/qa/source_roles.py index 568b8c4..aac5ce8 100644 --- a/arborist/qa/source_roles.py +++ b/arborist/qa/source_roles.py @@ -100,12 +100,13 @@ def classify_source_role( return "sequel_background_source" if any(k in t for k in _SECONDARY_TITLE_MARKERS): return "secondary_context_source" - # Lazy import — `_title_query_tokens` + `_stem_token_for_match` - # live in query.py today; they move to _text_norm.py in a later - # Phase 1 step. Import-cycle-safe by deferring to call time. - from arborist.qa.query import _title_query_tokens, _stem_token_for_match + # _title_query_tokens still lives in query.py (full fold stack is + # not lifted yet); the stemmer is now canonical in _text_norm. + # Lazy import keeps the import path acyclic. + from arborist.qa.query import _title_query_tokens + from arborist.qa._text_norm import stem_for_match title_tokens = _title_query_tokens(t.replace("_", " ")) - title_stems = {_stem_token_for_match(tok) for tok in title_tokens} + title_stems = {stem_for_match(tok) for tok in title_tokens} if qtokens_stem and len(title_stems & qtokens_stem) >= max( 1, len(qtokens_stem) - 1 ):