qa/_text_norm: canonical stem_for_match (dedup query.py + corpus.py)

Phase 1 step 2 of #53. Hoist the trailing-s stemmer to a single home
in arborist.qa._text_norm so query.py, corpus.py, and source_roles.py
all use the same implementation.

Before:
  - query.py:_stem_token_for_match — `len>4 and endswith('s') and not
    endswith('ss')` (apostrophes assumed pre-stripped by _TITLE_TOKEN_RE)
  - corpus.py:apply_title_boost._stem (inline) — same length/suffix
    check PLUS apostrophe strip ("'", "’")

Behaviorally compatible when inputs are pre-stripped, but the dual
implementations were a drift risk waiting to bite. The apostrophe-safe
version (corpus.py's) is the canonical now — handles raw title text
without an upstream sanitizer, no behavior change for the pre-stripped
call sites.

Re-exports in query.py + import-update in corpus.py + source_roles.py
keep every existing caller working. 257 query/corpus/sidecar/wallet/
bucket/claim_lattice tests pass.

Defers lifting _title_query_tokens (and its 5-fold variant helpers —
hyphen, numeral, accent, honorific, brit) for later: those carry
years of bench-tuned hot-loop optimization and a TITLE_TOKEN_POLICY
slug threaded into run-DAG provenance. source_roles still lazy-imports
_title_query_tokens from query.py; no change there.
This commit is contained in:
russell@unturf.com 2026-05-31 12:17:32 -04:00
parent 9ba6317382
commit a83e47b1ce
No known key found for this signature in database
4 changed files with 31 additions and 29 deletions

View file

@ -82,3 +82,23 @@ def tokenize_text(text: str) -> list[str]:
continue
out.append(t)
return out
def stem_for_match(t: str) -> str:
"""Light suffix-strip for query-token vs title/body matching.
Two normalizations rolled into one trailing-s strip:
- possessive "superman's" "supermans" "superman"
(apostrophe stripped first; ASCII and curly quotes
handled fold safely on tokens that pre-strippers
don't reach, e.g. raw title text)
- plural "powers" "power", "girlfriends" "girlfriend"
Conservative: only fires on tokens > 4 chars (preserves "is", "as",
"us") and skips ``ss``-enders ("class", "moss"). Idempotent
stemming an already-stemmed token is a no-op.
"""
t = t.replace("'", "").replace("", "")
if len(t) > 4 and t.endswith("s") and not t.endswith("ss"):
return t[:-1]
return t

View file

@ -89,15 +89,9 @@ def apply_title_boost(
return hits
from arborist.qa._text_norm import (
_WORD_RE, STOPWORDS,
fold_accents, numeral_expand, tokenize_text,
fold_accents, numeral_expand, stem_for_match as _stem, tokenize_text,
)
def _stem(t: str) -> str:
t = t.replace("'", "").replace("", "")
if len(t) > 4 and t.endswith("s") and not t.endswith("ss"):
return t[:-1]
return t
query_stems = numeral_expand({_stem(t) for t in tokenize_text(query)})
if not query_stems:
return hits

View file

@ -1048,23 +1048,10 @@ def _docs_with_core_keyword_match(
return rows
def _stem_token_for_match(t: str) -> str:
"""Light suffix-strip for query-token vs body matching.
Two normalizations:
possessive ``"superman's" -> "supermans" -> "superman"`` (the apostrophe
is already gone via _TITLE_TOKEN_RE; we drop the trailing
``s`` here so the lookup matches plain ``superman`` in body).
plural ``"powers" -> "power"``, ``"girlfriends" -> "girlfriend"``
so plural questions match singular source mentions.
Both are the same operation: strip trailing ``s`` for tokens > 4 chars.
Conservative on short tokens (``"is"``, ``"as"``, ``"us"`` would lose
meaning) and on tokens that don't end in ``s`` (no-op).
"""
if len(t) > 4 and t.endswith("s") and not t.endswith("ss"):
return t[:-1]
return t
# Canonical stemmer lives in _text_norm.py so query.py + corpus.py +
# source_roles.py share one implementation. Re-exported under the old
# name so existing call sites (and tests) keep working.
from arborist.qa._text_norm import stem_for_match as _stem_token_for_match # noqa: E402
def _body_count_with_stem(body: str, t: str) -> int:

View file

@ -100,12 +100,13 @@ def classify_source_role(
return "sequel_background_source"
if any(k in t for k in _SECONDARY_TITLE_MARKERS):
return "secondary_context_source"
# Lazy import — `_title_query_tokens` + `_stem_token_for_match`
# live in query.py today; they move to _text_norm.py in a later
# Phase 1 step. Import-cycle-safe by deferring to call time.
from arborist.qa.query import _title_query_tokens, _stem_token_for_match
# _title_query_tokens still lives in query.py (full fold stack is
# not lifted yet); the stemmer is now canonical in _text_norm.
# Lazy import keeps the import path acyclic.
from arborist.qa.query import _title_query_tokens
from arborist.qa._text_norm import stem_for_match
title_tokens = _title_query_tokens(t.replace("_", " "))
title_stems = {_stem_token_for_match(tok) for tok in title_tokens}
title_stems = {stem_for_match(tok) for tok in title_tokens}
if qtokens_stem and len(title_stems & qtokens_stem) >= max(
1, len(qtokens_stem) - 1
):