qa/_text_norm: canonical stem_for_match (dedup query.py + corpus.py)
Phase 1 step 2 of #53. Hoist the trailing-s stemmer to a single home in arborist.qa._text_norm so query.py, corpus.py, and source_roles.py all use the same implementation. Before: - query.py:_stem_token_for_match — `len>4 and endswith('s') and not endswith('ss')` (apostrophes assumed pre-stripped by _TITLE_TOKEN_RE) - corpus.py:apply_title_boost._stem (inline) — same length/suffix check PLUS apostrophe strip ("'", "’") Behaviorally compatible when inputs are pre-stripped, but the dual implementations were a drift risk waiting to bite. The apostrophe-safe version (corpus.py's) is the canonical now — handles raw title text without an upstream sanitizer, no behavior change for the pre-stripped call sites. Re-exports in query.py + import-update in corpus.py + source_roles.py keep every existing caller working. 257 query/corpus/sidecar/wallet/ bucket/claim_lattice tests pass. Defers lifting _title_query_tokens (and its 5-fold variant helpers — hyphen, numeral, accent, honorific, brit) for later: those carry years of bench-tuned hot-loop optimization and a TITLE_TOKEN_POLICY slug threaded into run-DAG provenance. source_roles still lazy-imports _title_query_tokens from query.py; no change there.
This commit is contained in:
parent
9ba6317382
commit
a83e47b1ce
4 changed files with 31 additions and 29 deletions
|
|
@ -82,3 +82,23 @@ def tokenize_text(text: str) -> list[str]:
|
|||
continue
|
||||
out.append(t)
|
||||
return out
|
||||
|
||||
|
||||
def stem_for_match(t: str) -> str:
|
||||
"""Light suffix-strip for query-token vs title/body matching.
|
||||
|
||||
Two normalizations rolled into one trailing-s strip:
|
||||
- possessive "superman's" → "supermans" → "superman"
|
||||
(apostrophe stripped first; ASCII and curly quotes
|
||||
handled — fold safely on tokens that pre-strippers
|
||||
don't reach, e.g. raw title text)
|
||||
- plural "powers" → "power", "girlfriends" → "girlfriend"
|
||||
|
||||
Conservative: only fires on tokens > 4 chars (preserves "is", "as",
|
||||
"us") and skips ``ss``-enders ("class", "moss"). Idempotent —
|
||||
stemming an already-stemmed token is a no-op.
|
||||
"""
|
||||
t = t.replace("'", "").replace("’", "")
|
||||
if len(t) > 4 and t.endswith("s") and not t.endswith("ss"):
|
||||
return t[:-1]
|
||||
return t
|
||||
|
|
|
|||
|
|
@ -89,15 +89,9 @@ def apply_title_boost(
|
|||
return hits
|
||||
from arborist.qa._text_norm import (
|
||||
_WORD_RE, STOPWORDS,
|
||||
fold_accents, numeral_expand, tokenize_text,
|
||||
fold_accents, numeral_expand, stem_for_match as _stem, tokenize_text,
|
||||
)
|
||||
|
||||
def _stem(t: str) -> str:
|
||||
t = t.replace("'", "").replace("’", "")
|
||||
if len(t) > 4 and t.endswith("s") and not t.endswith("ss"):
|
||||
return t[:-1]
|
||||
return t
|
||||
|
||||
query_stems = numeral_expand({_stem(t) for t in tokenize_text(query)})
|
||||
if not query_stems:
|
||||
return hits
|
||||
|
|
|
|||
|
|
@ -1048,23 +1048,10 @@ def _docs_with_core_keyword_match(
|
|||
return rows
|
||||
|
||||
|
||||
def _stem_token_for_match(t: str) -> str:
|
||||
"""Light suffix-strip for query-token vs body matching.
|
||||
|
||||
Two normalizations:
|
||||
possessive ``"superman's" -> "supermans" -> "superman"`` (the apostrophe
|
||||
is already gone via _TITLE_TOKEN_RE; we drop the trailing
|
||||
``s`` here so the lookup matches plain ``superman`` in body).
|
||||
plural ``"powers" -> "power"``, ``"girlfriends" -> "girlfriend"``
|
||||
so plural questions match singular source mentions.
|
||||
|
||||
Both are the same operation: strip trailing ``s`` for tokens > 4 chars.
|
||||
Conservative on short tokens (``"is"``, ``"as"``, ``"us"`` would lose
|
||||
meaning) and on tokens that don't end in ``s`` (no-op).
|
||||
"""
|
||||
if len(t) > 4 and t.endswith("s") and not t.endswith("ss"):
|
||||
return t[:-1]
|
||||
return t
|
||||
# Canonical stemmer lives in _text_norm.py so query.py + corpus.py +
|
||||
# source_roles.py share one implementation. Re-exported under the old
|
||||
# name so existing call sites (and tests) keep working.
|
||||
from arborist.qa._text_norm import stem_for_match as _stem_token_for_match # noqa: E402
|
||||
|
||||
|
||||
def _body_count_with_stem(body: str, t: str) -> int:
|
||||
|
|
|
|||
|
|
@ -100,12 +100,13 @@ def classify_source_role(
|
|||
return "sequel_background_source"
|
||||
if any(k in t for k in _SECONDARY_TITLE_MARKERS):
|
||||
return "secondary_context_source"
|
||||
# Lazy import — `_title_query_tokens` + `_stem_token_for_match`
|
||||
# live in query.py today; they move to _text_norm.py in a later
|
||||
# Phase 1 step. Import-cycle-safe by deferring to call time.
|
||||
from arborist.qa.query import _title_query_tokens, _stem_token_for_match
|
||||
# _title_query_tokens still lives in query.py (full fold stack is
|
||||
# not lifted yet); the stemmer is now canonical in _text_norm.
|
||||
# Lazy import keeps the import path acyclic.
|
||||
from arborist.qa.query import _title_query_tokens
|
||||
from arborist.qa._text_norm import stem_for_match
|
||||
title_tokens = _title_query_tokens(t.replace("_", " "))
|
||||
title_stems = {_stem_token_for_match(tok) for tok in title_tokens}
|
||||
title_stems = {stem_for_match(tok) for tok in title_tokens}
|
||||
if qtokens_stem and len(title_stems & qtokens_stem) >= max(
|
||||
1, len(qtokens_stem) - 1
|
||||
):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue