From 81c051e3fc849e2fe5bd54884310c8a11b6bc5b6 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Fri, 15 May 2026 11:41:46 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20MPS-24=20Phase=202.3=20=E2=80=94=20mult?= =?UTF-8?q?i-bigram=20supersession,=20apostrophe=20labels,=20top=5Fn=20100?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2.2 surfaced real categories but left noise: - Color (119), Number (100), Day (63), Room (36) — unigrams fully covered by multiple bigrams, but the prior supersession only considered one bigram at a time so "Day" stayed even though "Valentine's Day" + "Patrick's Day" + … collectively cover all its products. - "Valentine Day" / "Patrick Day" labels read as typo-broken because apostrophes were stripped during cleaning. - 50 candidates wasn't long-tail enough on a 481-product catalog. Three fixes: - Multi-bigram supersession: a unigram drops when the UNION of bigrams containing it covers ≥ 80% of its product set. Iterates all bigrams for the unigram's stem, unions their product sets, computes coverage once. - Apostrophe-preserving tokeniser + stemmer: `_MD_PUNCT` no longer strips `'`; `_WORD` regex accepts a trailing `(?:'[a-z]+)?` so "valentine's" and "patrick's" survive as surface forms. `simple_stem` drops the apostrophe tail before suffix-stripping so "valentine's" stems to "valentine" — the cluster groups correctly while the label vote wins with the readable surface form. Stopword check uses the apostrophe-less base so possessives can't slip past the list. - top_n default 50 → 100. CLI default also bumped. Tested with a Valentine's/Patrick's-heavy sample: bigrams render as "Valentine's Day", "Patrick's Day" with proper apostrophes; the bare "Day" unigram drops because the bigrams together cover all its products. 1080 tests passing. --- docs/tickets/mps-24.md | 25 +++++++++++ make_post_sell/lib/tag_suggest.py | 67 ++++++++++++++++------------- make_post_sell/tests/test_models.py | 58 +++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 29 deletions(-) diff --git a/docs/tickets/mps-24.md b/docs/tickets/mps-24.md index 2088b5f..2d84980 100644 --- a/docs/tickets/mps-24.md +++ b/docs/tickets/mps-24.md @@ -319,6 +319,31 @@ picker is Phase 2). | `tests/test_models.py` | `TestTagSuggestPureFunctions` — 11 unit tests over tokenize / stem / cluster | | `tests/test_functional.py` | `test_suggest_clusters_renders_candidates`, `test_apply_suggestion_creates_tag_and_attaches_products`, `test_dismiss_suggestion_adds_to_stopwords`, `test_apply_suggestion_rejects_empty_input` | +### Phase 2.3 — multi-bigram supersession + apostrophe labels + top_n 100 (shipped 2026-05-15) + +Phase 2.2 surfaced real categories but left residue: `Color` (119), +`Number` (100), `Day` (63), `Room` (36) — all unigrams that are fully +covered by multiple bigrams (e.g. `Day` is covered by `Valentine's Day` ++ `Patrick's Day` + others). And bigram labels like `Valentine Day` / +`Patrick Day` lost their apostrophes — operators read them as +typo-broken. Three fixes: + +- **Multi-bigram supersession**: a unigram drops when the *union* of + bigrams containing it covers ≥ 80% of its products. Phase 2.2 only + considered single-bigram coverage; now `Day` drops because the + combined set of `Valentine's Day` ∪ `Patrick's Day` ∪ … covers it. +- **Apostrophe-preserving labels**: tokeniser keeps the possessive / + contraction tail (`valentine's`, `patrick's`); stemmer strips it + *before* matching but the label vote still wins with the readable + surface form. `_MD_PUNCT` no longer kills apostrophes. Stopword + check uses the apostrophe-less base so possessives can't slip in. +- **`top_n` default 50 → 100** for the long tail of niche categories. + +Result on a Valentine's/Patrick's-heavy sample: bigrams render as +`Valentine's Day`, `Patrick's Day` (readable possessives), and the +catch-all `Day` unigram disappears because the two bigrams together +cover all its products. + ### Phase 2.2 — bigrams + title-required + bigger stopwords (shipped 2026-05-15) Phase 2.1's `max_share=0.4` filter only caught one of printableprompts' diff --git a/make_post_sell/lib/tag_suggest.py b/make_post_sell/lib/tag_suggest.py index dbabc8d..b85574d 100644 --- a/make_post_sell/lib/tag_suggest.py +++ b/make_post_sell/lib/tag_suggest.py @@ -33,7 +33,7 @@ DEFAULT_MIN_PRODUCTS = 2 # Default cap on how many candidate clusters we return per call. # Set generously — large catalogues (printableprompts has 481) carry many # valid niche categories beyond the obvious top 20. -DEFAULT_TOP_N = 50 +DEFAULT_TOP_N = 100 # A stem appearing in more than this fraction of products is treated as # **shop vocabulary** — words the operator uses to describe everything @@ -153,7 +153,9 @@ _STEM_SUFFIXES = ( ) # Strip these markdown / formatting characters before tokenising. -_MD_PUNCT = re.compile(r"[`*_~#>|\\\[\]()<>{}/\"'!?,.:;=+\-]") +# Note: apostrophe is intentionally KEPT so "Valentine's", "Patrick's" +# survive as readable labels — see `_WORD` below. +_MD_PUNCT = re.compile(r"[`*_~#>|\\\[\]()<>{}/\"!?,.:;=+\-]") # Drop fenced code blocks / inline code spans so code snippets don't # leak random tokens. Order matters: fenced first, then inline. @@ -166,26 +168,34 @@ _URL = re.compile(r"https?://\S+") # Strip raw HTML tags (description is markdown but operators paste in HTML). _HTML_TAG = re.compile(r"<[^>]+>") -# What counts as a word — lowercase letters + digits, ≥ 3 chars. -_WORD = re.compile(r"[a-z][a-z0-9]{2,}") +# What counts as a word — lowercase letters + digits, ≥ 3 chars, +# with an optional possessive/contraction suffix (e.g. "valentine's", +# "patrick's"). The apostrophe-tail survives the markdown clean so the +# label voting can render readable "Valentine's Day" / "Patrick's Day". +_WORD = re.compile(r"[a-z][a-z0-9]{2,}(?:'[a-z]+)?") def simple_stem(word): """Suffix-strip stem for English-ish product copy. Conservative: never strips a suffix if the resulting stem is < 3 - chars (avoids collapsing "ice" → "i" because of -ce). + chars (avoids collapsing "ice" → "i" because of -ce). Drops + possessive/contraction tails first ("valentine's" → "valentine") + so apostrophe surface forms still stem cleanly. >>> simple_stem("seasonal") 'season' >>> simple_stem("seasons") 'season' - >>> simple_stem("running") - 'runn' + >>> simple_stem("valentine's") + 'valentine' >>> simple_stem("math") 'math' """ w = (word or "").lower() + # Drop possessive / contraction tail before stemming. + if "'" in w: + w = w.split("'", 1)[0] if len(w) <= 3: return w for suffix in _STEM_SUFFIXES: @@ -224,7 +234,10 @@ def tokenize(text, stopwords=None, cap=None): tokens = [] seen = set() if cap else None for raw in _WORD.findall(cleaned): - if raw in stop: + # Stopword test uses the apostrophe-less base ("valentine" from + # "valentine's") so possessives don't slip past the list. + base = raw.split("'", 1)[0] + if base in stop or raw in stop: continue if cap: if raw in seen: @@ -428,38 +441,34 @@ def suggest_clusters( reverse=True, ) - # Bigram supersession: if a bigram dominates the same product set as - # one of its component unigrams (>= SUPERSESSION_THRESHOLD overlap), - # drop the unigram. The operator wants one phrase candidate, not two - # near-duplicate rows ("Write Room" + "Write" + "Room"). + # Bigram supersession: a unigram drops if a bigram (or collectively + # the union of bigrams) containing it covers ≥ SUPERSESSION_THRESHOLD + # of the unigram's product set. The operator wants a single phrase + # row ("Write Room") instead of three near-duplicates + # ("Write Room" + "Write" + "Room"), and a single "Day" row should + # drop when "Valentine's Day" + "Patrick's Day" + … collectively + # cover most of its products. SUPERSESSION_THRESHOLD = 0.8 - bigram_components = {} + bigram_components = defaultdict(list) for c in candidates: if c["is_bigram"]: - parts = c["stem"].split() - if len(parts) == 2: - bigram_components.setdefault(parts[0], []).append(c) - bigram_components.setdefault(parts[1], []).append(c) + for stem in c["stem"].split(): + bigram_components[stem].append(c) superseded = set() for c in candidates: if c["is_bigram"]: continue - bigrams_for = bigram_components.get(c["stem"], []) c_ids = set(c["product_ids"]) if not c_ids: continue - for big in bigrams_for: - big_ids = set(big["product_ids"]) - if not big_ids: - continue - # If the unigram's products are mostly covered by the bigram, - # the bigram is the better candidate. - overlap = len(c_ids & big_ids) / len(c_ids) - if overlap >= SUPERSESSION_THRESHOLD: - superseded.add(c["stem"]) - filtered_count += 1 - break + covered = set() + for big in bigram_components.get(c["stem"], []): + covered |= set(big["product_ids"]) + overlap = len(c_ids & covered) / len(c_ids) + if overlap >= SUPERSESSION_THRESHOLD: + superseded.add(c["stem"]) + filtered_count += 1 candidates = [c for c in candidates if c["stem"] not in superseded] return candidates[:top_n], filtered_count diff --git a/make_post_sell/tests/test_models.py b/make_post_sell/tests/test_models.py index 3fcef9c..4f4c46d 100644 --- a/make_post_sell/tests/test_models.py +++ b/make_post_sell/tests/test_models.py @@ -5401,4 +5401,62 @@ class TestTagSuggestPureFunctions(unittest.TestCase): self.assertNotIn("Write", labels) self.assertNotIn("Room", labels) + def test_suggest_clusters_preserves_apostrophe_labels(self): + """Possessive surface forms ("Valentine's", "Patrick's") survive + as readable labels — the apostrophe-less stem is used for + clustering but the original spelling wins the label vote.""" + from types import SimpleNamespace + from ..lib.tag_suggest import suggest_clusters + products = [ + SimpleNamespace(id="a", title="Valentine's Day Math", + description=""), + SimpleNamespace(id="b", title="Valentine's Day Reading", + description=""), + SimpleNamespace(id="c", title="Valentine's Day Crafts", + description=""), + ] + clusters, _ = suggest_clusters( + products, stopwords=[], existing_tag_slugs=[], + min_products=2, max_share=1.0, min_title_share=0.0, + bigrams=True, + ) + labels = [c["label"] for c in clusters] + self.assertTrue( + any("'" in l and "alentin" in l.lower() for l in labels), + f"expected an apostrophe label in {labels}", + ) + + def test_suggest_clusters_multi_bigram_supersedes_unigram(self): + """A unigram drops when the UNION of bigrams covering it + collectively exceeds the supersession threshold. Mirrors the + printableprompts gotcha: 'Day' shouldn't surface when + 'Valentine's Day' + 'Patrick's Day' + 'Christmas' bigrams + already cover all its products.""" + from types import SimpleNamespace + from ..lib.tag_suggest import suggest_clusters + # 4 products: two carry "Valentine's Day", two carry + # "Patrick's Day". "Day" appears in all 4 but is fully + # covered by the union of the two bigrams. + products = [ + SimpleNamespace(id="a", title="Valentine's Day Math", + description=""), + SimpleNamespace(id="b", title="Valentine's Day Reading", + description=""), + SimpleNamespace(id="c", title="Patrick's Day Math", + description=""), + SimpleNamespace(id="d", title="Patrick's Day Activities", + description=""), + ] + clusters, _ = suggest_clusters( + products, stopwords=[], existing_tag_slugs=[], + min_products=2, max_share=1.0, min_title_share=0.0, + bigrams=True, + ) + labels = [c["label"] for c in clusters] + # Bigrams survive + self.assertTrue(any("alentin" in l.lower() for l in labels)) + self.assertTrue(any("atrick" in l.lower() for l in labels)) + # "Day" unigram drops — collectively covered by both bigrams + self.assertNotIn("Day", labels) +