diff --git a/CLAUDE.md b/CLAUDE.md index 05dcd0d..7bd51f6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -587,7 +587,12 @@ and `Product.excerpt_sentences()` both consume the module-level Phase 2 (shipped): deterministic title-plus-description auto-tagger in `lib/tag_suggest.py`. Title tokens weight × 3, description × 1 -(capped at 100 unique tokens per product). Pipeline: tokenize → +(`DESCRIPTION_TOKEN_CAP=400` unique tokens per product — raised from +100 because long teaching-resource descriptions truncated +cross-cutting words like `holiday`/`seasonal` before they were +counted). Returns up to `DEFAULT_TOP_N=500` clusters (raised from +100 — large catalogues had real groups ranking past the cut). +Pipeline: tokenize → ~200-word English + per-shop stopwords → suffix-strip stem → form **bigrams** from adjacent non-stopword tokens (`write room`, `first grade`, `valentine day` — phrases get 2× unigram weight) → @@ -602,7 +607,8 @@ count desc → label = most frequent original word/phrase. Returns cluster (`action=apply_suggestion` / `action=dismiss_suggestion`). URL knobs: `?max_share=0.3` (stricter shop-vocab cut), `?max_share=1` (disable), `?min_title=0.5` (stricter title-required), `?min_title=0` -(disable), `?bigrams=0` (disable phrases), `?top_n=200` (show more). +(disable), `?bigrams=0` (disable phrases), `?top_n=N` (show more; +default 500, clamped ≤5000). CLI: `python -m make_post_sell.scripts.backfill_tags data/development.ini --shop= [--max-share=0.4] [--min-title-share=0.3] [--no-bigrams] [--apply]`. **Never diff --git a/docs/tickets/mps-24.md b/docs/tickets/mps-24.md index 6865539..49e436d 100644 --- a/docs/tickets/mps-24.md +++ b/docs/tickets/mps-24.md @@ -363,6 +363,20 @@ Tests (`test_functional.py::TestProductTagsSpa`): Deferred (occasional click, not the hot path): AJAX-ifying the "Suggest categories" link — still a full navigation by design. +**Phase 2.8k — auto-suggest: way more, stop missing `holiday`** +(shipped 2026-05-16): operator: "100 suggested tags is not enough, we +need way more — missing holiday holidays". Two `100` caps in +`lib/tag_suggest.py`: `DESCRIPTION_TOKEN_CAP` 100 → **400** (long +teaching-resource descriptions truncated cross-cutting words like +`holiday`/`holidays`/`seasonal` before they were ever counted — +verified neither word is a stopword) and `DEFAULT_TOP_N` 100 → **500** +(481-product catalogue had valid groups ranking past the cut; the +min_products / max_share / min_title_share filters already strip +noise, so a high ceiling surfaces the long tail safely). `?top_n=` +URL clamp raised 500 → 5000 for headroom. Both caps stay bounded +(deduped unique tokens / no unbounded query — CWE-407-safe). Test: +`test_deep_description_word_surfaces_after_cap_raise`. + **Phase 2.8j — drop redundant tag-detail header** (shipped 2026-05-16): operator: remove the tag title + "← All products" from the top of the tag SERP. With the always-on chip strip (active diff --git a/make_post_sell/lib/tag_suggest.py b/make_post_sell/lib/tag_suggest.py index b85574d..a0d7d18 100644 --- a/make_post_sell/lib/tag_suggest.py +++ b/make_post_sell/lib/tag_suggest.py @@ -16,10 +16,14 @@ import re from collections import Counter, defaultdict -# Word count cap per product for the description side of the input, to -# stop a long blog post from drowning the catalog signal. Title is never -# capped (titles are short by construction). -DESCRIPTION_TOKEN_CAP = 100 +# Unique-token cap per product for the description side of the input, +# to stop a single long blog post from drowning the catalog signal. +# Title is never capped (titles are short by construction). +# Raised 100 → 400: on long teaching-resource descriptions the old cap +# truncated cross-cutting words like "holiday"/"holidays"/"seasonal" +# before they were ever counted, so those categories never surfaced. +# Still bounded (deduped unique tokens per product) — CWE-407-safe. +DESCRIPTION_TOKEN_CAP = 400 # Title weight relative to description; tokens from the title count this # many times when scoring stem frequency per product. @@ -31,9 +35,13 @@ DESCRIPTION_WEIGHT = 1 DEFAULT_MIN_PRODUCTS = 2 # Default cap on how many candidate clusters we return per call. -# Set generously — large catalogues (printableprompts has 481) carry many -# valid niche categories beyond the obvious top 20. -DEFAULT_TOP_N = 100 +# Large catalogues (printableprompts has 481) carry many valid niche +# categories — at 100 the operator was missing real groups like +# "holiday"/"holidays" that ranked past the cut. Set high; the +# downstream filters (min_products, max_share, min_title_share) already +# remove noise, so a generous ceiling surfaces the long tail without +# resurfacing junk. Operator can still narrow via ?top_n=. +DEFAULT_TOP_N = 500 # A stem appearing in more than this fraction of products is treated as # **shop vocabulary** — words the operator uses to describe everything diff --git a/make_post_sell/tests/test_models.py b/make_post_sell/tests/test_models.py index ab36d03..e3bc30a 100644 --- a/make_post_sell/tests/test_models.py +++ b/make_post_sell/tests/test_models.py @@ -5278,6 +5278,40 @@ class TestTagSuggestPureFunctions(unittest.TestCase): for c in clusters: self.assertGreaterEqual(len(c["product_ids"]), 2) + def test_deep_description_word_surfaces_after_cap_raise(self): + """Operator: 'missing holiday holidays'. On long teaching-resource + descriptions a cross-cutting word that appears AFTER the first + 100 unique description tokens used to be truncated by + DESCRIPTION_TOKEN_CAP and never clustered. With the cap raised + (400) it now surfaces.""" + from types import SimpleNamespace + from ..lib.tag_suggest import suggest_clusters, DESCRIPTION_TOKEN_CAP + self.assertGreaterEqual(DESCRIPTION_TOKEN_CAP, 400) + # 150 unique filler tokens, THEN the signal word near the end — + # past the old 100 cap, within the new 400 cap. + filler = " ".join("filler%d" % i for i in range(150)) + products = [ + SimpleNamespace( + id="a", title="Addition Pack", + description=filler + " holiday holiday season fun", + ), + SimpleNamespace( + id="b", title="Counting Pack", + description=filler + " holiday holiday season fun", + ), + ] + clusters, _ = suggest_clusters( + products, + stopwords=[], + existing_tag_slugs=[], + min_products=2, + max_share=1.0, + min_title_share=0.0, # description-only word + bigrams=False, + ) + labels = [c["label"].lower() for c in clusters] + self.assertIn("holiday", labels) + def test_suggest_clusters_skips_existing_tags(self): from types import SimpleNamespace from ..lib.tag_suggest import suggest_clusters diff --git a/make_post_sell/views/shop.py b/make_post_sell/views/shop.py index cc7991d..32d4732 100644 --- a/make_post_sell/views/shop.py +++ b/make_post_sell/views/shop.py @@ -2496,7 +2496,10 @@ def shop_tags(request): return default return max(lo, min(hi, v)) - def _int_param(name, default, lo=1, hi=500): + # Only used for ?top_n=. Ceiling raised 500 → 5000 so the + # operator has real headroom above DEFAULT_TOP_N (500) on huge + # catalogues; still bounded (no unbounded query / DoS). + def _int_param(name, default, lo=1, hi=5000): raw = (request.params.get(name) or "").strip() if not raw: return default