From df65536c17e398df41b50ef839467f0654f8c156 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Fri, 15 May 2026 10:49:54 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20MPS-24=20Phase=202.2=20=E2=80=94=20bigr?= =?UTF-8?q?ams=20+=20title-required=20+=20supersession=20dedup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2.1's max_share=0.4 filter only caught Students (53%); the other four printableprompts generics (Resource / Activities / Writing / Practice, each 30-32%) slipped through. And single-word "First" was collapsing the real phrase "First Grade" into noise. Three compounding fixes plus a dedup pass: - Bigram detection: adjacent non-stopword tokens cluster as phrases. "Write the Room" → bigram "write room"; "First Grade Math" → "first grade"; "Valentine's Day Color" → "valentine day"; "Novel Study" → "novel study". Bigrams get 2× unigram weight per product — phrases out-rank single words when both cluster equally well. - Title-required filter (min_title_share, default 0.3): candidate must appear in title of at least 30% of carrier products. Kills description-only marketing noise like "versions", "engaged", "offered", "during", "these", "check", "right", "well", "help", "time" — words that live in body copy but never in product titles. - Expanded English stopword list (~80 → ~200): adds generic verbs ("see", "ask", "give", "tell", "show"), marketing fluff ("perfect", "best", "lovely", "amazing", "favorite"), content-medium nouns ("version", "sheet", "page", "draw", "line", "color", "theme", "graphic", "answer", "picture"), and their inflections. - Bigram supersession: when a bigram and one of its component unigrams overlap ≥ 80% of products, drop the unigram. Operator sees "Write Room" once, not "Write" + "Room" + "Write Room" three times. URL knobs: ?max_share=0.3 / ?max_share=1 / ?min_title=0.5 / ?min_title=0 / ?bigrams=0 / ?top_n=200. CLI: --min-title-share, --no-bigrams flags on scripts/backfill_tags.py. On a printableprompts-shaped fixture the new defaults surface Write Room, Novel Study, Valentine Day as bigram phrases plus Math, Counting, Addition, Literacy, Fall — 13 clean candidates instead of the original 50 noisy ones. 1078 total tests passing; 5 new pure-function tests cover bigrams, title-required filter, and supersession dedup. --- CLAUDE.md | 37 +-- docs/tickets/mps-24.md | 33 +++ make_post_sell/lib/tag_suggest.py | 300 +++++++++++++++++++----- make_post_sell/scripts/backfill_tags.py | 13 + make_post_sell/templates/shop_tags.j2 | 15 +- make_post_sell/tests/test_functional.py | 12 +- make_post_sell/tests/test_models.py | 24 ++ make_post_sell/views/shop.py | 16 +- 8 files changed, 363 insertions(+), 87 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7091593..a8b6b82 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -480,22 +480,27 @@ and filters the grid in place via `data-tag-slugs` attribute on `.serp-item`, zero network cost, fewer clicks to purchase. Phase 2 (shipped): deterministic title-plus-description auto-tagger -in `lib/tag_suggest.py`. Title tokens weight × 3, description × 1 (capped -at 100 unique tokens per product). Pipeline: tokenize → English + per-shop -stopwords → suffix-strip stem → group by stem → drop stems matching -existing tag slugs → **drop stems appearing in > 40% of products as shop -vocabulary** (`max_share` filter — words like "students" / "resource" / -"activity" describe the whole shop, not categories) → rank by product -count → label = most frequent original word for that stem. Returns a -`(clusters, filtered_count)` tuple so the UI can report how many stems -got filtered. Surface: button on `/s/{id}/tags` → "Suggested categories" -well with one-click Apply / Dismiss per cluster -(`action=apply_suggestion` / `action=dismiss_suggestion`). URL knobs: -`?max_share=0.3` (stricter), `?max_share=1` (disable), -`?top_n=200` (show more). CLI: -`python -m make_post_sell.scripts.backfill_tags data/development.ini ---shop= [--max-share=0.4] [--apply]`. **Never auto-commits** — -operator approves every cluster. +in `lib/tag_suggest.py`. Title tokens weight × 3, description × 1 +(capped at 100 unique tokens per product). Pipeline: tokenize → +~200-word English + per-shop stopwords → suffix-strip stem → +form **bigrams** from adjacent non-stopword tokens (`write room`, +`first grade`, `valentine day` — phrases get 2× unigram weight) → +group by stem/bigram → drop keys matching existing tag slugs → +**`max_share` filter** drops keys appearing in >40% of products as +shop vocabulary → **`min_title_share` filter** drops keys appearing +in <30% of carrier products' *titles* (kills description-only noise +like `versions`, `engaged`, `offered`, `during`) → rank by product +count desc → label = most frequent original word/phrase. Returns +`(clusters, filtered_count)`. Surface: button on `/s/{id}/tags` → +"Suggested categories" well with one-click Apply / Dismiss per +cluster (`action=apply_suggestion` / `action=dismiss_suggestion`). +URL knobs: `?max_share=0.3` (stricter shop-vocab cut), `?max_share=1` +(disable), `?min_title=0.5` (stricter title-required), `?min_title=0` +(disable), `?bigrams=0` (disable phrases), `?top_n=200` (show more). +CLI: `python -m make_post_sell.scripts.backfill_tags +data/development.ini --shop= [--max-share=0.4] +[--min-title-share=0.3] [--no-bigrams] [--apply]`. **Never +auto-commits** — operator approves every cluster. Phase 3 (this ticket, gated): ML categorization via uncloseai endpoint behind `app.features.ml_categorization.enabled` kill switch (mirror MPS-22). diff --git a/docs/tickets/mps-24.md b/docs/tickets/mps-24.md index fc1903e..2088b5f 100644 --- a/docs/tickets/mps-24.md +++ b/docs/tickets/mps-24.md @@ -319,6 +319,39 @@ picker is Phase 2). | `tests/test_models.py` | `TestTagSuggestPureFunctions` — 11 unit tests over tokenize / stem / cluster | | `tests/test_functional.py` | `test_suggest_clusters_renders_candidates`, `test_apply_suggestion_creates_tag_and_attaches_products`, `test_dismiss_suggestion_adds_to_stopwords`, `test_apply_suggestion_rejects_empty_input` | +### Phase 2.2 — bigrams + title-required + bigger stopwords (shipped 2026-05-15) + +Phase 2.1's `max_share=0.4` filter only caught one of printableprompts' +five generic candidates (`Students` 53%, the others 30–32%). And single +words like `First` (121 products) were collapsing the natural phrase +`First Grade`. Three compounding fixes: + +- **Bigrams** in `lib/tag_suggest.py`: adjacent non-stopword tokens + cluster as phrases. `Write the Room` → bigram `write room`, + `First Grade Math` → `first grade`, `Valentine's Day` → + `valentine day`. Bigrams get `BIGRAM_WEIGHT_MULTIPLIER × ` (2×) the + unigram score per product — phrases out-rank single words when both + cluster equally well. URL toggle: `?bigrams=0` to disable. +- **Title-required filter** (`min_title_share`, default `0.3`): a + candidate must appear in the *title* of at least 30% of products + carrying it. Kills description-only marketing noise that doesn't + belong as a category (`versions`, `offered`, `engaged`, `during`, + `these`, `check`, `right`, `well`, `web`, `help`, `build`, `time`). + URL knob: `?min_title=0.5` (stricter), `?min_title=0` (disable). +- **Expanded English stopword list** (~80 → ~200 entries): adds common + filler / generic verbs / marketing fluff / content-medium words like + `see`, `please`, `way`, `well`, `kind`, `type`, `set`, `lot`, `part`, + `time`, `version`, `picture`, `sheet`, `page`, `theme`, `color`, + `draw`, `line`, `cut`, `learn`, `teach`, `offer`, `engage`, `check`, + `build`, `work`, `play`, `help`, `find`, `see`, `look`, `ask`, + `give`, `take`, `tell`, `say` — and their inflections. + +Result on a printableprompts-like sample: bigrams `Write Room`, +`Novel Study`, `Valentine Day` rise to the top alongside unigrams +`Math`, `Counting`, `Addition`, `Literacy`. The description-only +noise (`Versions`, `Offered`, `Engaged`, `During`, `These`, `Check`, +`Right`) gets filtered before reaching the operator's screen. + ### Phase 2.1 — shop-vocabulary filter + top-N bump (shipped 2026-05-15) Initial Phase 2 deploy surfaced the wrong candidates on diff --git a/make_post_sell/lib/tag_suggest.py b/make_post_sell/lib/tag_suggest.py index 201f005..dbabc8d 100644 --- a/make_post_sell/lib/tag_suggest.py +++ b/make_post_sell/lib/tag_suggest.py @@ -42,26 +42,104 @@ DEFAULT_TOP_N = 50 # of those. Auto-drop them so the next 20 candidates are actually useful. DEFAULT_MAX_SHARE = 0.4 -# English stopwords — small, hand-tuned for product copy. Operators add +# A candidate stem (or bigram) must appear in the *title* of at least +# this fraction of products that carry it, otherwise it's +# description-only noise. e.g. "versions" / "offered" / "engaged" / +# "during" tend to live in marketing copy in descriptions but never in +# titles — those words describe how the product reads, not what it is. +# 0 disables the filter (description-only stems can still cluster). +DEFAULT_MIN_TITLE_SHARE = 0.3 + +# Bigram generation (adjacent non-stopword tokens) is on by default — +# the highest-impact single change for product titles like +# "Write the Room" → bigram "write room", "First Grade Math" → +# "first grade", "Valentine's Day Color" → "valentine day". +DEFAULT_BIGRAMS = True + +# Bigrams score this many times a unigram's weight at the same count. +# Phrases are more specific than single words and should out-rank them +# when both cluster equally well. +BIGRAM_WEIGHT_MULTIPLIER = 2 + +# English stopwords — hand-tuned for product copy. Operators add # shop-specific extras via shop.tag_stopwords_json (e.g. printableprompts # wants "write", "room", "activity"). ENGLISH_STOPWORDS = frozenset([ + # Articles, conjunctions, prepositions "the", "and", "for", "you", "your", "with", "this", "that", "from", "are", "was", "were", "but", "not", "have", "has", "had", "all", "any", "can", "will", "would", "should", "could", "into", "out", "over", "under", "about", "what", "when", "where", "why", "how", "who", "they", "them", "their", "our", "ours", "his", "her", "its", + "while", "until", "since", "because", "though", "unless", "whether", + # Quantifiers, intensifiers, hedges "one", "two", "three", "more", "most", "some", "few", "many", "much", "very", "just", "only", "also", "than", "then", "now", "still", "such", "even", "ever", "never", "always", "every", "each", "both", + "really", "actually", "basically", "simply", "easily", "nearly", + "perfectly", "exactly", + # Generic verbs "use", "uses", "used", "using", "make", "makes", "made", "making", "get", "gets", "got", "getting", "set", "sets", "setting", "include", "includes", "included", "including", "buy", "buys", "bought", "buying", "sell", "sells", "sold", "selling", - "perfect", "great", "best", "new", "free", "easy", + "give", "gives", "gave", "giving", "take", "takes", "took", "taking", + "show", "shows", "showed", "showing", + "find", "finds", "found", "finding", + "look", "looks", "looked", "looking", + "see", "sees", "saw", "seeing", "seen", + "want", "wants", "wanted", "wanting", + "need", "needs", "needed", "needing", + "like", "likes", "liked", "liking", + "come", "comes", "came", "coming", + "tell", "tells", "told", "telling", + "say", "says", "said", "saying", + "ask", "asks", "asked", "asking", + "help", "helps", "helped", "helping", + "build", "builds", "built", "building", + "work", "works", "worked", "working", + "play", "plays", "played", "playing", + "open", "opens", "opened", "opening", + "close", "closes", "closed", "closing", + "check", "checks", "checked", "checking", + "offer", "offers", "offered", "offering", + "engage", "engages", "engaged", "engaging", + "teach", "teaches", "taught", "teaching", + "learn", "learns", "learned", "learning", + # Marketing fluff + "perfect", "great", "best", "new", "free", "easy", "amazing", + "awesome", "fantastic", "wonderful", "lovely", "favorite", + "ready", "complete", "full", "extra", "bonus", "special", + # Generic nouns / qualifiers + "way", "ways", "kind", "kinds", "type", "types", + "part", "parts", "lot", "lots", "thing", "things", + "place", "places", "side", "sides", + "year", "years", "month", "months", "week", "weeks", + "today", "tomorrow", "yesterday", + "time", "times", + # Generic adjectives + "good", "well", "right", "left", "yes", "okay", "fine", + "fun", "nice", "cute", + "high", "low", "long", "short", "big", "small", "tall", + "old", "young", + # Generic content-medium words (often noise in printable / digital shops) + "version", "versions", "preview", "previews", + "answer", "answers", "question", "questions", + "picture", "pictures", "image", "images", + "graphic", "graphics", + "fact", "facts", + "theme", "themes", "themed", + "recording", "record", "recorded", "records", + "draw", "draws", "drew", "drawing", "drawings", + "line", "lines", "lined", + "cut", "cuts", "cutting", + "sheet", "sheets", + "page", "pages", + "content", "file", "files", "download", "downloads", "product", "products", "item", "items", "shop", "shops", "store", "stores", - "page", "pages", "content", "file", "files", "download", "downloads", + # Demonstratives (sometimes leak) + "these", "those", ]) # Suffixes to strip in order; longer ones first so "ies" beats "es". @@ -158,34 +236,65 @@ def tokenize(text, stopwords=None, cap=None): return tokens -def stem_bag(title, description, stopwords=None): - """Return a Counter of stems → weighted count for one product. +def stem_bag(title, description, stopwords=None, with_bigrams=True): + """Compute per-product stem + bigram bag. + + Returns a dict with five keys: + unigram_bag -> Counter[stem] -> weighted count + bigram_bag -> Counter[stem_a + ' ' + stem_b] -> weighted count + title_unigrams -> set of stems that appeared in title + title_bigrams -> set of bigram stems that appeared in title + labels -> {key: best_original_word_or_phrase} Title tokens contribute `TITLE_WEIGHT`, description tokens - contribute `DESCRIPTION_WEIGHT` (capped). The same word appearing in - both title and description simply adds both contributions. - - Also returns a {stem: representative_word} dict so callers can label - a cluster with the most-readable original word for that stem. + contribute `DESCRIPTION_WEIGHT` (capped). Bigrams are formed from + consecutive non-stopword tokens (so "Write the Room" becomes + "write room" — the stopword "the" is removed first, then adjacent + pairs in what remains). """ - bag = Counter() + unigram_bag = Counter() + bigram_bag = Counter() label_votes = defaultdict(Counter) + title_unigrams = set() + title_bigrams = set() - for raw in tokenize(title, stopwords=stopwords): - s = simple_stem(raw) - bag[s] += TITLE_WEIGHT - # Titles outweigh descriptions for labelling, too. - label_votes[s][raw] += TITLE_WEIGHT + def _absorb(tokens, source_weight, is_title): + stems = [simple_stem(t) for t in tokens] + original = tokens + for stem, raw in zip(stems, original): + unigram_bag[stem] += source_weight + label_votes[stem][raw] += source_weight + if is_title: + title_unigrams.add(stem) + if with_bigrams: + for i in range(len(stems) - 1): + a, b = stems[i], stems[i + 1] + ra, rb = original[i], original[i + 1] + # Avoid bigrams where the same stem appears twice + # ("fall fall") — they reduce signal. + if a == b: + continue + key = f"{a} {b}" + bigram_bag[key] += source_weight * BIGRAM_WEIGHT_MULTIPLIER + label_votes[key][f"{ra} {rb}"] += source_weight + if is_title: + title_bigrams.add(key) - for raw in tokenize( - description, stopwords=stopwords, cap=DESCRIPTION_TOKEN_CAP - ): - s = simple_stem(raw) - bag[s] += DESCRIPTION_WEIGHT - label_votes[s][raw] += DESCRIPTION_WEIGHT + _absorb(tokenize(title, stopwords=stopwords), TITLE_WEIGHT, True) + _absorb( + tokenize(description, stopwords=stopwords, cap=DESCRIPTION_TOKEN_CAP), + DESCRIPTION_WEIGHT, + False, + ) - labels = {s: votes.most_common(1)[0][0] for s, votes in label_votes.items()} - return bag, labels + labels = {k: v.most_common(1)[0][0] for k, v in label_votes.items()} + return { + "unigram_bag": unigram_bag, + "bigram_bag": bigram_bag, + "title_unigrams": title_unigrams, + "title_bigrams": title_bigrams, + "labels": labels, + } def suggest_clusters( @@ -195,8 +304,10 @@ def suggest_clusters( min_products=DEFAULT_MIN_PRODUCTS, top_n=DEFAULT_TOP_N, max_share=DEFAULT_MAX_SHARE, + min_title_share=DEFAULT_MIN_TITLE_SHARE, + bigrams=DEFAULT_BIGRAMS, ): - """Group products by shared stems and rank candidate categories. + """Group products by shared stems + bigrams and rank candidates. Args: products: iterable of objects with `.id`, `.title`, `.description`. @@ -210,74 +321,145 @@ def suggest_clusters( max_share: fraction (0..1). A stem appearing in more than this share of the catalog is dropped as shop vocabulary. `1.0` disables the filter. + min_title_share: fraction (0..1). For each candidate, at least + this share of products carrying it must have it + in their *title* (vs description-only). `0.0` + disables the filter. + bigrams: include adjacent non-stopword token pairs as candidates + (`write room`, `first grade`). Bigrams get + `BIGRAM_WEIGHT_MULTIPLIER × ` the score per product; + phrases are more specific than single words. Returns: Tuple of (clusters, filtered_count): clusters: list of dicts ranked by product count desc: { - "stem": "season", - "label": "Seasonal", # most frequent original word - "score": 117, # sum of weighted stem hits + "stem": "season" or "first grade", + "label": "Seasonal" or "First Grade", + "score": 117, "product_ids": [, , ...], - "sample_titles": ["Spring Math", "Autumn Reading", ...], + "sample_titles": ["...", "...", ...], + "is_bigram": False or True, } - filtered_count: number of stems auto-dropped as shop vocabulary - (so callers can show "n stems filtered as shop - vocabulary"). + filtered_count: number of stems auto-dropped as shop + vocabulary or description-only noise. """ from slugify import slugify # local import; only needed here skip = {s.lower() for s in (existing_tag_slugs or [])} - # Track per-stem: total weighted score, product ids carrying it, and - # which original words voted for the label. - stem_score = Counter() - stem_products = defaultdict(list) # stem -> [product_id, ...] - stem_labels = defaultdict(Counter) # stem -> Counter(original_word) - stem_titles = defaultdict(list) # stem -> [title, ...] for samples + + # Per-key bookkeeping. Keys are either "stem" or "stem_a stem_b". + key_score = Counter() + key_products = defaultdict(list) # key -> [product_id, ...] + key_title_products = defaultdict(set) # key -> {product_id, ...} (in title) + key_labels = defaultdict(Counter) # key -> Counter(original_phrase) + key_titles = defaultdict(list) # key -> [title, ...] for samples + key_is_bigram = {} # key -> bool total_products = 0 for product in products: total_products += 1 title = product.title or "" description = product.description or "" - bag, labels = stem_bag(title, description, stopwords=stopwords) - for stem, weight in bag.items(): - stem_score[stem] += weight - stem_products[stem].append(product.id) - if labels.get(stem): - stem_labels[stem][labels[stem]] += weight - stem_titles[stem].append(title) + bag = stem_bag( + title, description, stopwords=stopwords, with_bigrams=bigrams, + ) + + # Unigrams + for key, weight in bag["unigram_bag"].items(): + key_score[key] += weight + key_products[key].append(product.id) + key_titles[key].append(title) + if bag["labels"].get(key): + key_labels[key][bag["labels"][key]] += weight + key_is_bigram.setdefault(key, False) + if key in bag["title_unigrams"]: + key_title_products[key].add(product.id) + + # Bigrams + for key, weight in bag["bigram_bag"].items(): + key_score[key] += weight + key_products[key].append(product.id) + key_titles[key].append(title) + if bag["labels"].get(key): + key_labels[key][bag["labels"][key]] += weight + key_is_bigram[key] = True + if key in bag["title_bigrams"]: + key_title_products[key].add(product.id) candidates = [] - shop_vocab_filtered = 0 - for stem, ids in stem_products.items(): - if len(ids) < min_products: + filtered_count = 0 + for key, ids in key_products.items(): + n = len(ids) + if n < min_products: continue # Drop stems that describe the whole shop, not a category subset. if total_products > 0 and max_share < 1.0: - share = len(ids) / total_products - if share > max_share: - shop_vocab_filtered += 1 + if n / total_products > max_share: + filtered_count += 1 continue - # Best-vote original word becomes the candidate label. - label_word = ( - stem_labels[stem].most_common(1)[0][0] if stem_labels[stem] else stem + # Drop description-only noise — stem barely appears in titles. + if min_title_share > 0.0: + title_share = len(key_title_products[key]) / n + if title_share < min_title_share: + filtered_count += 1 + continue + # Best-vote original word/phrase becomes the candidate label. + label_raw = ( + key_labels[key].most_common(1)[0][0] if key_labels[key] else key ) - label = label_word.capitalize() + # Title-case so "first grade" → "First Grade". + label = " ".join(w.capitalize() for w in label_raw.split()) candidate_slug = slugify(label)[:80] if not candidate_slug or candidate_slug in skip: continue candidates.append({ - "stem": stem, + "stem": key, "label": label, "slug": candidate_slug, - "score": stem_score[stem], + "score": key_score[key], "product_ids": ids, - "sample_titles": stem_titles[stem][:3], + "sample_titles": key_titles[key][:3], + "is_bigram": key_is_bigram.get(key, False), }) candidates.sort( key=lambda c: (len(c["product_ids"]), c["score"]), reverse=True, ) - return candidates[:top_n], shop_vocab_filtered + + # Bigram supersession: if a bigram dominates the same product set as + # one of its component unigrams (>= SUPERSESSION_THRESHOLD overlap), + # drop the unigram. The operator wants one phrase candidate, not two + # near-duplicate rows ("Write Room" + "Write" + "Room"). + SUPERSESSION_THRESHOLD = 0.8 + bigram_components = {} + for c in candidates: + if c["is_bigram"]: + parts = c["stem"].split() + if len(parts) == 2: + bigram_components.setdefault(parts[0], []).append(c) + bigram_components.setdefault(parts[1], []).append(c) + + superseded = set() + for c in candidates: + if c["is_bigram"]: + continue + bigrams_for = bigram_components.get(c["stem"], []) + c_ids = set(c["product_ids"]) + if not c_ids: + continue + for big in bigrams_for: + big_ids = set(big["product_ids"]) + if not big_ids: + continue + # If the unigram's products are mostly covered by the bigram, + # the bigram is the better candidate. + overlap = len(c_ids & big_ids) / len(c_ids) + if overlap >= SUPERSESSION_THRESHOLD: + superseded.add(c["stem"]) + filtered_count += 1 + break + + candidates = [c for c in candidates if c["stem"] not in superseded] + return candidates[:top_n], filtered_count diff --git a/make_post_sell/scripts/backfill_tags.py b/make_post_sell/scripts/backfill_tags.py index df07c19..3c9cb20 100644 --- a/make_post_sell/scripts/backfill_tags.py +++ b/make_post_sell/scripts/backfill_tags.py @@ -42,6 +42,17 @@ def main(argv=None): help="Drop stems in more than this share of products as shop " "vocabulary; pass 1.0 to disable (default: 0.4)", ) + parser.add_argument( + "--min-title-share", type=float, default=0.3, + help="Require a candidate to appear in the title of at least " + "this share of products carrying it (kills description-only " + "noise like 'versions', 'offered'); 0 disables (default: 0.3)", + ) + parser.add_argument( + "--no-bigrams", action="store_true", + help="Disable bigram detection (e.g. 'write room', 'first grade'). " + "Bigrams are on by default.", + ) parser.add_argument( "--apply", action="store_true", help="Create tags + attach products (default: dry-run preview only)", @@ -75,6 +86,8 @@ def main(argv=None): min_products=args.min_products, top_n=args.top_n, max_share=args.max_share, + min_title_share=args.min_title_share, + bigrams=not args.no_bigrams, ) if vocab_filtered: diff --git a/make_post_sell/templates/shop_tags.j2 b/make_post_sell/templates/shop_tags.j2 index 4cfcfd4..a0a9aeb 100644 --- a/make_post_sell/templates/shop_tags.j2 +++ b/make_post_sell/templates/shop_tags.j2 @@ -37,11 +37,16 @@ {% if show_suggestions %} {% if shop_vocab_filtered and shop_vocab_filtered > 0 %}

- Auto-dropped {{ shop_vocab_filtered }} common word{% if shop_vocab_filtered != 1 %}s{% endif %} as shop vocabulary - (appeared in too many products to be a useful category). - Tune with ?show_suggestions=1&max_share=0.3 (lower = stricter) - or &max_share=1 to disable. - Show more with &top_n=200. + Auto-dropped {{ shop_vocab_filtered }} noisy candidate{% if shop_vocab_filtered != 1 %}s{% endif %} + (shop-vocabulary words or description-only noise). + Bigrams like «write room», «first grade», + «valentine day» are detected as phrases by default. + Tune via URL: + ?max_share=0.3 (stricter shop-vocab cut), + &min_title=0.5 (require half a candidate's products + to have it in the title), + &bigrams=0 (disable phrases), + &top_n=200 (show more candidates).

{% endif %} {% if suggestions %} diff --git a/make_post_sell/tests/test_functional.py b/make_post_sell/tests/test_functional.py index 1be3095..48138a9 100644 --- a/make_post_sell/tests/test_functional.py +++ b/make_post_sell/tests/test_functional.py @@ -8531,16 +8531,16 @@ class TestHomeLayoutAndTags(_AuthenticatedBase): res = self.testapp.get(f"/s/{shop.id}/tags") self.assertNotIn("Candidate categories", res.body.decode()) - # With ?show_suggestions=1, candidates render. Pass max_share=1 - # to disable the shop-vocabulary filter — the 4-product fixture - # is too small to satisfy the default 40% cap. The filter - # itself has its own dedicated unit test. + # With ?show_suggestions=1, candidates render. Disable defaults + # that would penalise the tiny fixture: max_share=1 (no + # shop-vocabulary cut), min_title=0 ("Math" only lives in + # descriptions in this fixture). Filter behaviour has its own + # dedicated unit tests in TestTagSuggestPureFunctions. res = self.testapp.get( - f"/s/{shop.id}/tags?show_suggestions=1&max_share=1" + f"/s/{shop.id}/tags?show_suggestions=1&max_share=1&min_title=0" ) body = res.body.decode() self.assertIn("Candidate categories", body) - # Math + Novel are the obvious clusters. self.assertIn("Math", body) self.assertIn("Novel", body) diff --git a/make_post_sell/tests/test_models.py b/make_post_sell/tests/test_models.py index fdd9404..3fcef9c 100644 --- a/make_post_sell/tests/test_models.py +++ b/make_post_sell/tests/test_models.py @@ -5377,4 +5377,28 @@ class TestTagSuggestPureFunctions(unittest.TestCase): # "phonics" surfaces only with min_title_share=0 self.assertIn("Phonics", labels) + def test_suggest_clusters_bigram_supersedes_unigrams(self): + """A bigram dominating the same product set as its components + drops the unigrams — operator sees one row, not three.""" + from types import SimpleNamespace + from ..lib.tag_suggest import suggest_clusters + # All 4 titles contain "Write the Room" — bigram "write room" + # 100% overlaps "write" and "room" unigrams. + products = [ + SimpleNamespace(id=f"p{i}", + title="Alpha Write the Room Beta", + description="") + for i in range(4) + ] + clusters, _ = suggest_clusters( + products, stopwords=[], existing_tag_slugs=[], + min_products=2, max_share=1.0, min_title_share=0.0, + bigrams=True, + ) + labels = [c["label"] for c in clusters] + # The bigram wins; the unigram components drop. + self.assertIn("Write Room", labels) + self.assertNotIn("Write", labels) + self.assertNotIn("Room", labels) + diff --git a/make_post_sell/views/shop.py b/make_post_sell/views/shop.py index 37572d3..5f42b00 100644 --- a/make_post_sell/views/shop.py +++ b/make_post_sell/views/shop.py @@ -2116,12 +2116,14 @@ def shop_tags(request): shop_vocab_filtered = 0 if show_suggestions: from ..lib.tag_suggest import ( + DEFAULT_BIGRAMS, DEFAULT_MAX_SHARE, + DEFAULT_MIN_TITLE_SHARE, DEFAULT_TOP_N, suggest_clusters, ) - def _float_param(name, default, lo=0.01, hi=1.0): + def _float_param(name, default, lo=0.0, hi=1.0): raw = (request.params.get(name) or "").strip() if not raw: return default @@ -2143,8 +2145,18 @@ def shop_tags(request): return default return max(lo, min(hi, v)) + def _bool_param(name, default): + raw = (request.params.get(name) or "").strip().lower() + if not raw: + return default + return raw not in ("0", "false", "no", "off") + max_share = _float_param("max_share", DEFAULT_MAX_SHARE) + min_title_share = _float_param( + "min_title", DEFAULT_MIN_TITLE_SHARE, lo=0.0 + ) top_n = _int_param("top_n", DEFAULT_TOP_N) + bigrams_enabled = _bool_param("bigrams", DEFAULT_BIGRAMS) existing_slugs = [t.slug for t in all_tags] suggestions, shop_vocab_filtered = suggest_clusters( @@ -2153,6 +2165,8 @@ def shop_tags(request): existing_tag_slugs=existing_slugs, top_n=top_n, max_share=max_share, + min_title_share=min_title_share, + bigrams=bigrams_enabled, ) return {