From 5dbbe697b6fcc47d8306960e6eae8bfa835e55e7 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Fri, 15 May 2026 09:09:40 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20MPS-24=20Phase=202=20=E2=80=94=20auto-s?= =?UTF-8?q?uggest=20tags=20from=20title=20+=20description?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator with 481 untagged products (printableprompts.com) gets a one-click path to a usable categorization without hand-tagging each product. Strictly suggest-then-approve — nothing writes Tag or ProductTag rows until the operator clicks Apply on a cluster. - lib/tag_suggest.py: pure-function clusterer. Tokenize title (weight 3) + description (weight 1, capped at 100 unique tokens per product), strip markdown / URLs / HTML, English + per-shop stopwords, simple suffix-strip stemmer, group by stem, drop stems matching existing tag slugs, rank by product count, label each cluster with the most frequent original word for its stem. No new deps, no ML. - scripts/backfill_tags.py: CLI preview + --apply for a single shop. - views/shop.py: shop_tags gains action=apply_suggestion (creates tag + bulk-attaches every product in cluster) and action=dismiss_suggestion (adds the cluster's words to shop.tag_stopwords_json so it never resurfaces). ?show_suggestions=1 triggers the cluster compute. - templates/shop_tags.j2: "Suggest categories from titles + descriptions" button + suggestions well with per-cluster sample titles, Apply, and Dismiss buttons. - 15 new tests (11 unit over tokenize / stem / cluster + 4 functional over the suggest/apply/dismiss flow). 1064 total passing. On a printableprompts-style sample the clusterer surfaces Math, Reading, Literacy, Seasonal, Novel, Activities, Comprehension — matching what an operator would manually pick. --- CLAUDE.md | 14 +- docs/architecture.md | 3 +- docs/design-system.md | 2 + docs/tickets/mps-24.md | 87 +++++--- make_post_sell/lib/tag_suggest.py | 257 ++++++++++++++++++++++++ make_post_sell/scripts/backfill_tags.py | 113 +++++++++++ make_post_sell/static/css/common.css | 51 +++++ make_post_sell/templates/shop_tags.j2 | 52 +++++ make_post_sell/tests/test_functional.py | 107 ++++++++++ make_post_sell/tests/test_models.py | 153 ++++++++++++++ make_post_sell/views/shop.py | 74 +++++++ 11 files changed, 884 insertions(+), 29 deletions(-) create mode 100644 make_post_sell/lib/tag_suggest.py create mode 100644 make_post_sell/scripts/backfill_tags.py diff --git a/CLAUDE.md b/CLAUDE.md index 11b0604..78ca5f2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -477,9 +477,17 @@ filters on no-JS. With JS, `static/js/tag_filter.js` intercepts clicks and filters the grid in place via `data-tag-slugs` attribute on `.serp-item`, zero network cost, fewer clicks to purchase. -Phase 2 (this ticket, follow-on commit): sectioned-lane layout (`==2`) -+ deterministic auto-tagger script (`scripts/backfill_tags.py`) that -clusters by shared title keywords minus stopwords. +Phase 2 (shipped): deterministic title-plus-description auto-tagger +in `lib/tag_suggest.py`. Title tokens weight × 3, description × 1 (capped +at 100 unique tokens per product). Pipeline: tokenize → English + per-shop +stopwords → suffix-strip stem → group by stem → drop stems matching +existing tag slugs → rank by product count → label = most frequent +original word for that stem. Surface: button on `/s/{id}/tags` → +"Suggested categories" well with one-click Apply / Dismiss per cluster +(`action=apply_suggestion` / `action=dismiss_suggestion`). CLI: +`python -m make_post_sell.scripts.backfill_tags data/development.ini +--shop= [--apply]`. **Never auto-commits** — operator approves +every cluster. Phase 3 (this ticket, gated): ML categorization via uncloseai endpoint behind `app.features.ml_categorization.enabled` kill switch (mirror MPS-22). diff --git a/docs/architecture.md b/docs/architecture.md index 8e088ea..ae6c95c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -215,6 +215,7 @@ mps_page_session (raw rows) | Tag chip / lane caps (MPS-24) | `shop.home_layout_tag_limit` / `shop.home_layout_per_lane_limit` | `home-layout-settings` | 8 / 10 | | Featured products strip (MPS-24) | `shop.featured_product_ids_json` | `home-layout-settings` | empty | | Product tags (MPS-24) | `Tag` + `ProductTag` association | product edit + `/s/{id}/tags` bulk editor | — | +| Tag auto-suggest (MPS-24 Phase 2) | `lib/tag_suggest.py` over `Product.title` + `Product.description` | `?show_suggestions=1` on `/s/{id}/tags` + `scripts/backfill_tags.py` | Never auto-applies | ## Ticket Index @@ -244,7 +245,7 @@ mps_page_session (raw rows) | [MPS-21](tickets/mps-21.md) | Make-an-Offer Mode | Complete | | [MPS-22](tickets/mps-22.md) | Kill-Switch Feature Flags — Karaoke + Torrent Off by Default | Complete | | [MPS-23](tickets/mps-23.md) | Consolidated Transactional Sender Identity + Shop Contact Email | Open | -| [MPS-24](tickets/mps-24.md) | Shop home page overhaul + product categorization (tags + chips + lanes) | In progress (Phase 1 landed) | +| [MPS-24](tickets/mps-24.md) | Shop home page overhaul + product categorization (tags + chips + lanes + auto-suggest) | In progress (Phases 1 + 2 landed) | ## Related Docs diff --git a/docs/design-system.md b/docs/design-system.md index 324689c..5ff4d3e 100644 --- a/docs/design-system.md +++ b/docs/design-system.md @@ -282,6 +282,8 @@ All components are documented with live examples at `/styleguide`. The styleguid | `[data-tag-strip]` | chip strip | JS hook for `tag_filter.js` | | `[data-tag-grid]` | flat `.serp` | JS hook — items inside carry `data-tag-slugs` for in-place filter | | `[data-tag-slugs]` | `.serp-item` | Space-separated tag slugs the item carries; consumed by `tag_filter.js` | +| `.tag-suggest-list` / `.tag-suggest-item` | `shop_tags.j2` (Phase 2) | Suggested-cluster well rendered when operator clicks "Suggest categories from titles + descriptions"; per-cluster grid with label, sample titles, Apply / Dismiss actions | +| `.tag-suggest-actions` | `shop_tags.j2` | Apply / Dismiss button row, `grid-auto-flow: column` | ## CSS Conventions diff --git a/docs/tickets/mps-24.md b/docs/tickets/mps-24.md index 63dab27..bdda917 100644 --- a/docs/tickets/mps-24.md +++ b/docs/tickets/mps-24.md @@ -2,11 +2,13 @@ ## Status -**PHASE 1 SHIPPED (2026-05-15).** Tag model + chip strip + sectioned-lanes -layout + bulk tagger live behind an opt-in `home_layout` selector (default -`0` = flat = unchanged). Phase 2 (auto-tag from titles) and Phase 3 -(ML-assisted via uncloseai) follow under this same ticket per -CLAUDE.md "One Feature, One Ticket". +**PHASES 1 + 2 SHIPPED (2026-05-15).** Tag model + chip strip + +sectioned-lanes layout + bulk tagger live behind an opt-in `home_layout` +selector (default `0` = flat = unchanged). Phase 2 adds a +title-plus-description auto-tagger surfaced as one-click cluster apply +in the bulk tagger UI + `scripts/backfill_tags.py` CLI. Phase 3 +(uncloseai-backed ML categorization behind a kill switch) follows under +this same ticket per CLAUDE.md "One Feature, One Ticket". **Background:** operator feedback on `shop.printableprompts.com` flagged our default home page as the reason for considering a move to Shopify. We @@ -160,28 +162,51 @@ from "scroll 481 items" to "click chip, scan ~50, click product." - Filter chips also added to `/search` results so shopper can refine by tag after a keyword query (`/search?keywords=X&tag=Y`). -### Phase 2 — Sectioned lanes + auto-tag from titles (A3 + B1) +### Phase 2 — Sectioned lanes shipped in Phase 1; auto-tag from title + description -Even fewer clicks: shopper sees *all* categories on land, no chip click needed. -Lanes are scoped to top-N tags by popularity. +Phase 1 already shipped sectioned-lane layout (`home_layout == 2`) — we +brought it forward because rendering the lanes was a one-line template +branch on top of the chip work. What remains for Phase 2 is the +**deterministic title + description auto-tagger** so an operator with 481 +untagged products gets a working categorization in one click. -- `home_layout == 2` renders one lane per top-N tags, each lane shows up to - ~10 products of that tag, with a "see all" link to the tag detail page. -- Lanes scroll horizontally on touch; stack as single-column below 800px - (matches our existing mobile reorder pattern). -- Deterministic title-keyword auto-tagger as a standalone script - (`scripts/backfill_tags.py`) and a "Suggest tags from titles" button on - the bulk tag editor. Rules: - - Tokenize all titles in shop. - - Drop stopwords + common shop-vocabulary words (per-shop configurable - list — printableprompts will drop `write`, `the`, `room`, `activity` - because they appear in nearly every title and group nothing). - - Stem (simple suffix strip — no new dep, no Porter ports). - - Cluster: products sharing >= 2 non-stopword stems form a candidate group. - - Label the candidate group by its most frequent shared stem. - - **Emit candidate tags to operator for approval — never auto-commit** - (A6 hybrid: suggest, don't impose). -- This phase has no ML, no external deps. It's grep-flavoured clustering. +Inputs: +- `Product.title` — full token weight × **3** (short, decisive, intentional). +- `Product.description` — raw markdown stripped of formatting, tokenised, + weight × **1**, capped at the first ~100 unique tokens per product so + long blog posts don't drown short product copy. + +Pipeline: +1. Tokenize title + description → lowercased words ≥ 3 chars. +2. Drop platform-default English stopwords + per-shop + `tag_stopwords_json` overrides. For printableprompts that adds + `write`, `room`, `activity`, `the`, etc. +3. Stem with a simple suffix-strip (no Porter port, no new dep) — + `seasonal`/`seasons`/`season` → `season`. +4. Build per-stem product sets across the catalog. +5. Drop stems whose slug already exists as a shop tag (we don't + re-suggest already-applied categories). +6. Keep stems carried by ≥ 2 products; rank by product count desc. +7. For each candidate stem, label = most frequent **original** word for + that stem (so `valentin` displays as `Valentine's`, not `valentin`). + +Surface: +- New "Suggest categories from titles + descriptions" button on + `/s/{shop_id}/tags`. Renders a "Suggested categories" well listing each + candidate cluster — label, sample product titles, product count. +- One-click apply per cluster — creates the tag + bulk-attaches every + product in the cluster, all under one form POST. +- One-click dismiss per cluster — adds the stem's label to + `tag_stopwords_json` so it never resurfaces. +- Standalone CLI `scripts/backfill_tags.py --shop= [--dry-run]` for + larger shops that prefer a terminal preview. + +**A6 hybrid — suggest, never auto-commit.** Cluster output is rendered to +the operator; nothing writes `Tag` / `ProductTag` rows until the operator +clicks Apply. + +No ML, no external deps. Pure Python over `Product.title` + +`Product.description`. O(N × tokens) over a shop's catalog. ### Phase 3 — ML-assisted categorization via uncloseai (A5) @@ -282,6 +307,18 @@ picker is Phase 2). | `CLAUDE.md` | New "Shop Home Layout + Tags" section + "Ticket Scoping — One Feature, One Ticket" rule | | `~/git/www.makepostsell.com/index.html` + `pricing.html` | "Categorized Home Page" feature card + pricing list entry | +### Phase 2 — shipped 2026-05-15 + +| File | Change | +|---|---| +| `lib/tag_suggest.py` (new) | Pure-function clusterer: `tokenize`, `simple_stem`, `stem_bag`, `suggest_clusters` over title (weight 3) + description (weight 1, capped at 100 unique tokens) | +| `scripts/backfill_tags.py` (new) | CLI: `--shop=` previews suggestions; `--apply` creates tags + attaches products | +| `views/shop.py` | `shop_tags` view gained `action=suggest`, `action=apply_suggestion`, `action=dismiss_suggestion`; `?show_suggestions=1` triggers cluster compute | +| `templates/shop_tags.j2` | "Suggest categories from titles + descriptions" button + suggestions well with one-click Apply / Dismiss per cluster | +| `static/css/common.css` | `.tag-suggest-list` / `.tag-suggest-item` / `.tag-suggest-actions` styles | +| `tests/test_models.py` | `TestTagSuggestPureFunctions` — 11 unit tests over tokenize / stem / cluster | +| `tests/test_functional.py` | `test_suggest_clusters_renders_candidates`, `test_apply_suggestion_creates_tag_and_attaches_products`, `test_dismiss_suggestion_adds_to_stopwords`, `test_apply_suggestion_rejects_empty_input` | + ## Tests (Phase 1) ### Unit (`test_models.py`) diff --git a/make_post_sell/lib/tag_suggest.py b/make_post_sell/lib/tag_suggest.py new file mode 100644 index 0000000..793008f --- /dev/null +++ b/make_post_sell/lib/tag_suggest.py @@ -0,0 +1,257 @@ +"""MPS-24 Phase 2: deterministic tag suggestions from product title + description. + +Pure functions; no DB, no ML, no external deps beyond the stdlib. Caller +hands us product (title, description, id) tuples + per-shop stopwords + +existing tag slugs; we return ranked candidate clusters for operator +approval. + +Pipeline: + tokenize(text) → lowercased word tokens ≥ 3 chars, markdown stripped + simple_stem(word) → suffix-strip; "seasonal"/"seasons" → "season" + suggest_clusters() → list of {stem, label, product_ids, score} + +Inputs are weighted: title × 3, description × 1 (capped per product). +""" +import re +from collections import Counter, defaultdict + + +# Word count cap per product for the description side of the input, to +# stop a long blog post from drowning the catalog signal. Title is never +# capped (titles are short by construction). +DESCRIPTION_TOKEN_CAP = 100 + +# Title weight relative to description; tokens from the title count this +# many times when scoring stem frequency per product. +TITLE_WEIGHT = 3 +DESCRIPTION_WEIGHT = 1 + +# Minimum number of products that must share a stem for it to surface +# as a candidate category. Singletons are noise. +DEFAULT_MIN_PRODUCTS = 2 + +# Default cap on how many candidate clusters we return per call. +DEFAULT_TOP_N = 20 + +# English stopwords — small, hand-tuned for product copy. Operators add +# shop-specific extras via shop.tag_stopwords_json (e.g. printableprompts +# wants "write", "room", "activity"). +ENGLISH_STOPWORDS = frozenset([ + "the", "and", "for", "you", "your", "with", "this", "that", "from", + "are", "was", "were", "but", "not", "have", "has", "had", "all", + "any", "can", "will", "would", "should", "could", "into", "out", + "over", "under", "about", "what", "when", "where", "why", "how", + "who", "they", "them", "their", "our", "ours", "his", "her", "its", + "one", "two", "three", "more", "most", "some", "few", "many", "much", + "very", "just", "only", "also", "than", "then", "now", "still", + "such", "even", "ever", "never", "always", "every", "each", "both", + "use", "uses", "used", "using", "make", "makes", "made", "making", + "get", "gets", "got", "getting", "set", "sets", "setting", + "include", "includes", "included", "including", + "buy", "buys", "bought", "buying", "sell", "sells", "sold", "selling", + "perfect", "great", "best", "new", "free", "easy", + "product", "products", "item", "items", + "shop", "shops", "store", "stores", + "page", "pages", "content", "file", "files", "download", "downloads", +]) + +# Suffixes to strip in order; longer ones first so "ies" beats "es". +_STEM_SUFFIXES = ( + "iness", "fulness", "tion", "ment", "ness", "able", "ible", + "ies", "ied", + "ing", "ers", "est", "ish", "ous", + "ly", "ed", "es", "er", "or", + "al", "ic", + "s", +) + +# Strip these markdown / formatting characters before tokenising. +_MD_PUNCT = re.compile(r"[`*_~#>|\\\[\]()<>{}/\"'!?,.:;=+\-]") + +# Drop fenced code blocks / inline code spans so code snippets don't +# leak random tokens. Order matters: fenced first, then inline. +_CODE_FENCE = re.compile(r"```.*?```", re.DOTALL) +_CODE_INLINE = re.compile(r"`[^`]*`") + +# Strip URLs (bare or markdown-linked) before tokenising. +_URL = re.compile(r"https?://\S+") + +# Strip raw HTML tags (description is markdown but operators paste in HTML). +_HTML_TAG = re.compile(r"<[^>]+>") + +# What counts as a word — lowercase letters + digits, ≥ 3 chars. +_WORD = re.compile(r"[a-z][a-z0-9]{2,}") + + +def simple_stem(word): + """Suffix-strip stem for English-ish product copy. + + Conservative: never strips a suffix if the resulting stem is < 3 + chars (avoids collapsing "ice" → "i" because of -ce). + + >>> simple_stem("seasonal") + 'season' + >>> simple_stem("seasons") + 'season' + >>> simple_stem("running") + 'runn' + >>> simple_stem("math") + 'math' + """ + w = (word or "").lower() + if len(w) <= 3: + return w + for suffix in _STEM_SUFFIXES: + if w.endswith(suffix) and len(w) - len(suffix) >= 3: + return w[: -len(suffix)] + return w + + +def _clean_text(text): + """Strip markdown formatting + URLs + HTML before tokenising.""" + if not text: + return "" + text = _CODE_FENCE.sub(" ", text) + text = _CODE_INLINE.sub(" ", text) + text = _URL.sub(" ", text) + text = _HTML_TAG.sub(" ", text) + text = _MD_PUNCT.sub(" ", text) + return text.lower() + + +def tokenize(text, stopwords=None, cap=None): + """Tokenize a string into a list of lowercase words ≥ 3 chars. + + Drops English stopwords and the operator's per-shop additions. When + `cap` is provided, returns the first `cap` *unique* tokens (in order + of first appearance) — used to stop very long descriptions from + dominating the cluster signal. + """ + if not text: + return [] + stop = set(ENGLISH_STOPWORDS) + if stopwords: + stop |= {w.lower() for w in stopwords} + + cleaned = _clean_text(text) + tokens = [] + seen = set() if cap else None + for raw in _WORD.findall(cleaned): + if raw in stop: + continue + if cap: + if raw in seen: + continue + seen.add(raw) + tokens.append(raw) + if cap and len(tokens) >= cap: + break + return tokens + + +def stem_bag(title, description, stopwords=None): + """Return a Counter of stems → weighted count for one product. + + Title tokens contribute `TITLE_WEIGHT`, description tokens + contribute `DESCRIPTION_WEIGHT` (capped). The same word appearing in + both title and description simply adds both contributions. + + Also returns a {stem: representative_word} dict so callers can label + a cluster with the most-readable original word for that stem. + """ + bag = Counter() + label_votes = defaultdict(Counter) + + for raw in tokenize(title, stopwords=stopwords): + s = simple_stem(raw) + bag[s] += TITLE_WEIGHT + # Titles outweigh descriptions for labelling, too. + label_votes[s][raw] += TITLE_WEIGHT + + for raw in tokenize( + description, stopwords=stopwords, cap=DESCRIPTION_TOKEN_CAP + ): + s = simple_stem(raw) + bag[s] += DESCRIPTION_WEIGHT + label_votes[s][raw] += DESCRIPTION_WEIGHT + + labels = {s: votes.most_common(1)[0][0] for s, votes in label_votes.items()} + return bag, labels + + +def suggest_clusters( + products, + stopwords=None, + existing_tag_slugs=None, + min_products=DEFAULT_MIN_PRODUCTS, + top_n=DEFAULT_TOP_N, +): + """Group products by shared stems and rank candidate categories. + + Args: + products: iterable of objects with `.id`, `.title`, `.description`. + stopwords: list of lowercase strings to exclude in addition to the + English defaults. + existing_tag_slugs: iterable of slugs already present as Tag rows + for the shop; skipped so we don't re-suggest + already-applied categories. + min_products: minimum products sharing a stem to qualify. + top_n: maximum candidate clusters returned. + + Returns: + List of dicts (ranked, longest first): + { + "stem": "season", + "label": "Seasonal", # most frequent original word + "score": 117, # sum of weighted stem hits + "product_ids": [, , ...], + "sample_titles": ["Spring Math", "Autumn Read-Alouds", ...], + } + """ + from slugify import slugify # local import; only needed here + + skip = {s.lower() for s in (existing_tag_slugs or [])} + # Track per-stem: total weighted score, product ids carrying it, and + # which original words voted for the label. + stem_score = Counter() + stem_products = defaultdict(list) # stem -> [product_id, ...] + stem_labels = defaultdict(Counter) # stem -> Counter(original_word) + stem_titles = defaultdict(list) # stem -> [title, ...] for samples + + for product in products: + title = product.title or "" + description = product.description or "" + bag, labels = stem_bag(title, description, stopwords=stopwords) + for stem, weight in bag.items(): + stem_score[stem] += weight + stem_products[stem].append(product.id) + if labels.get(stem): + stem_labels[stem][labels[stem]] += weight + stem_titles[stem].append(title) + + candidates = [] + for stem, ids in stem_products.items(): + if len(ids) < min_products: + continue + # Best-vote original word becomes the candidate label. + label_word = ( + stem_labels[stem].most_common(1)[0][0] if stem_labels[stem] else stem + ) + label = label_word.capitalize() + candidate_slug = slugify(label)[:80] + if not candidate_slug or candidate_slug in skip: + continue + candidates.append({ + "stem": stem, + "label": label, + "slug": candidate_slug, + "score": stem_score[stem], + "product_ids": ids, + "sample_titles": stem_titles[stem][:3], + }) + + candidates.sort( + key=lambda c: (len(c["product_ids"]), c["score"]), + reverse=True, + ) + return candidates[:top_n] diff --git a/make_post_sell/scripts/backfill_tags.py b/make_post_sell/scripts/backfill_tags.py new file mode 100644 index 0000000..fe20a06 --- /dev/null +++ b/make_post_sell/scripts/backfill_tags.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""MPS-24 Phase 2: preview tag suggestions for one shop. + +Usage: + python -m make_post_sell.scripts.backfill_tags data/development.ini --shop= + python -m make_post_sell.scripts.backfill_tags data/development.ini --shop= --apply + +Deterministic clustering of `Product.title` + `Product.description` — +prints candidate categories so an operator can preview before committing +in the bulk tagger UI. `--apply` creates the tags + attaches every product +in each cluster (use only when the preview is acceptable). +""" + +import argparse +import sys + +from pyramid.paster import bootstrap + +from make_post_sell.lib.tag_suggest import suggest_clusters +from make_post_sell.models.product import ( + get_all_products_from_a_shop, + get_product_by_id, +) +from make_post_sell.models.shop import get_shop_by_id +from make_post_sell.models.tag import get_or_create_tag, tags_by_popularity + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("config_uri", help="Pyramid config (e.g. data/development.ini)") + parser.add_argument("--shop", required=True, help="Shop UUID (with or without dashes)") + parser.add_argument( + "--min-products", type=int, default=2, + help="Minimum products sharing a stem to surface (default: 2)", + ) + parser.add_argument( + "--top-n", type=int, default=20, + help="Max candidate clusters to print (default: 20)", + ) + parser.add_argument( + "--apply", action="store_true", + help="Create tags + attach products (default: dry-run preview only)", + ) + args = parser.parse_args(argv) + + env = bootstrap(args.config_uri) + request = env["request"] + + with request.tm: + dbsession = request.dbsession + shop = get_shop_by_id(dbsession, args.shop) + if shop is None: + print(f"Shop {args.shop!r} not found.", file=sys.stderr) + return 1 + + products = list(get_all_products_from_a_shop(shop)) + existing_slugs = [t.slug for t in tags_by_popularity(dbsession, shop)] + + print(f"Shop: {shop.name} ({shop.id})") + print(f"Products scanned: {len(products)}") + print(f"Existing tags: {len(existing_slugs)}") + print(f"Stopword overrides: {shop.tag_stopwords or '(none)'}") + print(f"Mode: {'APPLY' if args.apply else 'DRY-RUN (preview)'}") + print("-" * 60) + + suggestions = suggest_clusters( + products, + stopwords=shop.tag_stopwords, + existing_tag_slugs=existing_slugs, + min_products=args.min_products, + top_n=args.top_n, + ) + + if not suggestions: + print("No candidate categories surfaced.") + return 0 + + for s in suggestions: + print( + f"{s['label']:24s} " + f"products={len(s['product_ids']):>4} " + f"score={s['score']:>4} " + f"e.g. {s['sample_titles'][0]}" + ) + + if not args.apply: + print() + print("Re-run with --apply to create the tags + attach products.") + return 0 + + print() + print("Applying...") + for s in suggestions: + tag = get_or_create_tag(dbsession, shop, s["label"]) + if tag is None: + print(f" skip {s['label']!r}: could not create tag") + continue + applied = 0 + for product_id in s["product_ids"]: + product = get_product_by_id(dbsession, product_id) + if product is None or product.shop_id != shop.id: + continue + if tag not in product.tags: + product.tags.append(tag) + applied += 1 + print(f" {tag.name!r} applied to {applied} products") + + print("Done.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/make_post_sell/static/css/common.css b/make_post_sell/static/css/common.css index e1e6474..c30c16c 100644 --- a/make_post_sell/static/css/common.css +++ b/make_post_sell/static/css/common.css @@ -1449,6 +1449,57 @@ form.tag-list-delete { margin: 0; } +/* MPS-24 Phase 2: tag-suggestion well */ +h3.tag-suggest-heading { + margin: var(--space-4, 16px) 0 var(--space-2, 8px) 0; +} + +ul.tag-suggest-list { + list-style: none; + padding: 0; + margin: 0; + display: grid; + gap: var(--space-2, 8px); +} + +li.tag-suggest-item { + display: grid; + grid-template-columns: 1fr; + gap: var(--space-1, 4px); + padding: var(--space-3, 12px); + border: 1px solid var(--color-border, #e5e7eb); + border-radius: var(--radius-sm, 4px); + background: var(--color-surface-2, #f9fafb); +} + +div.tag-suggest-label { + display: grid; + grid-template-columns: auto 1fr; + align-items: baseline; + gap: var(--space-2, 8px); +} + +div.tag-suggest-samples { + color: var(--color-text-muted, #6b7280); + font-size: var(--type-body-sm-size, 0.875rem); +} + +span.tag-suggest-sample { + /* inline tokens — no display: change, lets samples wrap naturally */ +} + +div.tag-suggest-actions { + display: grid; + grid-auto-flow: column; + grid-auto-columns: max-content; + gap: var(--space-2, 8px); + margin-top: var(--space-2, 8px); +} + +form.tag-suggest-form { + margin: 0; +} + /* Tag detail page header */ section.tag-detail-header { margin: var(--space-3, 12px) 0; diff --git a/make_post_sell/templates/shop_tags.j2 b/make_post_sell/templates/shop_tags.j2 index 0d5c962..a1af355 100644 --- a/make_post_sell/templates/shop_tags.j2 +++ b/make_post_sell/templates/shop_tags.j2 @@ -22,6 +22,58 @@ +{# MPS-24 Phase 2: suggest categories from product title + description. #} +
+

Suggest categories

+

+ Scan your product titles + descriptions and surface clusters of related + products. Suggestions are previews — nothing is applied until you click + Apply on a row. +

+ + Suggest categories from titles + descriptions + + + {% if show_suggestions %} + {% if suggestions %} +

Candidate categories ({{ suggestions|length }})

+
    + {% for s in suggestions %} +
  • +
    + {{ s.label }} + {{ s.product_ids|length }} product{% if s.product_ids|length != 1 %}s{% endif %} +
    +
    + {% for t in s.sample_titles %} + {{ t }}{% if not loop.last %} · {% endif %} + {% endfor %} +
    +
    +
    + + + + +
    +
    + + + +
    +
    +
    • + {% endfor %} +
+ {% else %} +

No clusters surfaced. Add more products or relax your stopword list in + shop settings.

+ {% endif %} + {% endif %} +
+

All tags ({{ tags|length }})

{% if not tags %} diff --git a/make_post_sell/tests/test_functional.py b/make_post_sell/tests/test_functional.py index cdeee12..97f4fb0 100644 --- a/make_post_sell/tests/test_functional.py +++ b/make_post_sell/tests/test_functional.py @@ -8261,3 +8261,110 @@ class TestHomeLayoutAndTags(_AuthenticatedBase): f"/s/{shop.id}/tag/does-not-exist", expect_errors=True ) self.assertEqual(res.status_int, 404) + + # --- MPS-24 Phase 2: suggest from title + description ----------------- + + def _make_shop_with_products(self, shop_name, products_meta): + """Create a shop and N products with (title, description) tuples. + + Returns (shop, [product, ...]). + """ + shop = self._create_shop_helper( + user_creds=self.user1_creds, + shop_params={**self.shop1_params, "name": shop_name}, + ) + for title, description in products_meta: + params = { + **self.product1_params, + "title": title, + "description": description, + } + self.testapp.post(f"/p/new?shop_id={shop.id}", params) + from ..models.product import get_all_products_from_a_shop + products = list(get_all_products_from_a_shop(shop)) + return shop, products + + def test_suggest_clusters_renders_candidates(self): + shop, _products = self._make_shop_with_products( + "suggest-shop", + [ + ("Addition to 10", "Math activity for first grade."), + ("Counting to 100", "Math activity for kindergarten."), + ("Stone Fox Novel Study", "Reading novel chapter questions."), + ("Chocolate Touch Novel Study", "Reading novel comprehension."), + ], + ) + # Without ?show_suggestions=1, page renders but no suggestions well. + res = self.testapp.get(f"/s/{shop.id}/tags") + self.assertNotIn("Candidate categories", res.body.decode()) + + # With ?show_suggestions=1, candidates render. + res = self.testapp.get(f"/s/{shop.id}/tags?show_suggestions=1") + body = res.body.decode() + self.assertIn("Candidate categories", body) + # Math + Novel are the obvious clusters. + self.assertIn("Math", body) + self.assertIn("Novel", body) + + def test_apply_suggestion_creates_tag_and_attaches_products(self): + shop, products = self._make_shop_with_products( + "apply-shop", + [ + ("Addition to 10", "Math activity."), + ("Counting to 100", "Math practice."), + ], + ) + product_ids = ",".join(str(p.id) for p in products) + res = self.testapp.post( + f"/s/{shop.id}/tags", + { + "action": "apply_suggestion", + "label": "Math", + "product_ids": product_ids, + }, + ) + if res.status_int == 302: + res = res.follow() + self.assertIn("Applied tag 'Math' to 2 products", res.body.decode()) + + from ..models.tag import get_tag_by_shop_and_slug + self.dbsession.expire_all() + tag = get_tag_by_shop_and_slug(self.dbsession, shop, "math") + self.assertIsNotNone(tag) + # Both products carry the tag now. + for product in products: + self.dbsession.refresh(product) + self.assertIn(tag, list(product.tags)) + + def test_dismiss_suggestion_adds_to_stopwords(self): + shop, _products = self._make_shop_with_products( + "dismiss-shop", + [ + ("Foo Bar", "Foo bar content."), + ("Foo Bar Two", "Foo bar more."), + ], + ) + res = self.testapp.post( + f"/s/{shop.id}/tags", + {"action": "dismiss_suggestion", "label": "Foo Bar"}, + ) + if res.status_int == 302: + res = res.follow() + self.assertIn("Dismissed 'Foo Bar'", res.body.decode()) + + self.dbsession.expire(shop) + # Both "foo" and "bar" are now in the stopwords list. + self.assertIn("foo", shop.tag_stopwords) + self.assertIn("bar", shop.tag_stopwords) + + def test_apply_suggestion_rejects_empty_input(self): + shop, _products = self._make_shop_with_products( + "reject-shop", [("First", "Body.")] + ) + res = self.testapp.post( + f"/s/{shop.id}/tags", + {"action": "apply_suggestion", "label": "", "product_ids": ""}, + ) + if res.status_int == 302: + res = res.follow() + self.assertIn("Could not apply suggestion", res.body.decode()) diff --git a/make_post_sell/tests/test_models.py b/make_post_sell/tests/test_models.py index 7929d76..2b89f69 100644 --- a/make_post_sell/tests/test_models.py +++ b/make_post_sell/tests/test_models.py @@ -5064,3 +5064,156 @@ class TestTagModel(unittest.TestCase): self.assertEqual(tag.name, "Math") self.assertEqual(tag.slug, "math") + +class TestTagSuggestPureFunctions(unittest.TestCase): + """MPS-24 Phase 2: tokenizer, stemmer, clusterer — no DB.""" + + def test_simple_stem_strips_common_suffixes(self): + from ..lib.tag_suggest import simple_stem + self.assertEqual(simple_stem("seasonal"), "season") + self.assertEqual(simple_stem("seasons"), "season") + self.assertEqual(simple_stem("activities"), "activit") + self.assertEqual(simple_stem("running"), "runn") + # Short stems must not be over-trimmed + self.assertEqual(simple_stem("ice"), "ice") + self.assertEqual(simple_stem("us"), "us") + + def test_simple_stem_handles_empty(self): + from ..lib.tag_suggest import simple_stem + self.assertEqual(simple_stem(""), "") + self.assertEqual(simple_stem(None), "") + + def test_tokenize_lowercases_and_filters(self): + from ..lib.tag_suggest import tokenize + out = tokenize("The Math Activities for FIRST grade") + # "the", "for" drop as stopwords; "first" is not stopword + self.assertIn("math", out) + self.assertIn("activities", out) + self.assertIn("first", out) + self.assertIn("grade", out) + self.assertNotIn("the", out) + self.assertNotIn("for", out) + + def test_tokenize_strips_markdown(self): + from ..lib.tag_suggest import tokenize + out = tokenize( + "**Math** _activity_ for [grade](https://x.example/page) `secret` 1" + ) + self.assertIn("math", out) + self.assertIn("activity", out) + self.assertIn("grade", out) + # URL contents stripped (real http(s) URL is removed before tokenising) + self.assertNotIn("example", out) + # Inline code block contents stripped + self.assertNotIn("secret", out) + + def test_tokenize_honours_custom_stopwords(self): + from ..lib.tag_suggest import tokenize + out = tokenize( + "Math Activities Write the Room", + stopwords=["write", "room", "activities"], + ) + self.assertIn("math", out) + self.assertNotIn("write", out) + self.assertNotIn("room", out) + self.assertNotIn("activities", out) + + def test_tokenize_caps_at_unique_token_limit(self): + from ..lib.tag_suggest import tokenize + # 200 unique tokens; cap=10 should return exactly 10 + words = " ".join(f"word{i:03d}" for i in range(200)) + out = tokenize(words, cap=10) + self.assertEqual(len(out), 10) + self.assertEqual(len(set(out)), 10) + + def test_suggest_clusters_finds_natural_groups(self): + """Printableprompts-style sample: math, seasonal, novel-study clusters.""" + from types import SimpleNamespace + from ..lib.tag_suggest import suggest_clusters + products = [ + SimpleNamespace(id="a", title="Addition to 10 Write the Room", description="Math activity."), + SimpleNamespace(id="b", title="Counting to 100 Write the Room", description="Math practice."), + SimpleNamespace(id="c", title="Valentine's Day Color by Number", description="Seasonal math."), + SimpleNamespace(id="d", title="St. Patrick's Day Activities", description="Seasonal crafts."), + SimpleNamespace(id="e", title="Stone Fox Novel Study", description="Reading novel."), + SimpleNamespace(id="f", title="Chocolate Touch Novel Study", description="Reading novel."), + ] + clusters = suggest_clusters( + products, + stopwords=["write", "room", "activity", "day"], + existing_tag_slugs=[], + min_products=2, + ) + labels = [c["label"] for c in clusters] + self.assertIn("Math", labels) + self.assertIn("Seasonal", labels) + self.assertIn("Novel", labels) + # All clusters should respect min_products + for c in clusters: + self.assertGreaterEqual(len(c["product_ids"]), 2) + + def test_suggest_clusters_skips_existing_tags(self): + from types import SimpleNamespace + from ..lib.tag_suggest import suggest_clusters + products = [ + SimpleNamespace(id="a", title="Math Worksheet One", description=""), + SimpleNamespace(id="b", title="Math Worksheet Two", description=""), + ] + clusters = suggest_clusters( + products, stopwords=[], existing_tag_slugs=["math"] + ) + labels = [c["label"] for c in clusters] + self.assertNotIn("Math", labels) + + def test_suggest_clusters_label_uses_best_original_word(self): + """'Valentine's' should beat 'valentin' (the stem) as a label.""" + from types import SimpleNamespace + from ..lib.tag_suggest import suggest_clusters + products = [ + SimpleNamespace(id="a", title="Valentine's Day Math", description=""), + SimpleNamespace(id="b", title="Valentine's Day Reading", description=""), + SimpleNamespace(id="c", title="Valentine's Day Crafts", description=""), + ] + clusters = suggest_clusters( + products, stopwords=["day"], existing_tag_slugs=[] + ) + # The label should be the most common original word, not the stem. + labels = [c["label"] for c in clusters] + # "Valentine" comes back as the readable label — the apostrophe is + # stripped at the markdown-cleaning step. + self.assertTrue( + any("alentin" in l.lower() for l in labels), + f"expected valentine-ish label in {labels}", + ) + + def test_suggest_clusters_ranks_by_product_count(self): + from types import SimpleNamespace + from ..lib.tag_suggest import suggest_clusters + products = [ + SimpleNamespace(id="a", title="Alpha Math", description=""), + SimpleNamespace(id="b", title="Beta Math", description=""), + SimpleNamespace(id="c", title="Gamma Math", description=""), + SimpleNamespace(id="d", title="Delta Reading", description=""), + SimpleNamespace(id="e", title="Epsilon Reading", description=""), + ] + clusters = suggest_clusters( + products, stopwords=[], existing_tag_slugs=[], min_products=2 + ) + # "Math" (3 products) ranks above "Reading" (2 products). + self.assertEqual(clusters[0]["label"], "Math") + self.assertEqual(len(clusters[0]["product_ids"]), 3) + + def test_suggest_clusters_drops_singletons(self): + from types import SimpleNamespace + from ..lib.tag_suggest import suggest_clusters + products = [ + SimpleNamespace(id="a", title="Unique Title One", description=""), + SimpleNamespace(id="b", title="Different Title Two", description=""), + ] + clusters = suggest_clusters( + products, stopwords=["title"], existing_tag_slugs=[], min_products=2 + ) + # No stem appears in both products → no clusters. + self.assertEqual(clusters, []) + + diff --git a/make_post_sell/views/shop.py b/make_post_sell/views/shop.py index de7bc96..1d4057a 100644 --- a/make_post_sell/views/shop.py +++ b/make_post_sell/views/shop.py @@ -2040,6 +2040,61 @@ def shop_tags(request): f"/s/{shop.id}/tags?focus={tag_slug}" ) + # MPS-24 Phase 2: dismiss a suggested cluster — add its slug to the + # shop's tag_stopwords_json so it never surfaces again. + if action == "dismiss_suggestion": + import json as _json + label = (request.params.get("label") or "").strip() + if label: + existing = list(shop.tag_stopwords) + tokens = [t.strip().lower() for t in label.split() if t.strip()] + for tok in tokens: + if tok and tok not in existing: + existing.append(tok) + shop.tag_stopwords_json = _json.dumps(existing) + request.session.flash( + ( + f"Dismissed '{label}'. We'll skip these words next time.", + "success", + ) + ) + return HTTPFound(f"/s/{shop.id}/tags?show_suggestions=1") + + # MPS-24 Phase 2: one-click apply a suggested cluster — create the + # Tag row + attach every product in the cluster. + if action == "apply_suggestion": + from ..models.product import get_product_by_id + + label = (request.params.get("label") or "").strip() + product_ids_raw = request.params.get("product_ids") or "" + product_ids = [p.strip() for p in product_ids_raw.split(",") if p.strip()] + if not label or not product_ids: + request.session.flash( + ("Could not apply suggestion (empty label or no products).", "error") + ) + return HTTPFound(f"/s/{shop.id}/tags?show_suggestions=1") + tag = get_or_create_tag(request.dbsession, shop, label) + if tag is None: + request.session.flash( + ("Could not create that tag (invalid name).", "error") + ) + return HTTPFound(f"/s/{shop.id}/tags?show_suggestions=1") + applied = 0 + for product_id in product_ids: + product = get_product_by_id(request.dbsession, product_id) + if product is None or product.shop_id != shop.id: + continue + if tag not in product.tags: + product.tags.append(tag) + applied += 1 + request.session.flash( + ( + f"Applied tag '{tag.name}' to {applied} product{'s' if applied != 1 else ''}.", + "success", + ) + ) + return HTTPFound(f"/s/{shop.id}/tags?show_suggestions=1") + # GET: render bulk tagger. all_tags = tags_by_popularity(request.dbsession, shop) focus_slug = (request.params.get("focus") or "").strip().lower() @@ -2049,9 +2104,28 @@ def shop_tags(request): else None ) all_products = get_all_products_from_a_shop(shop) + + # MPS-24 Phase 2: compute candidate clusters on demand. We always + # compute (cheap O(N × tokens) over the shop catalog), but the + # template only renders the well when the operator clicks the + # button (?show_suggestions=1). + show_suggestions = (request.params.get("show_suggestions") or "") == "1" + suggestions = [] + if show_suggestions: + from ..lib.tag_suggest import suggest_clusters + + existing_slugs = [t.slug for t in all_tags] + suggestions = suggest_clusters( + list(all_products), + stopwords=shop.tag_stopwords, + existing_tag_slugs=existing_slugs, + ) + return { "tags": all_tags, "focus_tag": focus_tag, "all_products": all_products, + "suggestions": suggestions, + "show_suggestions": show_suggestions, }