diff --git a/docs/wordpress-import-html.dot b/docs/wordpress-import-html.dot new file mode 100644 index 0000000..2d9931d --- /dev/null +++ b/docs/wordpress-import-html.dot @@ -0,0 +1,81 @@ +// WordPress HTML Conversion Pipeline — Detail +// Render: dot -Tsvg docs/wordpress-import-html.dot -o docs/wordpress-import-html.dot.svg +digraph html_conversion { + rankdir=LR; + node [shape=box, style="rounded,filled", fontname="monospace", fontsize=9]; + edge [fontname="monospace", fontsize=8]; + + raw_html [label="Raw WordPress HTML\n\n\n

\n[gallery ids=\"1,2,3\"]\n", fillcolor="#fce4ec", shape=note]; + + subgraph cluster_phase1 { + label="Phase 1: Pre-process (BeautifulSoup)"; + style=dashed; + color="#5871ad"; + + strip_gutenberg [label="Strip Gutenberg\ncomments\n", fillcolor="#e8eaf6"]; + strip_shortcodes [label="Strip shortcodes\n[vc_*] [et_pb_*]\n[fusion_*]\nkeep inner content", fillcolor="#e8eaf6"]; + convert_embeds [label="Convert embeds\n[embed]URL[/embed]\n→ bare URL", fillcolor="#e8eaf6"]; + convert_captions [label="Convert captions\n[caption] →

", fillcolor="#e8eaf6"]; + strip_srcset [label="Strip srcset/sizes\nStrip WP classes\nUnwrap empty divs", fillcolor="#e8eaf6"]; + } + + subgraph cluster_phase2 { + label="Phase 2: Image Migration"; + style=dashed; + color="#58ad71"; + + find_imgs [label="Find all src\nand background-image\nURLs", fillcolor="#e8f5e9"]; + strip_suffix [label="Strip WP size suffix\n-300x200 → original\nphoto-1024x768.jpg\n→ photo.jpg", fillcolor="#e8f5e9"]; + download [label="Download original\nfrom WP server\nrequests.get()\ntimeout=30s", fillcolor="#e8f5e9"]; + upload_s3 [label="Upload to S3\n{shop_id}/{product_id}/\ninline/{filename}\nACL=public-read", fillcolor="#e8f5e9"]; + rewrite [label="Rewrite src URLs\nold WP URL\n→ CDN URL", fillcolor="#e8f5e9"]; + } + + subgraph cluster_phase3 { + label="Phase 3: markdownify"; + style=dashed; + color="#ad8f58"; + + converter [label="WPConverter\n(MarkdownConverter\nsubclass)", fillcolor="#fff8e1"]; + figures [label="
→ ![alt](url)\n
→ caption", fillcolor="#fff8e1"]; + code [label="
\n→ fenced code block", fillcolor="#fff8e1"];
+        headings [label="ATX headings (#)\nbody_width=0\nunicode_snob=True", fillcolor="#fff8e1"];
+    }
+
+    subgraph cluster_phase4 {
+        label="Phase 4: Post-process";
+        style=dashed;
+        color="#666666";
+
+        collapse [label="Collapse blank lines\n3+ newlines → 2", fillcolor="#f5f5f5"];
+        metadata [label="Prepend metadata\n**Categories:** ...\n**Tags:** ...\n**Author:** ...", fillcolor="#f5f5f5"];
+        flag [label="Flag posts with\nunconverted \nfor manual review", fillcolor="#f5f5f5"];
+    }
+
+    output [label="MPS Content\n\nProduct.description\n(raw markdown)\n\nProduct.description_html\n(rendered HTML)", fillcolor="#e8f5e9", shape=note];
+
+    // Flow
+    raw_html -> strip_gutenberg;
+    strip_gutenberg -> strip_shortcodes;
+    strip_shortcodes -> convert_embeds;
+    convert_embeds -> convert_captions;
+    convert_captions -> strip_srcset;
+
+    strip_srcset -> find_imgs;
+    find_imgs -> strip_suffix;
+    strip_suffix -> download;
+    download -> upload_s3;
+    upload_s3 -> rewrite;
+
+    rewrite -> converter;
+    converter -> figures;
+    converter -> code;
+    converter -> headings;
+    figures -> collapse [style=invis];
+    code -> collapse [style=invis];
+    headings -> collapse;
+
+    collapse -> metadata;
+    metadata -> flag;
+    flag -> output;
+}
diff --git a/docs/wordpress-import-html.dot.svg b/docs/wordpress-import-html.dot.svg
new file mode 100644
index 0000000..6a46588
--- /dev/null
+++ b/docs/wordpress-import-html.dot.svg
@@ -0,0 +1,305 @@
+
+
+
+
+
+
+html_conversion
+
+
+cluster_phase1
+
+Phase 1: Pre-process (BeautifulSoup)
+
+
+cluster_phase2
+
+Phase 2: Image Migration
+
+
+cluster_phase3
+
+Phase 3: markdownify
+
+
+cluster_phase4
+
+Phase 4: Post-process
+
+
+
+raw_html
+
+
+
+Raw WordPress HTML
+<!-- wp:paragraph -->
+<p class="has-large-font-size">
+[gallery ids="1,2,3"]
+<img srcset="..." />
+
+
+
+strip_gutenberg
+
+Strip Gutenberg
+comments
+<!-- /?wp:\S+.*?-->
+
+
+
+raw_html->strip_gutenberg
+
+
+
+
+
+strip_shortcodes
+
+Strip shortcodes
+[vc_*] [et_pb_*]
+[fusion_*]
+keep inner content
+
+
+
+strip_gutenberg->strip_shortcodes
+
+
+
+
+
+convert_embeds
+
+Convert embeds
+[embed]URL[/embed]
+→ bare URL
+
+
+
+strip_shortcodes->convert_embeds
+
+
+
+
+
+convert_captions
+
+Convert captions
+[caption] → <figure>
+
+
+
+convert_embeds->convert_captions
+
+
+
+
+
+strip_srcset
+
+Strip srcset/sizes
+Strip WP classes
+Unwrap empty divs
+
+
+
+convert_captions->strip_srcset
+
+
+
+
+
+find_imgs
+
+Find all <img> src
+and background-image
+URLs
+
+
+
+strip_srcset->find_imgs
+
+
+
+
+
+strip_suffix
+
+Strip WP size suffix
+-300x200 → original
+photo-1024x768.jpg
+→ photo.jpg
+
+
+
+find_imgs->strip_suffix
+
+
+
+
+
+download
+
+Download original
+from WP server
+requests.get()
+timeout=30s
+
+
+
+strip_suffix->download
+
+
+
+
+
+upload_s3
+
+Upload to S3
+{shop_id}/{product_id}/
+inline/{filename}
+ACL=public-read
+
+
+
+download->upload_s3
+
+
+
+
+
+rewrite
+
+Rewrite src URLs
+old WP URL
+→ CDN URL
+
+
+
+upload_s3->rewrite
+
+
+
+
+
+converter
+
+WPConverter
+(MarkdownConverter
+subclass)
+
+
+
+rewrite->converter
+
+
+
+
+
+figures
+
+<figure> → ![alt](url)
+<figcaption> → caption
+
+
+
+converter->figures
+
+
+
+
+
+code
+
+<pre><code
+class="language-*">
+→ fenced code block
+
+
+
+converter->code
+
+
+
+
+
+headings
+
+ATX headings (#)
+body_width=0
+unicode_snob=True
+
+
+
+converter->headings
+
+
+
+
+
+collapse
+
+Collapse blank lines
+3+ newlines → 2
+
+
+
+
+
+headings->collapse
+
+
+
+
+
+metadata
+
+Prepend metadata
+**Categories:** ...
+**Tags:** ...
+**Author:** ...
+
+
+
+collapse->metadata
+
+
+
+
+
+flag
+
+Flag posts with
+unconverted <html>
+for manual review
+
+
+
+metadata->flag
+
+
+
+
+
+output
+
+
+
+MPS Content
+Product.description
+(raw markdown)
+Product.description_html
+(rendered HTML)
+
+
+
+flag->output
+
+
+
+
+
diff --git a/docs/wordpress-import-pipeline.dot b/docs/wordpress-import-pipeline.dot
new file mode 100644
index 0000000..a576fb7
--- /dev/null
+++ b/docs/wordpress-import-pipeline.dot
@@ -0,0 +1,82 @@
+// WordPress Import Pipeline — Overview
+// Render: dot -Tsvg docs/wordpress-import-pipeline.dot -o docs/wordpress-import-pipeline.dot.svg
+digraph wordpress_import {
+    rankdir=TB;
+    node [shape=box, style="rounded,filled", fontname="monospace", fontsize=10];
+    edge [fontname="monospace", fontsize=9];
+
+    subgraph cluster_source {
+        label="WordPress Source";
+        style=dashed;
+        color="#ad5871";
+
+        rest_api [label="REST API\n/wp-json/wp/v2/*\n(live site, no auth)", fillcolor="#fce4ec"];
+        wxr_file [label="WXR XML Export\n(offline file)\nTools > Export", fillcolor="#fce4ec"];
+    }
+
+    subgraph cluster_parser {
+        label="Source Parser";
+        style=dashed;
+        color="#5871ad";
+
+        api_parser [label="REST API Client\nrequests + pagination\nper_page=100, ?_embed", fillcolor="#e8eaf6"];
+        wxr_parser [label="WXR Parser\nxml.etree.ElementTree\nnamespace-aware", fillcolor="#e8eaf6"];
+        normalize [label="Normalize\nUnified post dict\n(title, html, date,\nmedia_urls, comments)", fillcolor="#e8eaf6"];
+    }
+
+    subgraph cluster_convert {
+        label="HTML Conversion Pipeline";
+        style=dashed;
+        color="#58ad71";
+
+        preprocess [label="Phase 1: Pre-process\nBeautifulSoup\n- strip Gutenberg comments\n- strip shortcodes\n- strip srcset/sizes\n- strip WP classes", fillcolor="#e8f5e9"];
+        images [label="Phase 2: Image Migration\n- strip -WxH suffixes\n- download originals\n- upload to S3\n- rewrite URLs to CDN", fillcolor="#e8f5e9"];
+        markdown [label="Phase 3: Markdown\nmarkdownify (MIT)\n- custom WPConverter\n- figures, code blocks\n- ATX headings", fillcolor="#e8f5e9"];
+        postprocess [label="Phase 4: Post-process\n- collapse blank lines\n- prepend metadata\n- flag unconverted HTML", fillcolor="#e8f5e9"];
+    }
+
+    subgraph cluster_mps {
+        label="MPS (Target)";
+        style=dashed;
+        color="#ad8f58";
+
+        create_product [label="Create Product\nis_sellable=False\nset title, description,\ntimestamps, visibility", fillcolor="#fff8e1"];
+        upload_thumb [label="Upload Thumbnail\nput_object to S3\nset_file_metadata\nACL=public-read", fillcolor="#fff8e1"];
+        create_comments [label="Create Comments\nthreaded via parent_id\nmarkdown + sentiment", fillcolor="#fff8e1"];
+        reforge [label="Reforge Discovery Ring\nJaccard similarity\nnearest-neighbor ordering", fillcolor="#fff8e1"];
+    }
+
+    subgraph cluster_storage {
+        label="Storage";
+        style=dashed;
+        color="#666666";
+
+        sqlite [label="SQLite\nproducts + comments", fillcolor="#f5f5f5"];
+        s3 [label="S3 / Spaces\nthumbnails + inline images\nBYOB-aware", fillcolor="#f5f5f5"];
+        cdn [label="CDN\npublic URLs\n?ts= cache bust", fillcolor="#f5f5f5"];
+    }
+
+    // Edges
+    rest_api -> api_parser;
+    wxr_file -> wxr_parser;
+    api_parser -> normalize;
+    wxr_parser -> normalize;
+
+    normalize -> preprocess;
+    preprocess -> images;
+    images -> markdown;
+    markdown -> postprocess;
+
+    postprocess -> create_product;
+    normalize -> create_comments [label="comments\n(optional)", style=dashed];
+    normalize -> upload_thumb [label="featured\nimage URL", style=dashed];
+
+    create_product -> sqlite;
+    create_comments -> sqlite;
+    upload_thumb -> s3;
+    images -> s3 [label="inline\nimages"];
+    s3 -> cdn;
+
+    create_product -> reforge [label="after all\nproducts"];
+    reforge -> sqlite;
+}
diff --git a/docs/wordpress-import-pipeline.dot.svg b/docs/wordpress-import-pipeline.dot.svg
new file mode 100644
index 0000000..401e2e6
--- /dev/null
+++ b/docs/wordpress-import-pipeline.dot.svg
@@ -0,0 +1,292 @@
+
+
+
+
+
+
+wordpress_import
+
+
+cluster_source
+
+WordPress Source
+
+
+cluster_parser
+
+Source Parser
+
+
+cluster_convert
+
+HTML Conversion Pipeline
+
+
+cluster_mps
+
+MPS (Target)
+
+
+cluster_storage
+
+Storage
+
+
+
+rest_api
+
+REST API
+/wp-json/wp/v2/*
+(live site, no auth)
+
+
+
+api_parser
+
+REST API Client
+requests + pagination
+per_page=100, ?_embed
+
+
+
+rest_api->api_parser
+
+
+
+
+
+wxr_file
+
+WXR XML Export
+(offline file)
+Tools > Export
+
+
+
+wxr_parser
+
+WXR Parser
+xml.etree.ElementTree
+namespace-aware
+
+
+
+wxr_file->wxr_parser
+
+
+
+
+
+normalize
+
+Normalize
+Unified post dict
+(title, html, date,
+media_urls, comments)
+
+
+
+api_parser->normalize
+
+
+
+
+
+wxr_parser->normalize
+
+
+
+
+
+preprocess
+
+Phase 1: Pre-process
+BeautifulSoup
+- strip Gutenberg comments
+- strip shortcodes
+- strip srcset/sizes
+- strip WP classes
+
+
+
+normalize->preprocess
+
+
+
+
+
+upload_thumb
+
+Upload Thumbnail
+put_object to S3
+set_file_metadata
+ACL=public-read
+
+
+
+normalize->upload_thumb
+
+
+featured
+image URL
+
+
+
+create_comments
+
+Create Comments
+threaded via parent_id
+markdown + sentiment
+
+
+
+normalize->create_comments
+
+
+comments
+(optional)
+
+
+
+images
+
+Phase 2: Image Migration
+- strip -WxH suffixes
+- download originals
+- upload to S3
+- rewrite URLs to CDN
+
+
+
+preprocess->images
+
+
+
+
+
+markdown
+
+Phase 3: Markdown
+markdownify (MIT)
+- custom WPConverter
+- figures, code blocks
+- ATX headings
+
+
+
+images->markdown
+
+
+
+
+
+s3
+
+S3 / Spaces
+thumbnails + inline images
+BYOB-aware
+
+
+
+images->s3
+
+
+inline
+images
+
+
+
+postprocess
+
+Phase 4: Post-process
+- collapse blank lines
+- prepend metadata
+- flag unconverted HTML
+
+
+
+markdown->postprocess
+
+
+
+
+
+create_product
+
+Create Product
+is_sellable=False
+set title, description,
+timestamps, visibility
+
+
+
+postprocess->create_product
+
+
+
+
+
+reforge
+
+Reforge Discovery Ring
+Jaccard similarity
+nearest-neighbor ordering
+
+
+
+create_product->reforge
+
+
+after all
+products
+
+
+
+sqlite
+
+SQLite
+products + comments
+
+
+
+create_product->sqlite
+
+
+
+
+
+upload_thumb->s3
+
+
+
+
+
+create_comments->sqlite
+
+
+
+
+
+reforge->sqlite
+
+
+
+
+
+cdn
+
+CDN
+public URLs
+?ts= cache bust
+
+
+
+s3->cdn
+
+
+
+
+
diff --git a/docs/wordpress-import-pipeline.md b/docs/wordpress-import-pipeline.md
new file mode 100644
index 0000000..1d0f758
--- /dev/null
+++ b/docs/wordpress-import-pipeline.md
@@ -0,0 +1,514 @@
+# WordPress Import Pipeline
+
+Import WordPress sites into MPS shops. Converts posts/pages into MPS content
+items (`is_sellable=False`), downloads and re-hosts media to S3, and optionally
+imports threaded comments.
+
+**Goal:** Make MPS a credible WordPress alternative. blog.makepostsell.com already
+proves the content model works — this pipeline automates migration at scale.
+
+## Architecture Overview
+
+```dot
+// Render: dot -Tsvg docs/wordpress-import-pipeline.dot -o docs/wordpress-import-pipeline.dot.svg
+digraph wordpress_import {
+    rankdir=TB;
+    node [shape=box, style="rounded,filled", fontname="monospace", fontsize=10];
+    edge [fontname="monospace", fontsize=9];
+
+    subgraph cluster_source {
+        label="WordPress Source";
+        style=dashed;
+        color="#ad5871";
+
+        rest_api [label="REST API\n/wp-json/wp/v2/*\n(live site, no auth)", fillcolor="#fce4ec"];
+        wxr_file [label="WXR XML Export\n(offline file)\nTools > Export", fillcolor="#fce4ec"];
+    }
+
+    subgraph cluster_parser {
+        label="Source Parser";
+        style=dashed;
+        color="#5871ad";
+
+        api_parser [label="REST API Client\nrequests + pagination\nper_page=100, ?_embed", fillcolor="#e8eaf6"];
+        wxr_parser [label="WXR Parser\nxml.etree.ElementTree\nnamespace-aware", fillcolor="#e8eaf6"];
+        normalize [label="Normalize\nUnified post dict\n(title, html, date,\nmedia_urls, comments)", fillcolor="#e8eaf6"];
+    }
+
+    subgraph cluster_convert {
+        label="HTML Conversion Pipeline";
+        style=dashed;
+        color="#58ad71";
+
+        preprocess [label="Phase 1: Pre-process\nBeautifulSoup\n- strip Gutenberg comments\n- strip shortcodes\n- strip srcset/sizes\n- strip WP classes", fillcolor="#e8f5e9"];
+        images [label="Phase 2: Image Migration\n- strip -WxH suffixes\n- download originals\n- upload to S3\n- rewrite URLs to CDN", fillcolor="#e8f5e9"];
+        markdown [label="Phase 3: Markdown\nmarkdownify (MIT)\n- custom WPConverter\n- figures, code blocks\n- ATX headings", fillcolor="#e8f5e9"];
+        postprocess [label="Phase 4: Post-process\n- collapse blank lines\n- prepend metadata\n- flag unconverted HTML", fillcolor="#e8f5e9"];
+    }
+
+    subgraph cluster_mps {
+        label="MPS (Target)";
+        style=dashed;
+        color="#ad8f58";
+
+        create_product [label="Create Product\nis_sellable=False\nset title, description,\ntimestamps, visibility", fillcolor="#fff8e1"];
+        upload_thumb [label="Upload Thumbnail\nput_object to S3\nset_file_metadata\nACL=public-read", fillcolor="#fff8e1"];
+        create_comments [label="Create Comments\nthreaded via parent_id\nmarkdown + sentiment", fillcolor="#fff8e1"];
+        reforge [label="Reforge Discovery Ring\nJaccard similarity\nnearest-neighbor ordering", fillcolor="#fff8e1"];
+    }
+
+    subgraph cluster_storage {
+        label="Storage";
+        style=dashed;
+        color="#666666";
+
+        sqlite [label="SQLite\nproducts + comments", fillcolor="#f5f5f5"];
+        s3 [label="S3 / Spaces\nthumbnails + inline images\nBYOB-aware", fillcolor="#f5f5f5"];
+        cdn [label="CDN\npublic URLs\n?ts= cache bust", fillcolor="#f5f5f5"];
+    }
+
+    // Edges
+    rest_api -> api_parser;
+    wxr_file -> wxr_parser;
+    api_parser -> normalize;
+    wxr_parser -> normalize;
+
+    normalize -> preprocess;
+    preprocess -> images;
+    images -> markdown;
+    markdown -> postprocess;
+
+    postprocess -> create_product;
+    normalize -> create_comments [label="comments\n(optional)", style=dashed];
+    normalize -> upload_thumb [label="featured\nimage URL", style=dashed];
+
+    create_product -> sqlite;
+    create_comments -> sqlite;
+    upload_thumb -> s3;
+    images -> s3 [label="inline\nimages"];
+    s3 -> cdn;
+
+    create_product -> reforge [label="after all\nproducts"];
+    reforge -> sqlite;
+}
+```
+
+![Architecture Diagram](wordpress-import-pipeline.dot.svg)
+
+## Two Input Modes
+
+### Mode 1: WP REST API (live site)
+
+```bash
+python -m make_post_sell.scripts.import_wordpress \
+    --source-url https://example.com \
+    --shop-id SHOP_UUID \
+    --config data/development.ini
+```
+
+Hits the public WordPress REST API. No authentication required for published
+content. Uses `?_embed` to inline featured images and taxonomy terms, avoiding
+N+1 requests.
+
+**Pagination:** `per_page=100`, iterate pages until `page > X-WP-TotalPages`
+header value. Configurable delay between requests (default 200ms) to respect
+hosting rate limits.
+
+**Endpoints consumed:**
+
+| Endpoint | Purpose |
+|----------|---------|
+| `GET /wp-json/` | Discovery — confirm API is available |
+| `GET /wp-json/wp/v2/posts?per_page=100&page=N&_embed` | All published posts |
+| `GET /wp-json/wp/v2/pages?per_page=100&page=N&_embed` | All published pages |
+| `GET /wp-json/wp/v2/media?per_page=100&page=N` | All media (for downloads) |
+| `GET /wp-json/wp/v2/categories?per_page=100` | Category taxonomy |
+| `GET /wp-json/wp/v2/tags?per_page=100` | Tag taxonomy |
+| `GET /wp-json/wp/v2/comments?post=ID&per_page=100` | Comments per post |
+| `GET /wp-json/wp/v2/users?per_page=100` | Author info (public fields) |
+
+**Fallback:** If pretty permalinks are disabled, the API lives at
+`/?rest_route=/wp/v2/posts` instead of `/wp-json/wp/v2/posts`. The discovery
+step detects this.
+
+**When REST API is unavailable:** Some sites disable it via security plugins
+(Wordfence, Disable REST API). Detection: `GET /wp-json/` returns 404 or
+`rest_disabled`. In this case, fall back to WXR mode or abort with instructions
+to the user.
+
+### Mode 2: WXR XML Export (offline file)
+
+```bash
+python -m make_post_sell.scripts.import_wordpress \
+    --wxr-file /path/to/export.xml \
+    --shop-id SHOP_UUID \
+    --config data/development.ini
+```
+
+Parses a WordPress eXtended RSS (WXR) export file. Works offline — no network
+access to the source site needed (except for downloading media assets referenced
+by URL in the export).
+
+**How users generate WXR exports:**
+- WordPress Admin: `Dashboard > Tools > Export > Download Export File`
+- WP-CLI: `wp export --dir=/path/to/output/`
+
+**WXR structure (RSS 2.0 + WordPress namespaces):**
+
+```
+
+  
+             — author definitions
+           — category hierarchy
+                — flat tags
+               — custom taxonomies
+                  — posts, pages, attachments, nav items, CPTs
+      
+      <content:encoded> — full HTML body (CDATA)
+      <excerpt:encoded> — excerpt (CDATA)
+      <wp:post_type>    — "post", "page", "attachment", etc.
+      <wp:status>       — "publish", "draft", "private", "trash"
+      <wp:post_date_gmt>
+      <wp:post_name>    — URL slug
+      <wp:postmeta>     — key/value metadata (featured image ID, etc.)
+      <wp:comment>      — threaded comments (wp:comment_parent)
+```
+
+**XML parsing:** Use `xml.etree.ElementTree` for files under 50MB. For larger
+exports (WP-CLI splits at 15MB by default), use `iterparse()` for streaming.
+
+**Namespace handling:** WXR version affects namespace URIs (1.0, 1.1, 1.2).
+Parse `wp:wxr_version` first, then set namespace dict accordingly.
+
+**WXR gotchas:**
+- Media files are NOT included — only URLs. Must download separately.
+- `content:encoded` contains raw HTML with shortcodes and Gutenberg comments.
+- Serialized PHP in `_wp_attachment_metadata` — use `phpserialize` to decode.
+- WP-CLI exports may omit `_wp_attached_file` and `_wp_attachment_metadata`.
+- May contain invalid UTF-8 or control characters — pre-clean before parsing.
+- `wp:post_parent` links pages hierarchically (0 = top-level).
+- Attachment items have `wp:attachment_url` with the source file URL.
+
+## Content Mapping
+
+### Posts & Pages → MPS Products (Content)
+
+| WordPress | MPS Product | Notes |
+|-----------|-------------|-------|
+| `title` | `title` | Truncate to 256 chars |
+| `content:encoded` / `content.rendered` | `description` (markdown) + `description_html` | See HTML conversion pipeline below |
+| `excerpt:encoded` / `excerpt.rendered` | First line of `description` | Only if no excerpt, use first paragraph |
+| `post_date_gmt` | `created_timestamp` | Convert to milliseconds |
+| `modified_gmt` | `updated_timestamp` | Convert to milliseconds |
+| `status=publish` | `visibility=1` (public) | |
+| `status=draft` | `visibility=0` (private) | |
+| `status=private` | `visibility=0` (private) | |
+| `status=pending` | `visibility=0` (private) | |
+| `featured_media` / `_thumbnail_id` | `thumbnail1` on S3 | Download + upload |
+| Categories + tags | Metadata line in description | `**Categories:** Tech, Python` |
+| `post_type=post` | `is_sellable=False` | Content item |
+| `post_type=page` | `is_sellable=False` | Content item |
+| `slug` | Used in `absolute_url()` | MPS auto-generates from title |
+| `author` | Stored in description metadata | MPS has no per-product author field |
+| `comment_status=open` | Comments enabled | Per-shop setting in MPS |
+
+### Media/Attachments → S3
+
+| WordPress | MPS S3 | Notes |
+|-----------|--------|-------|
+| Featured image | `{shop_id}/{product_id}/thumbnail1` | Download original (strip `-WxH` suffix) |
+| Inline images in content | `{shop_id}/{product_id}/inline/{filename}` | Download, rewrite URLs in content |
+| `wp-content/uploads/YYYY/MM/file.jpg` | Flat structure under product S3 path | |
+| `srcset` / `sizes` attributes | Stripped | MPS serves single-resolution |
+
+### Comments → MPS Comments
+
+| WordPress | MPS Comment | Notes |
+|-----------|-------------|-------|
+| `comment_content` | `data` (markdown) + `data_html` | HTML→markdown conversion |
+| `comment_date_gmt` | `created_timestamp` | Milliseconds |
+| `comment_parent` | `parent_id` | 0 → null (top-level) |
+| `comment_approved=1` | `approved=True` | |
+| `comment_approved=0` | `approved=False` | |
+| `comment_approved=spam/trash` | Skipped | |
+| `comment_type=pingback/trackback` | Skipped | |
+| `comment_author` | `data` attribution line | MPS comments require a user; attribute to shop owner |
+
+## HTML Conversion Pipeline
+
+WordPress content is HTML with WordPress-specific markup. Conversion to MPS
+markdown happens in 4 phases:
+
+```dot
+// Render: dot -Tsvg docs/wordpress-import-html.dot -o docs/wordpress-import-html.dot.svg
+digraph html_conversion {
+    rankdir=LR;
+    node [shape=box, style="rounded,filled", fontname="monospace", fontsize=9];
+    edge [fontname="monospace", fontsize=8];
+
+    raw_html [label="Raw WordPress HTML\n\n<!-- wp:paragraph -->\n<p class=\"has-large-font-size\">\n[gallery ids=\"1,2,3\"]\n<img srcset=\"...\" />", fillcolor="#fce4ec", shape=note];
+
+    subgraph cluster_phase1 {
+        label="Phase 1: Pre-process (BeautifulSoup)";
+        style=dashed;
+        color="#5871ad";
+
+        strip_gutenberg [label="Strip Gutenberg\ncomments\n<!-- /?wp:\\S+.*?-->", fillcolor="#e8eaf6"];
+        strip_shortcodes [label="Strip shortcodes\n[vc_*] [et_pb_*]\n[fusion_*]\nkeep inner content", fillcolor="#e8eaf6"];
+        convert_embeds [label="Convert embeds\n[embed]URL[/embed]\n→ bare URL", fillcolor="#e8eaf6"];
+        convert_captions [label="Convert captions\n[caption] → <figure>", fillcolor="#e8eaf6"];
+        strip_srcset [label="Strip srcset/sizes\nStrip WP classes\nUnwrap empty divs", fillcolor="#e8eaf6"];
+    }
+
+    subgraph cluster_phase2 {
+        label="Phase 2: Image Migration";
+        style=dashed;
+        color="#58ad71";
+
+        find_imgs [label="Find all <img> src\nand background-image\nURLs", fillcolor="#e8f5e9"];
+        strip_suffix [label="Strip WP size suffix\n-300x200 → original\nphoto-1024x768.jpg\n→ photo.jpg", fillcolor="#e8f5e9"];
+        download [label="Download original\nfrom WP server\nrequests.get()\ntimeout=30s", fillcolor="#e8f5e9"];
+        upload_s3 [label="Upload to S3\n{shop_id}/{product_id}/\ninline/{filename}\nACL=public-read", fillcolor="#e8f5e9"];
+        rewrite [label="Rewrite src URLs\nold WP URL\n→ CDN URL", fillcolor="#e8f5e9"];
+    }
+
+    subgraph cluster_phase3 {
+        label="Phase 3: markdownify";
+        style=dashed;
+        color="#ad8f58";
+
+        converter [label="WPConverter\n(MarkdownConverter\nsubclass)", fillcolor="#fff8e1"];
+        figures [label="<figure> → ![alt](url)\n<figcaption> → caption", fillcolor="#fff8e1"];
+        code [label="<pre><code\nclass=\"language-*\">\n→ fenced code block", fillcolor="#fff8e1"];
+        headings [label="ATX headings (#)\nbody_width=0\nunicode_snob=True", fillcolor="#fff8e1"];
+    }
+
+    subgraph cluster_phase4 {
+        label="Phase 4: Post-process";
+        style=dashed;
+        color="#666666";
+
+        collapse [label="Collapse blank lines\n3+ newlines → 2", fillcolor="#f5f5f5"];
+        metadata [label="Prepend metadata\n**Categories:** ...\n**Tags:** ...\n**Author:** ...", fillcolor="#f5f5f5"];
+        flag [label="Flag posts with\nunconverted <html>\nfor manual review", fillcolor="#f5f5f5"];
+    }
+
+    output [label="MPS Content\n\nProduct.description\n(raw markdown)\n\nProduct.description_html\n(rendered HTML)", fillcolor="#e8f5e9", shape=note];
+
+    // Flow
+    raw_html -> strip_gutenberg;
+    strip_gutenberg -> strip_shortcodes;
+    strip_shortcodes -> convert_embeds;
+    convert_embeds -> convert_captions;
+    convert_captions -> strip_srcset;
+
+    strip_srcset -> find_imgs;
+    find_imgs -> strip_suffix;
+    strip_suffix -> download;
+    download -> upload_s3;
+    upload_s3 -> rewrite;
+
+    rewrite -> converter;
+    converter -> figures;
+    converter -> code;
+    converter -> headings;
+    figures -> collapse [style=invis];
+    code -> collapse [style=invis];
+    headings -> collapse;
+
+    collapse -> metadata;
+    metadata -> flag;
+    flag -> output;
+}
+```
+
+![HTML Conversion Pipeline](wordpress-import-html.dot.svg)
+
+### Library Choice: markdownify (MIT license)
+
+`markdownify` over `html2text` (GPL-3.0) because:
+- MIT license vs GPL-3.0
+- Subclassing allows per-tag override for WP-specific patterns
+- Better handling of nested lists, figures, code blocks
+- BeautifulSoup backend enables pre-processing in the same pipeline
+
+### Shortcode Handling
+
+| Shortcode | Strategy |
+|-----------|----------|
+| `[gallery ids="1,2,3"]` | Resolve attachment IDs to image URLs, emit markdown images |
+| `[caption]...[/caption]` | Convert to `<figure>`, let markdownify handle |
+| `[embed]URL[/embed]` | Extract bare URL |
+| `[video]` / `[audio]` | Extract src URL |
+| `[vc_*]` (WPBakery) | Strip shortcode tags, keep inner content |
+| `[et_pb_*]` (Divi) | Strip shortcode tags, keep inner content |
+| `[fusion_*]` (Avada) | Strip shortcode tags, keep inner content |
+| Unknown shortcodes | Strip tags, keep inner content, log warning |
+
+### Page Builder Content
+
+Page builders (Elementor, Divi, WPBakery, Beaver Builder) store layout as
+shortcodes or custom block markup. After stripping layout shortcodes, the
+remaining text content is usually extractable. Posts with heavy page builder
+usage are flagged for manual review in the import report.
+
+**Best practice:** If the WordPress site is still running, install the
+"Export Without Shortcodes" plugin before exporting. This renders shortcodes
+to HTML during WXR export, producing much cleaner content.
+
+## Product Creation Flow
+
+For each imported post/page, the script follows the same code path as
+`views/product.py:product_new()`:
+
+```python
+product = Product(title, description_markdown)
+product.shop = shop
+product.is_sellable = False
+product.is_bundle = False
+product.is_physical = False
+product.visibility = visibility  # mapped from WP status
+product.created_timestamp = wp_timestamp_to_ms(post_date_gmt)
+product.updated_timestamp = wp_timestamp_to_ms(modified_gmt)
+
+# Override description_html with the pre-rendered HTML
+# (markdownify round-trip may differ from MPS markdown renderer)
+product.description_html = rendered_html
+
+dbsession.add(product)
+dbsession.flush()
+```
+
+### S3 Upload (bypassing browser upload flow)
+
+The import script writes directly to S3, bypassing the presigned-URL webhook
+flow used by the browser:
+
+```python
+s3_client.put_object(
+    Bucket=bucket_name,
+    Key=f"{product.s3_path}/thumbnail1",
+    Body=image_bytes,
+    ContentType=content_type,
+    ACL="public-read",  # content thumbnails are always public
+    CacheControl="private, max-age=172800",
+)
+product.set_file_metadata("thumbnail1", extension, original_filename)
+product.file_bytes = {"thumbnail1": len(image_bytes)}
+```
+
+### Discovery Ring
+
+After all products are imported, trigger a single discovery ring reforge:
+
+```python
+from make_post_sell.models.shop import reforge_discovery_ring
+reforge_discovery_ring(shop)
+```
+
+This computes the greedy nearest-neighbor ordering across all public products
+using stemmed word Jaccard similarity.
+
+## CLI Interface
+
+```
+usage: import_wordpress.py [-h] --config INI --shop-id UUID
+                           [--source-url URL | --wxr-file FILE]
+                           [--dry-run] [--skip-media] [--skip-comments]
+                           [--delay MS] [--verbose]
+
+Import a WordPress site into an MPS shop.
+
+required:
+  --config INI          Path to MPS .ini config (e.g. data/development.ini)
+  --shop-id UUID        Target MPS shop UUID
+
+source (one required):
+  --source-url URL      WordPress site URL (uses REST API)
+  --wxr-file FILE       Path to WXR XML export file
+
+options:
+  --dry-run             Parse and report without writing to DB or S3
+  --skip-media          Skip image download/upload (keep original URLs)
+  --skip-comments       Skip comment import
+  --delay MS            Delay between API requests in ms (default: 200)
+  --verbose             Print detailed progress
+```
+
+## Import Report
+
+The script prints a summary after completion:
+
+```
+WordPress Import Complete
+─────────────────────────
+Source:     https://example.com (REST API)
+Target:     My Shop (shop_id: abc123...)
+Duration:   2m 34s
+
+Posts imported:      47 / 50  (3 skipped: empty content)
+Pages imported:      12 / 12
+Images downloaded:   183
+Images uploaded:     183
+Comments imported:   234
+Categories found:    8
+Tags found:          24
+
+Warnings:
+  - 3 posts had page builder shortcodes (flagged for review)
+  - 2 images returned 404 (kept original URLs)
+  - 1 post title truncated from 312 to 256 chars
+
+Discovery ring reforged with 59 products.
+```
+
+## Dependencies
+
+| Package | Purpose | License |
+|---------|---------|---------|
+| `markdownify` | HTML→Markdown conversion | MIT |
+| `beautifulsoup4` | HTML pre-processing | MIT |
+| `requests` | HTTP client (REST API + image downloads) | Apache-2.0 |
+| `lxml` | Fast XML parsing for large WXR files | BSD |
+
+All are pip-installable. `requests` and `beautifulsoup4` are likely already
+in the MPS dependency tree.
+
+## Competitive Landscape
+
+How other platforms handle WordPress migration:
+
+| Platform | Input | Posts | Pages | Images | Comments | Shortcodes |
+|----------|-------|-------|-------|--------|----------|------------|
+| **Ghost** | WXR | Yes | Yes | Scraped from live site | No (no comment system) | Partial (vc_, et_) |
+| **Squarespace** | WXR | Yes | Yes | Linked (not re-hosted) | Yes | No |
+| **Shopify** | Third-party apps | Yes | Limited | Via apps | No | No |
+| **Substack** | WXR | Yes | No | Linked | No | No |
+| **Eleventy** | REST API | Yes | Yes | Downloaded + re-hosted | No | No |
+| **Hugo (wp2hugo)** | WXR | Yes | Yes | Downloaded | Optional | Best (converts to Hugo shortcodes) |
+| **MPS (this)** | REST API + WXR | Yes | Yes | Downloaded + re-hosted to S3 | Yes (threaded) | Strip + keep content |
+
+### MPS advantages over competitors
+
+1. **Both input modes** — REST API (zero-touch) and WXR (offline). Most competitors support only one.
+2. **Image re-hosting** — Downloads and uploads to S3/CDN. Squarespace and Substack leave images on the old server (break when it goes down).
+3. **Comments** — Threaded comment import. Ghost and Substack have no comment system. Squarespace imports but loses threading.
+4. **Existing content features** — Search, RSS/Atom feeds, sitemap, comments, watch mode, discovery ring all work on imported content with zero additional setup.
+5. **No vendor lock-in** — BYOB (Bring Your Own Bucket) means media stays on infrastructure the shop owner controls.
+
+### Common migration complaints (from competitor users)
+
+1. **Images break** — #1 complaint across all platforms. We solve this by downloading + re-hosting.
+2. **Shortcode garbage** — Page builder content becomes unreadable. We strip layout shortcodes and keep text content, flagging posts for review.
+3. **Formatting loss** — We preserve HTML and also generate markdown for future editing.
+4. **SEO loss** — Average 523-day recovery from botched migration. We preserve dates, slugs, and content structure.
+5. **Timeouts on large sites** — We paginate (REST API) and stream-parse (WXR) to handle any size.
+
+## Future Enhancements
+
+- **Web UI** — Upload WXR file through shop settings (new `form_section: import-settings`)
+- **WooCommerce products** — Import sellable products with prices (`is_sellable=True`)
+- **URL redirect map** — Generate nginx/Caddy redirect rules from old URLs to new MPS URLs
+- **Incremental sync** — Re-run import to pick up new posts (skip existing by slug match)
+- **Elementor JSON** — Parse Elementor's post meta JSON for richer content extraction
+- **Multi-author** — Create MPS editor accounts per WordPress author