russell@unturf.com
01bd9086fe
Design doc for ingesting WordPress sites into MPS shops. Covers two input modes (REST API + WXR XML), 4-phase HTML conversion pipeline, content/media/comment mapping, CLI interface, competitive analysis, and future enhancements. Includes rendered dot diagrams for the architecture overview and HTML conversion detail flow.
81 lines
3.5 KiB
Text
81 lines
3.5 KiB
Text
// WordPress HTML Conversion Pipeline — Detail
|
|
// Render: dot -Tsvg docs/wordpress-import-html.dot -o docs/wordpress-import-html.dot.svg
|
|
digraph html_conversion {
|
|
rankdir=LR;
|
|
node [shape=box, style="rounded,filled", fontname="monospace", fontsize=9];
|
|
edge [fontname="monospace", fontsize=8];
|
|
|
|
raw_html [label="Raw WordPress HTML\n\n<!-- wp:paragraph -->\n<p class=\"has-large-font-size\">\n[gallery ids=\"1,2,3\"]\n<img srcset=\"...\" />", fillcolor="#fce4ec", shape=note];
|
|
|
|
subgraph cluster_phase1 {
|
|
label="Phase 1: Pre-process (BeautifulSoup)";
|
|
style=dashed;
|
|
color="#5871ad";
|
|
|
|
strip_gutenberg [label="Strip Gutenberg\ncomments\n<!-- /?wp:\\S+.*?-->", fillcolor="#e8eaf6"];
|
|
strip_shortcodes [label="Strip shortcodes\n[vc_*] [et_pb_*]\n[fusion_*]\nkeep inner content", fillcolor="#e8eaf6"];
|
|
convert_embeds [label="Convert embeds\n[embed]URL[/embed]\n→ bare URL", fillcolor="#e8eaf6"];
|
|
convert_captions [label="Convert captions\n[caption] → <figure>", fillcolor="#e8eaf6"];
|
|
strip_srcset [label="Strip srcset/sizes\nStrip WP classes\nUnwrap empty divs", fillcolor="#e8eaf6"];
|
|
}
|
|
|
|
subgraph cluster_phase2 {
|
|
label="Phase 2: Image Migration";
|
|
style=dashed;
|
|
color="#58ad71";
|
|
|
|
find_imgs [label="Find all <img> src\nand background-image\nURLs", fillcolor="#e8f5e9"];
|
|
strip_suffix [label="Strip WP size suffix\n-300x200 → original\nphoto-1024x768.jpg\n→ photo.jpg", fillcolor="#e8f5e9"];
|
|
download [label="Download original\nfrom WP server\nrequests.get()\ntimeout=30s", fillcolor="#e8f5e9"];
|
|
upload_s3 [label="Upload to S3\n{shop_id}/{product_id}/\ninline/{filename}\nACL=public-read", fillcolor="#e8f5e9"];
|
|
rewrite [label="Rewrite src URLs\nold WP URL\n→ CDN URL", fillcolor="#e8f5e9"];
|
|
}
|
|
|
|
subgraph cluster_phase3 {
|
|
label="Phase 3: markdownify";
|
|
style=dashed;
|
|
color="#ad8f58";
|
|
|
|
converter [label="WPConverter\n(MarkdownConverter\nsubclass)", fillcolor="#fff8e1"];
|
|
figures [label="<figure> → \n<figcaption> → caption", fillcolor="#fff8e1"];
|
|
code [label="<pre><code\nclass=\"language-*\">\n→ fenced code block", fillcolor="#fff8e1"];
|
|
headings [label="ATX headings (#)\nbody_width=0\nunicode_snob=True", fillcolor="#fff8e1"];
|
|
}
|
|
|
|
subgraph cluster_phase4 {
|
|
label="Phase 4: Post-process";
|
|
style=dashed;
|
|
color="#666666";
|
|
|
|
collapse [label="Collapse blank lines\n3+ newlines → 2", fillcolor="#f5f5f5"];
|
|
metadata [label="Prepend metadata\n**Categories:** ...\n**Tags:** ...\n**Author:** ...", fillcolor="#f5f5f5"];
|
|
flag [label="Flag posts with\nunconverted <html>\nfor manual review", fillcolor="#f5f5f5"];
|
|
}
|
|
|
|
output [label="MPS Content\n\nProduct.description\n(raw markdown)\n\nProduct.description_html\n(rendered HTML)", fillcolor="#e8f5e9", shape=note];
|
|
|
|
// Flow
|
|
raw_html -> strip_gutenberg;
|
|
strip_gutenberg -> strip_shortcodes;
|
|
strip_shortcodes -> convert_embeds;
|
|
convert_embeds -> convert_captions;
|
|
convert_captions -> strip_srcset;
|
|
|
|
strip_srcset -> find_imgs;
|
|
find_imgs -> strip_suffix;
|
|
strip_suffix -> download;
|
|
download -> upload_s3;
|
|
upload_s3 -> rewrite;
|
|
|
|
rewrite -> converter;
|
|
converter -> figures;
|
|
converter -> code;
|
|
converter -> headings;
|
|
figures -> collapse [style=invis];
|
|
code -> collapse [style=invis];
|
|
headings -> collapse;
|
|
|
|
collapse -> metadata;
|
|
metadata -> flag;
|
|
flag -> output;
|
|
}
|