russell@unturf.com
01bd9086fe
Design doc for ingesting WordPress sites into MPS shops. Covers two input modes (REST API + WXR XML), 4-phase HTML conversion pipeline, content/media/comment mapping, CLI interface, competitive analysis, and future enhancements. Includes rendered dot diagrams for the architecture overview and HTML conversion detail flow.
82 lines
3.4 KiB
Text
82 lines
3.4 KiB
Text
// WordPress Import Pipeline — Overview
|
|
// Render: dot -Tsvg docs/wordpress-import-pipeline.dot -o docs/wordpress-import-pipeline.dot.svg
|
|
digraph wordpress_import {
|
|
rankdir=TB;
|
|
node [shape=box, style="rounded,filled", fontname="monospace", fontsize=10];
|
|
edge [fontname="monospace", fontsize=9];
|
|
|
|
subgraph cluster_source {
|
|
label="WordPress Source";
|
|
style=dashed;
|
|
color="#ad5871";
|
|
|
|
rest_api [label="REST API\n/wp-json/wp/v2/*\n(live site, no auth)", fillcolor="#fce4ec"];
|
|
wxr_file [label="WXR XML Export\n(offline file)\nTools > Export", fillcolor="#fce4ec"];
|
|
}
|
|
|
|
subgraph cluster_parser {
|
|
label="Source Parser";
|
|
style=dashed;
|
|
color="#5871ad";
|
|
|
|
api_parser [label="REST API Client\nrequests + pagination\nper_page=100, ?_embed", fillcolor="#e8eaf6"];
|
|
wxr_parser [label="WXR Parser\nxml.etree.ElementTree\nnamespace-aware", fillcolor="#e8eaf6"];
|
|
normalize [label="Normalize\nUnified post dict\n(title, html, date,\nmedia_urls, comments)", fillcolor="#e8eaf6"];
|
|
}
|
|
|
|
subgraph cluster_convert {
|
|
label="HTML Conversion Pipeline";
|
|
style=dashed;
|
|
color="#58ad71";
|
|
|
|
preprocess [label="Phase 1: Pre-process\nBeautifulSoup\n- strip Gutenberg comments\n- strip shortcodes\n- strip srcset/sizes\n- strip WP classes", fillcolor="#e8f5e9"];
|
|
images [label="Phase 2: Image Migration\n- strip -WxH suffixes\n- download originals\n- upload to S3\n- rewrite URLs to CDN", fillcolor="#e8f5e9"];
|
|
markdown [label="Phase 3: Markdown\nmarkdownify (MIT)\n- custom WPConverter\n- figures, code blocks\n- ATX headings", fillcolor="#e8f5e9"];
|
|
postprocess [label="Phase 4: Post-process\n- collapse blank lines\n- prepend metadata\n- flag unconverted HTML", fillcolor="#e8f5e9"];
|
|
}
|
|
|
|
subgraph cluster_mps {
|
|
label="MPS (Target)";
|
|
style=dashed;
|
|
color="#ad8f58";
|
|
|
|
create_product [label="Create Product\nis_sellable=False\nset title, description,\ntimestamps, visibility", fillcolor="#fff8e1"];
|
|
upload_thumb [label="Upload Thumbnail\nput_object to S3\nset_file_metadata\nACL=public-read", fillcolor="#fff8e1"];
|
|
create_comments [label="Create Comments\nthreaded via parent_id\nmarkdown + sentiment", fillcolor="#fff8e1"];
|
|
reforge [label="Reforge Discovery Ring\nJaccard similarity\nnearest-neighbor ordering", fillcolor="#fff8e1"];
|
|
}
|
|
|
|
subgraph cluster_storage {
|
|
label="Storage";
|
|
style=dashed;
|
|
color="#666666";
|
|
|
|
sqlite [label="SQLite\nproducts + comments", fillcolor="#f5f5f5"];
|
|
s3 [label="S3 / Spaces\nthumbnails + inline images\nBYOB-aware", fillcolor="#f5f5f5"];
|
|
cdn [label="CDN\npublic URLs\n?ts= cache bust", fillcolor="#f5f5f5"];
|
|
}
|
|
|
|
// Edges
|
|
rest_api -> api_parser;
|
|
wxr_file -> wxr_parser;
|
|
api_parser -> normalize;
|
|
wxr_parser -> normalize;
|
|
|
|
normalize -> preprocess;
|
|
preprocess -> images;
|
|
images -> markdown;
|
|
markdown -> postprocess;
|
|
|
|
postprocess -> create_product;
|
|
normalize -> create_comments [label="comments\n(optional)", style=dashed];
|
|
normalize -> upload_thumb [label="featured\nimage URL", style=dashed];
|
|
|
|
create_product -> sqlite;
|
|
create_comments -> sqlite;
|
|
upload_thumb -> s3;
|
|
images -> s3 [label="inline\nimages"];
|
|
s3 -> cdn;
|
|
|
|
create_product -> reforge [label="after all\nproducts"];
|
|
reforge -> sqlite;
|
|
}
|