engineering/make_post_sell
1
0
Fork 0
Code Issues 2 Pull requests Projects Releases 2 Packages Wiki Activity Actions

docs: wordpress import pipeline architecture with dot diagrams

Browse source 
Design doc for ingesting WordPress sites into MPS shops. Covers two
input modes (REST API + WXR XML), 4-phase HTML conversion pipeline,
content/media/comment mapping, CLI interface, competitive analysis,
and future enhancements. Includes rendered dot diagrams for the
architecture overview and HTML conversion detail flow.
This commit is contained in:
russell@unturf.com 2026-03-12 11:32:47 -04:00
parent 8649e6aaae
commit 01bd9086fe
5 changed files with 1274 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

82
docs/wordpress-import-pipeline.dot Normal file
View file

@ -0,0 +1,82 @@
// WordPress Import Pipeline — Overview
// Render: dot -Tsvg docs/wordpress-import-pipeline.dot -o docs/wordpress-import-pipeline.dot.svg
digraph wordpress_import {
rankdir=TB;
node [shape=box, style="rounded,filled", fontname="monospace", fontsize=10];
edge [fontname="monospace", fontsize=9];
subgraph cluster_source {
label="WordPress Source";
style=dashed;
color="#ad5871";
rest_api [label="REST API\n/wp-json/wp/v2/*\n(live site, no auth)", fillcolor="#fce4ec"];
wxr_file [label="WXR XML Export\n(offline file)\nTools > Export", fillcolor="#fce4ec"];
}
subgraph cluster_parser {
label="Source Parser";
style=dashed;
color="#5871ad";
api_parser [label="REST API Client\nrequests + pagination\nper_page=100, ?_embed", fillcolor="#e8eaf6"];
wxr_parser [label="WXR Parser\nxml.etree.ElementTree\nnamespace-aware", fillcolor="#e8eaf6"];
normalize [label="Normalize\nUnified post dict\n(title, html, date,\nmedia_urls, comments)", fillcolor="#e8eaf6"];
}
subgraph cluster_convert {
label="HTML Conversion Pipeline";
style=dashed;
color="#58ad71";
preprocess [label="Phase 1: Pre-process\nBeautifulSoup\n- strip Gutenberg comments\n- strip shortcodes\n- strip srcset/sizes\n- strip WP classes", fillcolor="#e8f5e9"];
images [label="Phase 2: Image Migration\n- strip -WxH suffixes\n- download originals\n- upload to S3\n- rewrite URLs to CDN", fillcolor="#e8f5e9"];
markdown [label="Phase 3: Markdown\nmarkdownify (MIT)\n- custom WPConverter\n- figures, code blocks\n- ATX headings", fillcolor="#e8f5e9"];
postprocess [label="Phase 4: Post-process\n- collapse blank lines\n- prepend metadata\n- flag unconverted HTML", fillcolor="#e8f5e9"];
}
subgraph cluster_mps {
label="MPS (Target)";
style=dashed;
color="#ad8f58";
create_product [label="Create Product\nis_sellable=False\nset title, description,\ntimestamps, visibility", fillcolor="#fff8e1"];
upload_thumb [label="Upload Thumbnail\nput_object to S3\nset_file_metadata\nACL=public-read", fillcolor="#fff8e1"];
create_comments [label="Create Comments\nthreaded via parent_id\nmarkdown + sentiment", fillcolor="#fff8e1"];
reforge [label="Reforge Discovery Ring\nJaccard similarity\nnearest-neighbor ordering", fillcolor="#fff8e1"];
}
subgraph cluster_storage {
label="Storage";
style=dashed;
color="#666666";
sqlite [label="SQLite\nproducts + comments", fillcolor="#f5f5f5"];
s3 [label="S3 / Spaces\nthumbnails + inline images\nBYOB-aware", fillcolor="#f5f5f5"];
cdn [label="CDN\npublic URLs\n?ts= cache bust", fillcolor="#f5f5f5"];
}
// Edges
rest_api -> api_parser;
wxr_file -> wxr_parser;
api_parser -> normalize;
wxr_parser -> normalize;
normalize -> preprocess;
preprocess -> images;
images -> markdown;
markdown -> postprocess;
postprocess -> create_product;
normalize -> create_comments [label="comments\n(optional)", style=dashed];
normalize -> upload_thumb [label="featured\nimage URL", style=dashed];
create_product -> sqlite;
create_comments -> sqlite;
upload_thumb -> s3;
images -> s3 [label="inline\nimages"];
s3 -> cdn;
create_product -> reforge [label="after all\nproducts"];
reforge -> sqlite;
}