russell.ballestrini.net/lib/generate_jsonblog.py

#!/usr/bin/env python3
"""
Generate JSON Blog (jsonblog.dev) format - single JSON file with all posts.
Links to Markdown files (index.md) generated by rst2md.py instead of embedding content.
"""

import json
import sys
import xml.etree.ElementTree as ET
from pathlib import Path
import re


def strip_html(html):
    """Remove HTML tags and decode entities."""
    if not html:
        return ''
    # Remove HTML tags
    text = re.sub(r'<[^>]+>', '', html)
    # Decode common HTML entities
    text = text.replace('&amp;', '&')
    text = text.replace('&lt;', '<')
    text = text.replace('&gt;', '>')
    text = text.replace('&quot;', '"')
    text = text.replace('&#39;', "'")
    text = text.replace('&nbsp;', ' ')
    # Collapse whitespace
    text = re.sub(r'\s+', ' ', text).strip()
    return text


def parse_atom_feed(atom_path):
    """Parse Atom feed and extract entries."""
    tree = ET.parse(atom_path)
    root = tree.getroot()
    ns = {'atom': 'http://www.w3.org/2005/Atom'}

    feed_info = {
        'title': root.find('atom:title', ns).text if root.find('atom:title', ns) is not None else '',
        'subtitle': root.find('atom:subtitle', ns).text if root.find('atom:subtitle', ns) is not None else '',
        'author': '',
    }

    author_elem = root.find('atom:author/atom:name', ns)
    if author_elem is not None:
        feed_info['author'] = author_elem.text

    entries = []
    for entry in root.findall('atom:entry', ns):
        item = {
            'title': '',
            'slug': '',
            'content': '',
            'content_html': '',
            'summary': '',
            'createdAt': '',
            'updatedAt': '',
            'tags': [],
            'url': '',
        }

        # URL and slug
        for link in entry.findall('atom:link', ns):
            if link.get('rel') == 'alternate':
                item['url'] = link.get('href', '')
                # Extract slug from URL
                slug_match = re.search(r'/([^/]+)/?$', item['url'])
                if slug_match:
                    item['slug'] = slug_match.group(1)
                break

        # Title
        title_elem = entry.find('atom:title', ns)
        if title_elem is not None:
            item['title'] = title_elem.text

        # Content (will be replaced with markdown later)
        content_elem = entry.find('atom:content', ns)
        if content_elem is not None:
            item['content_html'] = content_elem.text or ''
            item['content'] = strip_html(item['content_html'])

        # Summary
        summary_elem = entry.find('atom:summary', ns)
        if summary_elem is not None:
            item['summary'] = summary_elem.text or ''

        # Dates (convert to YYYY-MM-DD for JSON Blog format)
        published_elem = entry.find('atom:published', ns)
        if published_elem is not None:
            date_str = published_elem.text
            if date_str:
                item['createdAt'] = date_str[:10]  # YYYY-MM-DD

        updated_elem = entry.find('atom:updated', ns)
        if updated_elem is not None:
            date_str = updated_elem.text
            if date_str:
                item['updatedAt'] = date_str[:10]

        # Tags
        for category in entry.findall('atom:category', ns):
            term = category.get('term')
            if term:
                item['tags'].append(term)

        entries.append(item)

    return feed_info, entries


def generate_jsonblog(atom_path, output_dir, site_url):
    """Generate JSON Blog format with links to Markdown content."""
    feed_info, entries = parse_atom_feed(atom_path)

    jsonblog = {
        'site': {
            'title': feed_info['title'],
            'description': feed_info['subtitle'],
            'url': site_url,
        },
        'basics': {
            'name': feed_info['author'] or 'Russell Ballestrini',
            'url': site_url,
        },
        'posts': []
    }

    markdown_found = 0
    for entry in entries:
        post = {
            'title': entry['title'],
            'slug': entry['slug'],
            'createdAt': entry['createdAt'],
        }

        # Check if markdown file exists, then link to it
        md_path = output_dir / entry['slug'] / 'index.md'
        if md_path.exists():
            post['content_url'] = f"{site_url}/{entry['slug']}/index.md"
            post['content_type'] = 'markdown'
            markdown_found += 1

        if entry['updatedAt'] and entry['updatedAt'] != entry['createdAt']:
            post['updatedAt'] = entry['updatedAt']

        if entry['tags']:
            post['tags'] = entry['tags']

        if entry['summary']:
            post['summary'] = entry['summary']

        if entry['url']:
            post['url'] = entry['url']

        jsonblog['posts'].append(post)

    print(f"Generated {len(jsonblog['posts'])} posts ({markdown_found} with markdown links)")
    return jsonblog


if __name__ == '__main__':
    if len(sys.argv) < 2:
        print("Usage: generate_jsonblog.py <output_directory> [site_url]")
        sys.exit(1)

    output_dir = Path(sys.argv[1])
    site_url = sys.argv[2] if len(sys.argv) > 2 else 'https://russell.ballestrini.net'

    atom_path = output_dir / 'feeds' / 'all.atom.xml'
    if not atom_path.exists():
        print(f"Error: Atom feed not found at {atom_path}")
        sys.exit(1)

    jsonblog = generate_jsonblog(atom_path, output_dir, site_url)

    output_path = output_dir / 'blog.json'
    output_path.write_text(json.dumps(jsonblog, indent=2, ensure_ascii=False))
    print(f"Generated {output_path}")