russell.ballestrini.net/lib/generate_jsonblog.py

#!/usr/bin/env python3
"""
Generate JSON Blog (jsonblog.dev) format - single JSON file with all posts.
Uses Markdown content from index.md files generated by rst2md.py.
"""

import json
import sys
import xml.etree.ElementTree as ET
from pathlib import Path
import re


def strip_html(html):
    """Remove HTML tags and decode entities."""
    if not html:
        return ''
    # Remove HTML tags
    text = re.sub(r'<[^>]+>', '', html)
    # Decode common HTML entities
    text = text.replace('&amp;', '&')
    text = text.replace('&lt;', '<')
    text = text.replace('&gt;', '>')
    text = text.replace('&quot;', '"')
    text = text.replace('&#39;', "'")
    text = text.replace('&nbsp;', ' ')
    # Collapse whitespace
    text = re.sub(r'\s+', ' ', text).strip()
    return text


def parse_atom_feed(atom_path):
    """Parse Atom feed and extract entries."""
    tree = ET.parse(atom_path)
    root = tree.getroot()
    ns = {'atom': 'http://www.w3.org/2005/Atom'}

    feed_info = {
        'title': root.find('atom:title', ns).text if root.find('atom:title', ns) is not None else '',
        'subtitle': root.find('atom:subtitle', ns).text if root.find('atom:subtitle', ns) is not None else '',
        'author': '',
    }

    author_elem = root.find('atom:author/atom:name', ns)
    if author_elem is not None:
        feed_info['author'] = author_elem.text

    entries = []
    for entry in root.findall('atom:entry', ns):
        item = {
            'title': '',
            'slug': '',
            'content': '',
            'content_html': '',
            'summary': '',
            'createdAt': '',
            'updatedAt': '',
            'tags': [],
            'url': '',
        }

        # URL and slug
        for link in entry.findall('atom:link', ns):
            if link.get('rel') == 'alternate':
                item['url'] = link.get('href', '')
                # Extract slug from URL
                slug_match = re.search(r'/([^/]+)/?$', item['url'])
                if slug_match:
                    item['slug'] = slug_match.group(1)
                break

        # Title
        title_elem = entry.find('atom:title', ns)
        if title_elem is not None:
            item['title'] = title_elem.text

        # Content (will be replaced with markdown later)
        content_elem = entry.find('atom:content', ns)
        if content_elem is not None:
            item['content_html'] = content_elem.text or ''
            item['content'] = strip_html(item['content_html'])

        # Summary
        summary_elem = entry.find('atom:summary', ns)
        if summary_elem is not None:
            item['summary'] = summary_elem.text or ''

        # Dates (convert to YYYY-MM-DD for JSON Blog format)
        published_elem = entry.find('atom:published', ns)
        if published_elem is not None:
            date_str = published_elem.text
            if date_str:
                item['createdAt'] = date_str[:10]  # YYYY-MM-DD

        updated_elem = entry.find('atom:updated', ns)
        if updated_elem is not None:
            date_str = updated_elem.text
            if date_str:
                item['updatedAt'] = date_str[:10]

        # Tags
        for category in entry.findall('atom:category', ns):
            term = category.get('term')
            if term:
                item['tags'].append(term)

        entries.append(item)

    return feed_info, entries


def load_markdown_content(output_dir, slug):
    """Load markdown content from index.md file."""
    md_path = output_dir / slug / 'index.md'
    if md_path.exists():
        try:
            return md_path.read_text(encoding='utf-8')
        except Exception as e:
            print(f"Warning: Could not read {md_path}: {e}")
            return None
    return None


def generate_jsonblog(atom_path, output_dir, site_url):
    """Generate JSON Blog format with Markdown content."""
    feed_info, entries = parse_atom_feed(atom_path)

    jsonblog = {
        'site': {
            'title': feed_info['title'],
            'description': feed_info['subtitle'],
            'url': site_url,
        },
        'basics': {
            'name': feed_info['author'] or 'Russell Ballestrini',
            'url': site_url,
        },
        'posts': []
    }

    markdown_found = 0
    for entry in entries:
        post = {
            'title': entry['title'],
            'slug': entry['slug'],
            'createdAt': entry['createdAt'],
        }

        # Try to load markdown content
        md_content = load_markdown_content(output_dir, entry['slug'])
        if md_content:
            post['content'] = md_content
            post['content_type'] = 'markdown'
            markdown_found += 1
        else:
            # Fallback to plain text from HTML
            post['content'] = entry['content']

        if entry['updatedAt'] and entry['updatedAt'] != entry['createdAt']:
            post['updatedAt'] = entry['updatedAt']

        if entry['tags']:
            post['tags'] = entry['tags']

        if entry['summary']:
            post['summary'] = entry['summary']

        if entry['url']:
            post['url'] = entry['url']

        jsonblog['posts'].append(post)

    print(f"Generated {len(jsonblog['posts'])} posts ({markdown_found} with markdown content)")
    return jsonblog


if __name__ == '__main__':
    if len(sys.argv) < 2:
        print("Usage: generate_jsonblog.py <output_directory> [site_url]")
        sys.exit(1)

    output_dir = Path(sys.argv[1])
    site_url = sys.argv[2] if len(sys.argv) > 2 else 'https://russell.ballestrini.net'

    atom_path = output_dir / 'feeds' / 'all.atom.xml'
    if not atom_path.exists():
        print(f"Error: Atom feed not found at {atom_path}")
        sys.exit(1)

    jsonblog = generate_jsonblog(atom_path, output_dir, site_url)

    output_path = output_dir / 'blog.json'
    output_path.write_text(json.dumps(jsonblog, indent=2, ensure_ascii=False))
    print(f"Generated {output_path}")