Add lib/ format generation scripts for agent-friendly formats

- lib/rst2md.py - RST to Markdown converter using pandoc
- lib/generate_jsonfeed.py - JSON Feed v1.1 generator
- lib/generate_jsonblog.py - JSON Blog format (jsonblog.dev)
- lib/generate_plaintext.py - HTML to plain text converter
- lib/generate_llms_txt.py - AI agent onboarding file

These scripts generate multiple formats for AI/LLM consumption:
- blog.json at root (entire blog in one file)
- feeds/all.json (JSON Feed format)
- llms.txt (onboarding documentation)
- index.md (Markdown versions)
- index.txt (plain text versions)
This commit is contained in:
russell@unturf.com 2026-01-19 14:36:20 -05:00
parent 6576bef263
commit 7c91647a55
5 changed files with 776 additions and 0 deletions

170
lib/generate_jsonblog.py Normal file
View file

@ -0,0 +1,170 @@
#!/usr/bin/env python3
"""
Generate JSON Blog (jsonblog.dev) format - single JSON file with all posts.
"""
import json
import sys
import xml.etree.ElementTree as ET
from pathlib import Path
import re
def strip_html(html):
"""Remove HTML tags and decode entities."""
if not html:
return ''
# Remove HTML tags
text = re.sub(r'<[^>]+>', '', html)
# Decode common HTML entities
text = text.replace('&amp;', '&')
text = text.replace('&lt;', '<')
text = text.replace('&gt;', '>')
text = text.replace('&quot;', '"')
text = text.replace('&#39;', "'")
text = text.replace('&nbsp;', ' ')
# Collapse whitespace
text = re.sub(r'\s+', ' ', text).strip()
return text
def parse_atom_feed(atom_path):
"""Parse Atom feed and extract entries."""
tree = ET.parse(atom_path)
root = tree.getroot()
ns = {'atom': 'http://www.w3.org/2005/Atom'}
feed_info = {
'title': root.find('atom:title', ns).text if root.find('atom:title', ns) is not None else '',
'subtitle': root.find('atom:subtitle', ns).text if root.find('atom:subtitle', ns) is not None else '',
'author': '',
}
author_elem = root.find('atom:author/atom:name', ns)
if author_elem is not None:
feed_info['author'] = author_elem.text
entries = []
for entry in root.findall('atom:entry', ns):
item = {
'title': '',
'slug': '',
'content': '',
'content_html': '',
'summary': '',
'createdAt': '',
'updatedAt': '',
'tags': [],
'url': '',
}
# URL and slug
for link in entry.findall('atom:link', ns):
if link.get('rel') == 'alternate':
item['url'] = link.get('href', '')
# Extract slug from URL
slug_match = re.search(r'/([^/]+)/?$', item['url'])
if slug_match:
item['slug'] = slug_match.group(1)
break
# Title
title_elem = entry.find('atom:title', ns)
if title_elem is not None:
item['title'] = title_elem.text
# Content
content_elem = entry.find('atom:content', ns)
if content_elem is not None:
item['content_html'] = content_elem.text or ''
item['content'] = strip_html(item['content_html'])
# Summary
summary_elem = entry.find('atom:summary', ns)
if summary_elem is not None:
item['summary'] = summary_elem.text or ''
# Dates (convert to YYYY-MM-DD for JSON Blog format)
published_elem = entry.find('atom:published', ns)
if published_elem is not None:
date_str = published_elem.text
if date_str:
item['createdAt'] = date_str[:10] # YYYY-MM-DD
updated_elem = entry.find('atom:updated', ns)
if updated_elem is not None:
date_str = updated_elem.text
if date_str:
item['updatedAt'] = date_str[:10]
# Tags
for category in entry.findall('atom:category', ns):
term = category.get('term')
if term:
item['tags'].append(term)
entries.append(item)
return feed_info, entries
def generate_jsonblog(atom_path, site_url):
"""Generate JSON Blog format."""
feed_info, entries = parse_atom_feed(atom_path)
jsonblog = {
'site': {
'title': feed_info['title'],
'description': feed_info['subtitle'],
'url': site_url,
},
'basics': {
'name': feed_info['author'] or 'Russell Ballestrini',
'url': site_url,
},
'posts': []
}
for entry in entries:
post = {
'title': entry['title'],
'slug': entry['slug'],
'content': entry['content'], # Plain text for JSON Blog
'createdAt': entry['createdAt'],
}
if entry['updatedAt'] and entry['updatedAt'] != entry['createdAt']:
post['updatedAt'] = entry['updatedAt']
if entry['tags']:
post['tags'] = entry['tags']
if entry['summary']:
post['summary'] = entry['summary']
if entry['url']:
post['url'] = entry['url']
jsonblog['posts'].append(post)
return jsonblog
if __name__ == '__main__':
if len(sys.argv) < 2:
print("Usage: generate_jsonblog.py <output_directory> [site_url]")
sys.exit(1)
output_dir = Path(sys.argv[1])
site_url = sys.argv[2] if len(sys.argv) > 2 else 'https://russell.ballestrini.net'
atom_path = output_dir / 'feeds' / 'all.atom.xml'
if not atom_path.exists():
print(f"Error: Atom feed not found at {atom_path}")
sys.exit(1)
jsonblog = generate_jsonblog(atom_path, site_url)
output_path = output_dir / 'blog.json'
output_path.write_text(json.dumps(jsonblog, indent=2, ensure_ascii=False))
print(f"Generated {output_path} ({len(jsonblog['posts'])} posts)")

162
lib/generate_jsonfeed.py Normal file
View file

@ -0,0 +1,162 @@
#!/usr/bin/env python3
"""
Generate JSON Feed (jsonfeed.org) from Pelican Atom feed.
"""
import json
import sys
import xml.etree.ElementTree as ET
from datetime import datetime
from pathlib import Path
def parse_atom_feed(atom_path):
"""Parse Atom feed and extract entries."""
tree = ET.parse(atom_path)
root = tree.getroot()
# Atom namespace
ns = {'atom': 'http://www.w3.org/2005/Atom'}
feed_info = {
'title': root.find('atom:title', ns).text if root.find('atom:title', ns) is not None else '',
'subtitle': root.find('atom:subtitle', ns).text if root.find('atom:subtitle', ns) is not None else '',
'link': '',
'author': '',
}
# Get feed link
for link in root.findall('atom:link', ns):
if link.get('rel') == 'alternate':
feed_info['link'] = link.get('href', '')
break
# Get author
author_elem = root.find('atom:author/atom:name', ns)
if author_elem is not None:
feed_info['author'] = author_elem.text
entries = []
for entry in root.findall('atom:entry', ns):
item = {
'id': '',
'url': '',
'title': '',
'content_html': '',
'summary': '',
'date_published': '',
'date_modified': '',
'tags': [],
}
# ID
id_elem = entry.find('atom:id', ns)
if id_elem is not None:
item['id'] = id_elem.text
# URL
for link in entry.findall('atom:link', ns):
if link.get('rel') == 'alternate':
item['url'] = link.get('href', '')
break
# Title
title_elem = entry.find('atom:title', ns)
if title_elem is not None:
item['title'] = title_elem.text
# Content
content_elem = entry.find('atom:content', ns)
if content_elem is not None:
item['content_html'] = content_elem.text or ''
# Summary
summary_elem = entry.find('atom:summary', ns)
if summary_elem is not None:
item['summary'] = summary_elem.text or ''
# Dates
published_elem = entry.find('atom:published', ns)
if published_elem is not None:
item['date_published'] = published_elem.text
updated_elem = entry.find('atom:updated', ns)
if updated_elem is not None:
item['date_modified'] = updated_elem.text
# Tags/categories
for category in entry.findall('atom:category', ns):
term = category.get('term')
if term:
item['tags'].append(term)
entries.append(item)
return feed_info, entries
def generate_jsonfeed(atom_path, site_url):
"""Generate JSON Feed from Atom feed."""
feed_info, entries = parse_atom_feed(atom_path)
jsonfeed = {
'version': 'https://jsonfeed.org/version/1.1',
'title': feed_info['title'],
'home_page_url': site_url,
'feed_url': f'{site_url}/feeds/all.json',
'description': feed_info['subtitle'],
'language': 'en-US',
'authors': [
{
'name': feed_info['author'] or 'Russell Ballestrini',
'url': site_url,
}
],
'items': []
}
for entry in entries:
item = {
'id': entry['id'],
'url': entry['url'],
'title': entry['title'],
}
if entry['content_html']:
item['content_html'] = entry['content_html']
if entry['summary']:
item['summary'] = entry['summary']
if entry['date_published']:
item['date_published'] = entry['date_published']
if entry['date_modified']:
item['date_modified'] = entry['date_modified']
if entry['tags']:
item['tags'] = entry['tags']
jsonfeed['items'].append(item)
return jsonfeed
if __name__ == '__main__':
if len(sys.argv) < 2:
print("Usage: generate_jsonfeed.py <output_directory> [site_url]")
sys.exit(1)
output_dir = Path(sys.argv[1])
site_url = sys.argv[2] if len(sys.argv) > 2 else 'https://russell.ballestrini.net'
atom_path = output_dir / 'feeds' / 'all.atom.xml'
if not atom_path.exists():
print(f"Error: Atom feed not found at {atom_path}")
sys.exit(1)
jsonfeed = generate_jsonfeed(atom_path, site_url)
output_path = output_dir / 'feeds' / 'all.json'
output_path.write_text(json.dumps(jsonfeed, indent=2, ensure_ascii=False))
print(f"Generated {output_path}")

199
lib/generate_llms_txt.py Normal file
View file

@ -0,0 +1,199 @@
#!/usr/bin/env python3
"""
Generate llms.txt for AI agent onboarding.
Provides structured information about the site for LLM crawlers & agents.
"""
import json
import sys
from datetime import datetime
from pathlib import Path
def get_tags_and_categories(output_dir):
"""Extract tags & categories from output directory structure."""
output_path = Path(output_dir)
tags = []
tags_dir = output_path / 'tags'
if tags_dir.exists():
tags = sorted([d.name for d in tags_dir.iterdir() if d.is_dir()])
categories = []
cat_dir = output_path / 'category'
if cat_dir.exists():
categories = sorted([d.name for d in cat_dir.iterdir() if d.is_dir()])
return tags, categories
def get_recent_articles(output_dir, limit=20):
"""Get list of recent articles from output directory."""
output_path = Path(output_dir)
articles = []
# Find all index.html files that look like articles (have dates in parent dir name)
for index_file in output_path.glob('*/index.html'):
dirname = index_file.parent.name
# Skip non-article directories
if dirname in ('tags', 'category', 'author', 'uploads', 'feeds', 'about', 'archives'):
continue
if dirname.startswith('20'): # Date-prefixed directories (cover letters, etc.)
continue
articles.append(dirname)
return sorted(articles)[:limit]
def generate_llms_txt(output_dir, site_url='https://russell.ballestrini.net'):
"""Generate llms.txt content."""
tags, categories = get_tags_and_categories(output_dir)
recent_articles = get_recent_articles(output_dir)
content = f"""# llms.txt - AI Agent Onboarding for russell.ballestrini.net
> Russell Ballestrini's technical blog covering software engineering, DevOps,
> infrastructure, Python, cloud computing, and entrepreneurship.
## Site Information
- **Author**: Russell Ballestrini
- **Site URL**: {site_url}
- **Generated**: {datetime.now().isoformat()}
## Feeds (Recommended for Crawling)
### Atom Feed
- URL: {site_url}/feeds/all.atom.xml
- Format: Atom XML
- Description: Complete feed of all blog posts, newest first
### JSON Feed (jsonfeed.org v1.1)
- URL: {site_url}/feeds/all.json
- Format: JSON Feed 1.1
- Description: Same as Atom feed, but JSON for easier parsing
### JSON Blog (jsonblog.dev)
- URL: {site_url}/blog.json
- Format: JSON Blog
- Description: **Entire blog in one file** - all posts with plain text content
## Content Formats
Each article is available in multiple formats:
- **HTML**: `/{{slug}}/index.html` - Rendered page with styling
- **RST**: `/{{slug}}/index.rst` - reStructuredText source
- **Markdown**: `/{{slug}}/index.md` - GitHub-flavored Markdown
- **Plain Text**: `/{{slug}}/index.txt` - Raw text, no markup (best for LLMs)
## API-Friendly Endpoints
### Resume (JSONResume Format)
- URL: {site_url}/uploads/russell.ballestrini.resume.json
- Format: JSONResume standard schema
- Description: Structured resume data in machine-readable format
## Navigation Structure
### By Category
"""
if categories:
for cat in categories[:15]:
content += f"- [{cat}]({site_url}/category/{cat}/)\n"
content += f"""
### By Tag
"""
if tags:
for tag in tags[:30]:
content += f"- [{tag}]({site_url}/tags/{tag}/)\n"
content += f"""
## Crawling Recommendations
1. **Start with the Atom feed** for a complete list of articles
2. **Use .md or .rst formats** for cleaner text extraction (less HTML noise)
3. **Rate limit requests** - be respectful, this is a personal blog
4. **Cache responses** - content doesn't change frequently
## Topics Covered
- Python programming & web frameworks (Pyramid, Flask, Django)
- DevOps & Infrastructure (Salt, Ansible, AWS, Terraform)
- Cloud computing & containerization
- System administration & Linux
- Software architecture & best practices
- Entrepreneurship & side projects
- Security & networking
## Contact
For questions about AI/agent access or API usage:
- Website: {site_url}/about/
- GitHub: https://github.com/russellballestrini
## Optional: Sitemap
A full sitemap may be available at: {site_url}/sitemap.xml
---
# Technical Details for Agents
## URL Patterns
```
Article: /{{slug}}/index.html (HTML)
/{{slug}}/index.rst (reStructuredText)
/{{slug}}/index.md (Markdown)
/{{slug}}/index.txt (Plain text)
Feeds: /feeds/all.atom.xml (Atom)
/feeds/all.json (JSON Feed)
/blog.json (JSON Blog - all posts)
Category: /category/{{category-slug}}/
Tag: /tags/{{tag-slug}}/
Author: /author/{{author-slug}}/
Uploads: /uploads/{{filename}}
```
## Sample Recent Articles
"""
for article in recent_articles[:10]:
content += f"- [{article}]({site_url}/{article}/)\n"
content += """
## Preferred Access Methods (by use case)
| Use Case | Best Format | URL |
|----------|-------------|-----|
| Get all posts (1 request) | JSON Blog | /blog.json |
| Subscribe/poll for updates | JSON Feed | /feeds/all.json |
| Read single article (LLM) | Plain text | /{slug}/index.txt |
| Read single article (human) | Markdown | /{slug}/index.md |
| Full rendered content | HTML | /{slug}/index.html |
"""
return content
if __name__ == '__main__':
if len(sys.argv) < 2:
print("Usage: generate_llms_txt.py <output_directory> [site_url]")
sys.exit(1)
output_dir = sys.argv[1]
site_url = sys.argv[2] if len(sys.argv) > 2 else 'https://russell.ballestrini.net'
content = generate_llms_txt(output_dir, site_url)
# Write to output directory
llms_path = Path(output_dir) / 'llms.txt'
llms_path.write_text(content)
print(f"Generated {llms_path}")

186
lib/generate_plaintext.py Normal file
View file

@ -0,0 +1,186 @@
#!/usr/bin/env python3
"""
Generate plain text versions of articles (index.txt) from HTML.
Strips all markup for clean LLM ingestion.
"""
import re
import sys
from html.parser import HTMLParser
from pathlib import Path
class HTMLToText(HTMLParser):
"""Convert HTML to plain text."""
def __init__(self):
super().__init__()
self.text = []
self.in_pre = False
self.in_code = False
self.skip_tags = {'script', 'style', 'head', 'meta', 'link'}
self.skip_depth = 0
def handle_starttag(self, tag, attrs):
if tag in self.skip_tags:
self.skip_depth += 1
elif tag == 'pre':
self.in_pre = True
self.text.append('\n```\n')
elif tag == 'code' and not self.in_pre:
self.in_code = True
self.text.append('`')
elif tag in ('p', 'div', 'article', 'section'):
self.text.append('\n\n')
elif tag in ('br',):
self.text.append('\n')
elif tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
level = int(tag[1])
self.text.append('\n\n' + '#' * level + ' ')
elif tag == 'li':
self.text.append('\n- ')
elif tag in ('ul', 'ol'):
self.text.append('\n')
elif tag == 'blockquote':
self.text.append('\n> ')
elif tag == 'a':
pass # Handle in data
elif tag == 'img':
for name, value in attrs:
if name == 'alt' and value:
self.text.append(f'[Image: {value}]')
break
def handle_endtag(self, tag):
if tag in self.skip_tags:
self.skip_depth -= 1
elif tag == 'pre':
self.in_pre = False
self.text.append('\n```\n')
elif tag == 'code' and not self.in_pre:
self.in_code = False
self.text.append('`')
elif tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
self.text.append('\n')
elif tag in ('ul', 'ol'):
self.text.append('\n')
def handle_data(self, data):
if self.skip_depth > 0:
return
if self.in_pre:
self.text.append(data)
else:
# Collapse whitespace in normal text
cleaned = re.sub(r'\s+', ' ', data)
self.text.append(cleaned)
def handle_entityref(self, name):
entities = {
'amp': '&', 'lt': '<', 'gt': '>', 'quot': '"',
'apos': "'", 'nbsp': ' ', 'mdash': '', 'ndash': '',
'ldquo': '"', 'rdquo': '"', 'lsquo': "'", 'rsquo': "'",
}
self.text.append(entities.get(name, f'&{name};'))
def handle_charref(self, name):
try:
if name.startswith('x'):
char = chr(int(name[1:], 16))
else:
char = chr(int(name))
self.text.append(char)
except (ValueError, OverflowError):
self.text.append(f'&#{name};')
def get_text(self):
text = ''.join(self.text)
# Clean up excessive newlines
text = re.sub(r'\n{3,}', '\n\n', text)
# Clean up spaces around newlines
text = re.sub(r' *\n *', '\n', text)
# Remove leading/trailing whitespace per line
lines = [line.strip() for line in text.split('\n')]
text = '\n'.join(lines)
return text.strip()
def html_to_text(html_content):
"""Convert HTML to plain text."""
parser = HTMLToText()
parser.feed(html_content)
return parser.get_text()
def extract_article_content(html_path):
"""Extract main article content from HTML file."""
content = html_path.read_text(encoding='utf-8')
# Try to extract just the article content
# Look for common article containers
patterns = [
r'<article[^>]*>(.*?)</article>',
r'<div[^>]*class="[^"]*entry-content[^"]*"[^>]*>(.*?)</div>',
r'<div[^>]*class="[^"]*post-content[^"]*"[^>]*>(.*?)</div>',
r'<div[^>]*class="[^"]*content[^"]*"[^>]*>(.*?)</div>',
]
for pattern in patterns:
match = re.search(pattern, content, re.DOTALL | re.IGNORECASE)
if match:
return html_to_text(match.group(1))
# Fallback: try to get body content
body_match = re.search(r'<body[^>]*>(.*?)</body>', content, re.DOTALL | re.IGNORECASE)
if body_match:
return html_to_text(body_match.group(1))
# Last resort: convert entire HTML
return html_to_text(content)
def convert_output_directory(output_dir):
"""Convert all index.html files to index.txt."""
output_path = Path(output_dir)
count = 0
for html_file in output_path.rglob('index.html'):
# Skip non-article directories
parent_name = html_file.parent.name
if parent_name in ('feeds', 'uploads', 'theme'):
continue
txt_file = html_file.with_suffix('.txt')
try:
text_content = extract_article_content(html_file)
if text_content.strip():
txt_file.write_text(text_content, encoding='utf-8')
count += 1
except Exception as e:
print(f"Warning: Failed to convert {html_file}: {e}")
return count
if __name__ == '__main__':
if len(sys.argv) < 2:
print("Usage: generate_plaintext.py <output_directory>")
print(" generate_plaintext.py <file.html> [output.txt]")
sys.exit(1)
path = Path(sys.argv[1])
if path.is_dir():
count = convert_output_directory(path)
print(f"Generated {count} plain text files")
elif path.is_file():
output = sys.argv[2] if len(sys.argv) > 2 else None
text = extract_article_content(path)
if output:
Path(output).write_text(text, encoding='utf-8')
print(f"Converted {path} -> {output}")
else:
print(text)
else:
print(f"Error: {path} not found")
sys.exit(1)

59
lib/rst2md.py Normal file
View file

@ -0,0 +1,59 @@
#!/usr/bin/env python3
"""
Convert RST files to Markdown using pandoc.
"""
import subprocess
import sys
from pathlib import Path
def convert_file(rst_path, md_path=None):
"""Convert a single RST file to Markdown using pandoc."""
rst_path = Path(rst_path)
if md_path is None:
md_path = rst_path.with_suffix('.md')
else:
md_path = Path(md_path)
subprocess.run(
['pandoc', '-f', 'rst', '-t', 'gfm', '-o', str(md_path), str(rst_path)],
check=True
)
return md_path
def convert_output_directory(output_dir):
"""Convert all index.rst files in output directory to index.md."""
output_path = Path(output_dir)
count = 0
for rst_file in output_path.rglob('index.rst'):
md_file = rst_file.with_suffix('.md')
try:
convert_file(rst_file, md_file)
count += 1
except subprocess.CalledProcessError as e:
print(f"Warning: Failed to convert {rst_file}: {e}")
return count
if __name__ == '__main__':
if len(sys.argv) < 2:
print("Usage: rst2md.py <output_directory>")
print(" rst2md.py <file.rst> [output.md]")
sys.exit(1)
path = Path(sys.argv[1])
if path.is_dir():
count = convert_output_directory(path)
print(f"Converted {count} RST files to Markdown")
elif path.is_file():
output = sys.argv[2] if len(sys.argv) > 2 else None
md_path = convert_file(path, output)
print(f"Converted {path} -> {md_path}")
else:
print(f"Error: {path} not found")
sys.exit(1)