Add lib/ format generation scripts for agent-friendly formats
- lib/rst2md.py - RST to Markdown converter using pandoc - lib/generate_jsonfeed.py - JSON Feed v1.1 generator - lib/generate_jsonblog.py - JSON Blog format (jsonblog.dev) - lib/generate_plaintext.py - HTML to plain text converter - lib/generate_llms_txt.py - AI agent onboarding file These scripts generate multiple formats for AI/LLM consumption: - blog.json at root (entire blog in one file) - feeds/all.json (JSON Feed format) - llms.txt (onboarding documentation) - index.md (Markdown versions) - index.txt (plain text versions)
This commit is contained in:
parent
6576bef263
commit
7c91647a55
5 changed files with 776 additions and 0 deletions
170
lib/generate_jsonblog.py
Normal file
170
lib/generate_jsonblog.py
Normal file
|
|
@ -0,0 +1,170 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Generate JSON Blog (jsonblog.dev) format - single JSON file with all posts.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
def strip_html(html):
|
||||||
|
"""Remove HTML tags and decode entities."""
|
||||||
|
if not html:
|
||||||
|
return ''
|
||||||
|
# Remove HTML tags
|
||||||
|
text = re.sub(r'<[^>]+>', '', html)
|
||||||
|
# Decode common HTML entities
|
||||||
|
text = text.replace('&', '&')
|
||||||
|
text = text.replace('<', '<')
|
||||||
|
text = text.replace('>', '>')
|
||||||
|
text = text.replace('"', '"')
|
||||||
|
text = text.replace(''', "'")
|
||||||
|
text = text.replace(' ', ' ')
|
||||||
|
# Collapse whitespace
|
||||||
|
text = re.sub(r'\s+', ' ', text).strip()
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
def parse_atom_feed(atom_path):
|
||||||
|
"""Parse Atom feed and extract entries."""
|
||||||
|
tree = ET.parse(atom_path)
|
||||||
|
root = tree.getroot()
|
||||||
|
ns = {'atom': 'http://www.w3.org/2005/Atom'}
|
||||||
|
|
||||||
|
feed_info = {
|
||||||
|
'title': root.find('atom:title', ns).text if root.find('atom:title', ns) is not None else '',
|
||||||
|
'subtitle': root.find('atom:subtitle', ns).text if root.find('atom:subtitle', ns) is not None else '',
|
||||||
|
'author': '',
|
||||||
|
}
|
||||||
|
|
||||||
|
author_elem = root.find('atom:author/atom:name', ns)
|
||||||
|
if author_elem is not None:
|
||||||
|
feed_info['author'] = author_elem.text
|
||||||
|
|
||||||
|
entries = []
|
||||||
|
for entry in root.findall('atom:entry', ns):
|
||||||
|
item = {
|
||||||
|
'title': '',
|
||||||
|
'slug': '',
|
||||||
|
'content': '',
|
||||||
|
'content_html': '',
|
||||||
|
'summary': '',
|
||||||
|
'createdAt': '',
|
||||||
|
'updatedAt': '',
|
||||||
|
'tags': [],
|
||||||
|
'url': '',
|
||||||
|
}
|
||||||
|
|
||||||
|
# URL and slug
|
||||||
|
for link in entry.findall('atom:link', ns):
|
||||||
|
if link.get('rel') == 'alternate':
|
||||||
|
item['url'] = link.get('href', '')
|
||||||
|
# Extract slug from URL
|
||||||
|
slug_match = re.search(r'/([^/]+)/?$', item['url'])
|
||||||
|
if slug_match:
|
||||||
|
item['slug'] = slug_match.group(1)
|
||||||
|
break
|
||||||
|
|
||||||
|
# Title
|
||||||
|
title_elem = entry.find('atom:title', ns)
|
||||||
|
if title_elem is not None:
|
||||||
|
item['title'] = title_elem.text
|
||||||
|
|
||||||
|
# Content
|
||||||
|
content_elem = entry.find('atom:content', ns)
|
||||||
|
if content_elem is not None:
|
||||||
|
item['content_html'] = content_elem.text or ''
|
||||||
|
item['content'] = strip_html(item['content_html'])
|
||||||
|
|
||||||
|
# Summary
|
||||||
|
summary_elem = entry.find('atom:summary', ns)
|
||||||
|
if summary_elem is not None:
|
||||||
|
item['summary'] = summary_elem.text or ''
|
||||||
|
|
||||||
|
# Dates (convert to YYYY-MM-DD for JSON Blog format)
|
||||||
|
published_elem = entry.find('atom:published', ns)
|
||||||
|
if published_elem is not None:
|
||||||
|
date_str = published_elem.text
|
||||||
|
if date_str:
|
||||||
|
item['createdAt'] = date_str[:10] # YYYY-MM-DD
|
||||||
|
|
||||||
|
updated_elem = entry.find('atom:updated', ns)
|
||||||
|
if updated_elem is not None:
|
||||||
|
date_str = updated_elem.text
|
||||||
|
if date_str:
|
||||||
|
item['updatedAt'] = date_str[:10]
|
||||||
|
|
||||||
|
# Tags
|
||||||
|
for category in entry.findall('atom:category', ns):
|
||||||
|
term = category.get('term')
|
||||||
|
if term:
|
||||||
|
item['tags'].append(term)
|
||||||
|
|
||||||
|
entries.append(item)
|
||||||
|
|
||||||
|
return feed_info, entries
|
||||||
|
|
||||||
|
|
||||||
|
def generate_jsonblog(atom_path, site_url):
|
||||||
|
"""Generate JSON Blog format."""
|
||||||
|
feed_info, entries = parse_atom_feed(atom_path)
|
||||||
|
|
||||||
|
jsonblog = {
|
||||||
|
'site': {
|
||||||
|
'title': feed_info['title'],
|
||||||
|
'description': feed_info['subtitle'],
|
||||||
|
'url': site_url,
|
||||||
|
},
|
||||||
|
'basics': {
|
||||||
|
'name': feed_info['author'] or 'Russell Ballestrini',
|
||||||
|
'url': site_url,
|
||||||
|
},
|
||||||
|
'posts': []
|
||||||
|
}
|
||||||
|
|
||||||
|
for entry in entries:
|
||||||
|
post = {
|
||||||
|
'title': entry['title'],
|
||||||
|
'slug': entry['slug'],
|
||||||
|
'content': entry['content'], # Plain text for JSON Blog
|
||||||
|
'createdAt': entry['createdAt'],
|
||||||
|
}
|
||||||
|
|
||||||
|
if entry['updatedAt'] and entry['updatedAt'] != entry['createdAt']:
|
||||||
|
post['updatedAt'] = entry['updatedAt']
|
||||||
|
|
||||||
|
if entry['tags']:
|
||||||
|
post['tags'] = entry['tags']
|
||||||
|
|
||||||
|
if entry['summary']:
|
||||||
|
post['summary'] = entry['summary']
|
||||||
|
|
||||||
|
if entry['url']:
|
||||||
|
post['url'] = entry['url']
|
||||||
|
|
||||||
|
jsonblog['posts'].append(post)
|
||||||
|
|
||||||
|
return jsonblog
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
if len(sys.argv) < 2:
|
||||||
|
print("Usage: generate_jsonblog.py <output_directory> [site_url]")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
output_dir = Path(sys.argv[1])
|
||||||
|
site_url = sys.argv[2] if len(sys.argv) > 2 else 'https://russell.ballestrini.net'
|
||||||
|
|
||||||
|
atom_path = output_dir / 'feeds' / 'all.atom.xml'
|
||||||
|
if not atom_path.exists():
|
||||||
|
print(f"Error: Atom feed not found at {atom_path}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
jsonblog = generate_jsonblog(atom_path, site_url)
|
||||||
|
|
||||||
|
output_path = output_dir / 'blog.json'
|
||||||
|
output_path.write_text(json.dumps(jsonblog, indent=2, ensure_ascii=False))
|
||||||
|
print(f"Generated {output_path} ({len(jsonblog['posts'])} posts)")
|
||||||
162
lib/generate_jsonfeed.py
Normal file
162
lib/generate_jsonfeed.py
Normal file
|
|
@ -0,0 +1,162 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Generate JSON Feed (jsonfeed.org) from Pelican Atom feed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def parse_atom_feed(atom_path):
|
||||||
|
"""Parse Atom feed and extract entries."""
|
||||||
|
tree = ET.parse(atom_path)
|
||||||
|
root = tree.getroot()
|
||||||
|
|
||||||
|
# Atom namespace
|
||||||
|
ns = {'atom': 'http://www.w3.org/2005/Atom'}
|
||||||
|
|
||||||
|
feed_info = {
|
||||||
|
'title': root.find('atom:title', ns).text if root.find('atom:title', ns) is not None else '',
|
||||||
|
'subtitle': root.find('atom:subtitle', ns).text if root.find('atom:subtitle', ns) is not None else '',
|
||||||
|
'link': '',
|
||||||
|
'author': '',
|
||||||
|
}
|
||||||
|
|
||||||
|
# Get feed link
|
||||||
|
for link in root.findall('atom:link', ns):
|
||||||
|
if link.get('rel') == 'alternate':
|
||||||
|
feed_info['link'] = link.get('href', '')
|
||||||
|
break
|
||||||
|
|
||||||
|
# Get author
|
||||||
|
author_elem = root.find('atom:author/atom:name', ns)
|
||||||
|
if author_elem is not None:
|
||||||
|
feed_info['author'] = author_elem.text
|
||||||
|
|
||||||
|
entries = []
|
||||||
|
for entry in root.findall('atom:entry', ns):
|
||||||
|
item = {
|
||||||
|
'id': '',
|
||||||
|
'url': '',
|
||||||
|
'title': '',
|
||||||
|
'content_html': '',
|
||||||
|
'summary': '',
|
||||||
|
'date_published': '',
|
||||||
|
'date_modified': '',
|
||||||
|
'tags': [],
|
||||||
|
}
|
||||||
|
|
||||||
|
# ID
|
||||||
|
id_elem = entry.find('atom:id', ns)
|
||||||
|
if id_elem is not None:
|
||||||
|
item['id'] = id_elem.text
|
||||||
|
|
||||||
|
# URL
|
||||||
|
for link in entry.findall('atom:link', ns):
|
||||||
|
if link.get('rel') == 'alternate':
|
||||||
|
item['url'] = link.get('href', '')
|
||||||
|
break
|
||||||
|
|
||||||
|
# Title
|
||||||
|
title_elem = entry.find('atom:title', ns)
|
||||||
|
if title_elem is not None:
|
||||||
|
item['title'] = title_elem.text
|
||||||
|
|
||||||
|
# Content
|
||||||
|
content_elem = entry.find('atom:content', ns)
|
||||||
|
if content_elem is not None:
|
||||||
|
item['content_html'] = content_elem.text or ''
|
||||||
|
|
||||||
|
# Summary
|
||||||
|
summary_elem = entry.find('atom:summary', ns)
|
||||||
|
if summary_elem is not None:
|
||||||
|
item['summary'] = summary_elem.text or ''
|
||||||
|
|
||||||
|
# Dates
|
||||||
|
published_elem = entry.find('atom:published', ns)
|
||||||
|
if published_elem is not None:
|
||||||
|
item['date_published'] = published_elem.text
|
||||||
|
|
||||||
|
updated_elem = entry.find('atom:updated', ns)
|
||||||
|
if updated_elem is not None:
|
||||||
|
item['date_modified'] = updated_elem.text
|
||||||
|
|
||||||
|
# Tags/categories
|
||||||
|
for category in entry.findall('atom:category', ns):
|
||||||
|
term = category.get('term')
|
||||||
|
if term:
|
||||||
|
item['tags'].append(term)
|
||||||
|
|
||||||
|
entries.append(item)
|
||||||
|
|
||||||
|
return feed_info, entries
|
||||||
|
|
||||||
|
|
||||||
|
def generate_jsonfeed(atom_path, site_url):
|
||||||
|
"""Generate JSON Feed from Atom feed."""
|
||||||
|
feed_info, entries = parse_atom_feed(atom_path)
|
||||||
|
|
||||||
|
jsonfeed = {
|
||||||
|
'version': 'https://jsonfeed.org/version/1.1',
|
||||||
|
'title': feed_info['title'],
|
||||||
|
'home_page_url': site_url,
|
||||||
|
'feed_url': f'{site_url}/feeds/all.json',
|
||||||
|
'description': feed_info['subtitle'],
|
||||||
|
'language': 'en-US',
|
||||||
|
'authors': [
|
||||||
|
{
|
||||||
|
'name': feed_info['author'] or 'Russell Ballestrini',
|
||||||
|
'url': site_url,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
'items': []
|
||||||
|
}
|
||||||
|
|
||||||
|
for entry in entries:
|
||||||
|
item = {
|
||||||
|
'id': entry['id'],
|
||||||
|
'url': entry['url'],
|
||||||
|
'title': entry['title'],
|
||||||
|
}
|
||||||
|
|
||||||
|
if entry['content_html']:
|
||||||
|
item['content_html'] = entry['content_html']
|
||||||
|
|
||||||
|
if entry['summary']:
|
||||||
|
item['summary'] = entry['summary']
|
||||||
|
|
||||||
|
if entry['date_published']:
|
||||||
|
item['date_published'] = entry['date_published']
|
||||||
|
|
||||||
|
if entry['date_modified']:
|
||||||
|
item['date_modified'] = entry['date_modified']
|
||||||
|
|
||||||
|
if entry['tags']:
|
||||||
|
item['tags'] = entry['tags']
|
||||||
|
|
||||||
|
jsonfeed['items'].append(item)
|
||||||
|
|
||||||
|
return jsonfeed
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
if len(sys.argv) < 2:
|
||||||
|
print("Usage: generate_jsonfeed.py <output_directory> [site_url]")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
output_dir = Path(sys.argv[1])
|
||||||
|
site_url = sys.argv[2] if len(sys.argv) > 2 else 'https://russell.ballestrini.net'
|
||||||
|
|
||||||
|
atom_path = output_dir / 'feeds' / 'all.atom.xml'
|
||||||
|
if not atom_path.exists():
|
||||||
|
print(f"Error: Atom feed not found at {atom_path}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
jsonfeed = generate_jsonfeed(atom_path, site_url)
|
||||||
|
|
||||||
|
output_path = output_dir / 'feeds' / 'all.json'
|
||||||
|
output_path.write_text(json.dumps(jsonfeed, indent=2, ensure_ascii=False))
|
||||||
|
print(f"Generated {output_path}")
|
||||||
199
lib/generate_llms_txt.py
Normal file
199
lib/generate_llms_txt.py
Normal file
|
|
@ -0,0 +1,199 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Generate llms.txt for AI agent onboarding.
|
||||||
|
Provides structured information about the site for LLM crawlers & agents.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def get_tags_and_categories(output_dir):
|
||||||
|
"""Extract tags & categories from output directory structure."""
|
||||||
|
output_path = Path(output_dir)
|
||||||
|
|
||||||
|
tags = []
|
||||||
|
tags_dir = output_path / 'tags'
|
||||||
|
if tags_dir.exists():
|
||||||
|
tags = sorted([d.name for d in tags_dir.iterdir() if d.is_dir()])
|
||||||
|
|
||||||
|
categories = []
|
||||||
|
cat_dir = output_path / 'category'
|
||||||
|
if cat_dir.exists():
|
||||||
|
categories = sorted([d.name for d in cat_dir.iterdir() if d.is_dir()])
|
||||||
|
|
||||||
|
return tags, categories
|
||||||
|
|
||||||
|
|
||||||
|
def get_recent_articles(output_dir, limit=20):
|
||||||
|
"""Get list of recent articles from output directory."""
|
||||||
|
output_path = Path(output_dir)
|
||||||
|
articles = []
|
||||||
|
|
||||||
|
# Find all index.html files that look like articles (have dates in parent dir name)
|
||||||
|
for index_file in output_path.glob('*/index.html'):
|
||||||
|
dirname = index_file.parent.name
|
||||||
|
# Skip non-article directories
|
||||||
|
if dirname in ('tags', 'category', 'author', 'uploads', 'feeds', 'about', 'archives'):
|
||||||
|
continue
|
||||||
|
if dirname.startswith('20'): # Date-prefixed directories (cover letters, etc.)
|
||||||
|
continue
|
||||||
|
articles.append(dirname)
|
||||||
|
|
||||||
|
return sorted(articles)[:limit]
|
||||||
|
|
||||||
|
|
||||||
|
def generate_llms_txt(output_dir, site_url='https://russell.ballestrini.net'):
|
||||||
|
"""Generate llms.txt content."""
|
||||||
|
tags, categories = get_tags_and_categories(output_dir)
|
||||||
|
recent_articles = get_recent_articles(output_dir)
|
||||||
|
|
||||||
|
content = f"""# llms.txt - AI Agent Onboarding for russell.ballestrini.net
|
||||||
|
|
||||||
|
> Russell Ballestrini's technical blog covering software engineering, DevOps,
|
||||||
|
> infrastructure, Python, cloud computing, and entrepreneurship.
|
||||||
|
|
||||||
|
## Site Information
|
||||||
|
|
||||||
|
- **Author**: Russell Ballestrini
|
||||||
|
- **Site URL**: {site_url}
|
||||||
|
- **Generated**: {datetime.now().isoformat()}
|
||||||
|
|
||||||
|
## Feeds (Recommended for Crawling)
|
||||||
|
|
||||||
|
### Atom Feed
|
||||||
|
- URL: {site_url}/feeds/all.atom.xml
|
||||||
|
- Format: Atom XML
|
||||||
|
- Description: Complete feed of all blog posts, newest first
|
||||||
|
|
||||||
|
### JSON Feed (jsonfeed.org v1.1)
|
||||||
|
- URL: {site_url}/feeds/all.json
|
||||||
|
- Format: JSON Feed 1.1
|
||||||
|
- Description: Same as Atom feed, but JSON for easier parsing
|
||||||
|
|
||||||
|
### JSON Blog (jsonblog.dev)
|
||||||
|
- URL: {site_url}/blog.json
|
||||||
|
- Format: JSON Blog
|
||||||
|
- Description: **Entire blog in one file** - all posts with plain text content
|
||||||
|
|
||||||
|
## Content Formats
|
||||||
|
|
||||||
|
Each article is available in multiple formats:
|
||||||
|
- **HTML**: `/{{slug}}/index.html` - Rendered page with styling
|
||||||
|
- **RST**: `/{{slug}}/index.rst` - reStructuredText source
|
||||||
|
- **Markdown**: `/{{slug}}/index.md` - GitHub-flavored Markdown
|
||||||
|
- **Plain Text**: `/{{slug}}/index.txt` - Raw text, no markup (best for LLMs)
|
||||||
|
|
||||||
|
## API-Friendly Endpoints
|
||||||
|
|
||||||
|
### Resume (JSONResume Format)
|
||||||
|
- URL: {site_url}/uploads/russell.ballestrini.resume.json
|
||||||
|
- Format: JSONResume standard schema
|
||||||
|
- Description: Structured resume data in machine-readable format
|
||||||
|
|
||||||
|
## Navigation Structure
|
||||||
|
|
||||||
|
### By Category
|
||||||
|
"""
|
||||||
|
|
||||||
|
if categories:
|
||||||
|
for cat in categories[:15]:
|
||||||
|
content += f"- [{cat}]({site_url}/category/{cat}/)\n"
|
||||||
|
|
||||||
|
content += f"""
|
||||||
|
### By Tag
|
||||||
|
"""
|
||||||
|
|
||||||
|
if tags:
|
||||||
|
for tag in tags[:30]:
|
||||||
|
content += f"- [{tag}]({site_url}/tags/{tag}/)\n"
|
||||||
|
|
||||||
|
content += f"""
|
||||||
|
## Crawling Recommendations
|
||||||
|
|
||||||
|
1. **Start with the Atom feed** for a complete list of articles
|
||||||
|
2. **Use .md or .rst formats** for cleaner text extraction (less HTML noise)
|
||||||
|
3. **Rate limit requests** - be respectful, this is a personal blog
|
||||||
|
4. **Cache responses** - content doesn't change frequently
|
||||||
|
|
||||||
|
## Topics Covered
|
||||||
|
|
||||||
|
- Python programming & web frameworks (Pyramid, Flask, Django)
|
||||||
|
- DevOps & Infrastructure (Salt, Ansible, AWS, Terraform)
|
||||||
|
- Cloud computing & containerization
|
||||||
|
- System administration & Linux
|
||||||
|
- Software architecture & best practices
|
||||||
|
- Entrepreneurship & side projects
|
||||||
|
- Security & networking
|
||||||
|
|
||||||
|
## Contact
|
||||||
|
|
||||||
|
For questions about AI/agent access or API usage:
|
||||||
|
- Website: {site_url}/about/
|
||||||
|
- GitHub: https://github.com/russellballestrini
|
||||||
|
|
||||||
|
## Optional: Sitemap
|
||||||
|
|
||||||
|
A full sitemap may be available at: {site_url}/sitemap.xml
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Technical Details for Agents
|
||||||
|
|
||||||
|
## URL Patterns
|
||||||
|
|
||||||
|
```
|
||||||
|
Article: /{{slug}}/index.html (HTML)
|
||||||
|
/{{slug}}/index.rst (reStructuredText)
|
||||||
|
/{{slug}}/index.md (Markdown)
|
||||||
|
/{{slug}}/index.txt (Plain text)
|
||||||
|
|
||||||
|
Feeds: /feeds/all.atom.xml (Atom)
|
||||||
|
/feeds/all.json (JSON Feed)
|
||||||
|
/blog.json (JSON Blog - all posts)
|
||||||
|
|
||||||
|
Category: /category/{{category-slug}}/
|
||||||
|
Tag: /tags/{{tag-slug}}/
|
||||||
|
Author: /author/{{author-slug}}/
|
||||||
|
Uploads: /uploads/{{filename}}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Sample Recent Articles
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
for article in recent_articles[:10]:
|
||||||
|
content += f"- [{article}]({site_url}/{article}/)\n"
|
||||||
|
|
||||||
|
content += """
|
||||||
|
## Preferred Access Methods (by use case)
|
||||||
|
|
||||||
|
| Use Case | Best Format | URL |
|
||||||
|
|----------|-------------|-----|
|
||||||
|
| Get all posts (1 request) | JSON Blog | /blog.json |
|
||||||
|
| Subscribe/poll for updates | JSON Feed | /feeds/all.json |
|
||||||
|
| Read single article (LLM) | Plain text | /{slug}/index.txt |
|
||||||
|
| Read single article (human) | Markdown | /{slug}/index.md |
|
||||||
|
| Full rendered content | HTML | /{slug}/index.html |
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
return content
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
if len(sys.argv) < 2:
|
||||||
|
print("Usage: generate_llms_txt.py <output_directory> [site_url]")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
output_dir = sys.argv[1]
|
||||||
|
site_url = sys.argv[2] if len(sys.argv) > 2 else 'https://russell.ballestrini.net'
|
||||||
|
|
||||||
|
content = generate_llms_txt(output_dir, site_url)
|
||||||
|
|
||||||
|
# Write to output directory
|
||||||
|
llms_path = Path(output_dir) / 'llms.txt'
|
||||||
|
llms_path.write_text(content)
|
||||||
|
print(f"Generated {llms_path}")
|
||||||
186
lib/generate_plaintext.py
Normal file
186
lib/generate_plaintext.py
Normal file
|
|
@ -0,0 +1,186 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Generate plain text versions of articles (index.txt) from HTML.
|
||||||
|
Strips all markup for clean LLM ingestion.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
class HTMLToText(HTMLParser):
|
||||||
|
"""Convert HTML to plain text."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.text = []
|
||||||
|
self.in_pre = False
|
||||||
|
self.in_code = False
|
||||||
|
self.skip_tags = {'script', 'style', 'head', 'meta', 'link'}
|
||||||
|
self.skip_depth = 0
|
||||||
|
|
||||||
|
def handle_starttag(self, tag, attrs):
|
||||||
|
if tag in self.skip_tags:
|
||||||
|
self.skip_depth += 1
|
||||||
|
elif tag == 'pre':
|
||||||
|
self.in_pre = True
|
||||||
|
self.text.append('\n```\n')
|
||||||
|
elif tag == 'code' and not self.in_pre:
|
||||||
|
self.in_code = True
|
||||||
|
self.text.append('`')
|
||||||
|
elif tag in ('p', 'div', 'article', 'section'):
|
||||||
|
self.text.append('\n\n')
|
||||||
|
elif tag in ('br',):
|
||||||
|
self.text.append('\n')
|
||||||
|
elif tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
|
||||||
|
level = int(tag[1])
|
||||||
|
self.text.append('\n\n' + '#' * level + ' ')
|
||||||
|
elif tag == 'li':
|
||||||
|
self.text.append('\n- ')
|
||||||
|
elif tag in ('ul', 'ol'):
|
||||||
|
self.text.append('\n')
|
||||||
|
elif tag == 'blockquote':
|
||||||
|
self.text.append('\n> ')
|
||||||
|
elif tag == 'a':
|
||||||
|
pass # Handle in data
|
||||||
|
elif tag == 'img':
|
||||||
|
for name, value in attrs:
|
||||||
|
if name == 'alt' and value:
|
||||||
|
self.text.append(f'[Image: {value}]')
|
||||||
|
break
|
||||||
|
|
||||||
|
def handle_endtag(self, tag):
|
||||||
|
if tag in self.skip_tags:
|
||||||
|
self.skip_depth -= 1
|
||||||
|
elif tag == 'pre':
|
||||||
|
self.in_pre = False
|
||||||
|
self.text.append('\n```\n')
|
||||||
|
elif tag == 'code' and not self.in_pre:
|
||||||
|
self.in_code = False
|
||||||
|
self.text.append('`')
|
||||||
|
elif tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
|
||||||
|
self.text.append('\n')
|
||||||
|
elif tag in ('ul', 'ol'):
|
||||||
|
self.text.append('\n')
|
||||||
|
|
||||||
|
def handle_data(self, data):
|
||||||
|
if self.skip_depth > 0:
|
||||||
|
return
|
||||||
|
if self.in_pre:
|
||||||
|
self.text.append(data)
|
||||||
|
else:
|
||||||
|
# Collapse whitespace in normal text
|
||||||
|
cleaned = re.sub(r'\s+', ' ', data)
|
||||||
|
self.text.append(cleaned)
|
||||||
|
|
||||||
|
def handle_entityref(self, name):
|
||||||
|
entities = {
|
||||||
|
'amp': '&', 'lt': '<', 'gt': '>', 'quot': '"',
|
||||||
|
'apos': "'", 'nbsp': ' ', 'mdash': '—', 'ndash': '–',
|
||||||
|
'ldquo': '"', 'rdquo': '"', 'lsquo': "'", 'rsquo': "'",
|
||||||
|
}
|
||||||
|
self.text.append(entities.get(name, f'&{name};'))
|
||||||
|
|
||||||
|
def handle_charref(self, name):
|
||||||
|
try:
|
||||||
|
if name.startswith('x'):
|
||||||
|
char = chr(int(name[1:], 16))
|
||||||
|
else:
|
||||||
|
char = chr(int(name))
|
||||||
|
self.text.append(char)
|
||||||
|
except (ValueError, OverflowError):
|
||||||
|
self.text.append(f'&#{name};')
|
||||||
|
|
||||||
|
def get_text(self):
|
||||||
|
text = ''.join(self.text)
|
||||||
|
# Clean up excessive newlines
|
||||||
|
text = re.sub(r'\n{3,}', '\n\n', text)
|
||||||
|
# Clean up spaces around newlines
|
||||||
|
text = re.sub(r' *\n *', '\n', text)
|
||||||
|
# Remove leading/trailing whitespace per line
|
||||||
|
lines = [line.strip() for line in text.split('\n')]
|
||||||
|
text = '\n'.join(lines)
|
||||||
|
return text.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def html_to_text(html_content):
|
||||||
|
"""Convert HTML to plain text."""
|
||||||
|
parser = HTMLToText()
|
||||||
|
parser.feed(html_content)
|
||||||
|
return parser.get_text()
|
||||||
|
|
||||||
|
|
||||||
|
def extract_article_content(html_path):
|
||||||
|
"""Extract main article content from HTML file."""
|
||||||
|
content = html_path.read_text(encoding='utf-8')
|
||||||
|
|
||||||
|
# Try to extract just the article content
|
||||||
|
# Look for common article containers
|
||||||
|
patterns = [
|
||||||
|
r'<article[^>]*>(.*?)</article>',
|
||||||
|
r'<div[^>]*class="[^"]*entry-content[^"]*"[^>]*>(.*?)</div>',
|
||||||
|
r'<div[^>]*class="[^"]*post-content[^"]*"[^>]*>(.*?)</div>',
|
||||||
|
r'<div[^>]*class="[^"]*content[^"]*"[^>]*>(.*?)</div>',
|
||||||
|
]
|
||||||
|
|
||||||
|
for pattern in patterns:
|
||||||
|
match = re.search(pattern, content, re.DOTALL | re.IGNORECASE)
|
||||||
|
if match:
|
||||||
|
return html_to_text(match.group(1))
|
||||||
|
|
||||||
|
# Fallback: try to get body content
|
||||||
|
body_match = re.search(r'<body[^>]*>(.*?)</body>', content, re.DOTALL | re.IGNORECASE)
|
||||||
|
if body_match:
|
||||||
|
return html_to_text(body_match.group(1))
|
||||||
|
|
||||||
|
# Last resort: convert entire HTML
|
||||||
|
return html_to_text(content)
|
||||||
|
|
||||||
|
|
||||||
|
def convert_output_directory(output_dir):
|
||||||
|
"""Convert all index.html files to index.txt."""
|
||||||
|
output_path = Path(output_dir)
|
||||||
|
count = 0
|
||||||
|
|
||||||
|
for html_file in output_path.rglob('index.html'):
|
||||||
|
# Skip non-article directories
|
||||||
|
parent_name = html_file.parent.name
|
||||||
|
if parent_name in ('feeds', 'uploads', 'theme'):
|
||||||
|
continue
|
||||||
|
|
||||||
|
txt_file = html_file.with_suffix('.txt')
|
||||||
|
try:
|
||||||
|
text_content = extract_article_content(html_file)
|
||||||
|
if text_content.strip():
|
||||||
|
txt_file.write_text(text_content, encoding='utf-8')
|
||||||
|
count += 1
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Warning: Failed to convert {html_file}: {e}")
|
||||||
|
|
||||||
|
return count
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
if len(sys.argv) < 2:
|
||||||
|
print("Usage: generate_plaintext.py <output_directory>")
|
||||||
|
print(" generate_plaintext.py <file.html> [output.txt]")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
path = Path(sys.argv[1])
|
||||||
|
|
||||||
|
if path.is_dir():
|
||||||
|
count = convert_output_directory(path)
|
||||||
|
print(f"Generated {count} plain text files")
|
||||||
|
elif path.is_file():
|
||||||
|
output = sys.argv[2] if len(sys.argv) > 2 else None
|
||||||
|
text = extract_article_content(path)
|
||||||
|
if output:
|
||||||
|
Path(output).write_text(text, encoding='utf-8')
|
||||||
|
print(f"Converted {path} -> {output}")
|
||||||
|
else:
|
||||||
|
print(text)
|
||||||
|
else:
|
||||||
|
print(f"Error: {path} not found")
|
||||||
|
sys.exit(1)
|
||||||
59
lib/rst2md.py
Normal file
59
lib/rst2md.py
Normal file
|
|
@ -0,0 +1,59 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Convert RST files to Markdown using pandoc.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def convert_file(rst_path, md_path=None):
|
||||||
|
"""Convert a single RST file to Markdown using pandoc."""
|
||||||
|
rst_path = Path(rst_path)
|
||||||
|
if md_path is None:
|
||||||
|
md_path = rst_path.with_suffix('.md')
|
||||||
|
else:
|
||||||
|
md_path = Path(md_path)
|
||||||
|
|
||||||
|
subprocess.run(
|
||||||
|
['pandoc', '-f', 'rst', '-t', 'gfm', '-o', str(md_path), str(rst_path)],
|
||||||
|
check=True
|
||||||
|
)
|
||||||
|
return md_path
|
||||||
|
|
||||||
|
|
||||||
|
def convert_output_directory(output_dir):
|
||||||
|
"""Convert all index.rst files in output directory to index.md."""
|
||||||
|
output_path = Path(output_dir)
|
||||||
|
count = 0
|
||||||
|
|
||||||
|
for rst_file in output_path.rglob('index.rst'):
|
||||||
|
md_file = rst_file.with_suffix('.md')
|
||||||
|
try:
|
||||||
|
convert_file(rst_file, md_file)
|
||||||
|
count += 1
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
print(f"Warning: Failed to convert {rst_file}: {e}")
|
||||||
|
|
||||||
|
return count
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
if len(sys.argv) < 2:
|
||||||
|
print("Usage: rst2md.py <output_directory>")
|
||||||
|
print(" rst2md.py <file.rst> [output.md]")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
path = Path(sys.argv[1])
|
||||||
|
|
||||||
|
if path.is_dir():
|
||||||
|
count = convert_output_directory(path)
|
||||||
|
print(f"Converted {count} RST files to Markdown")
|
||||||
|
elif path.is_file():
|
||||||
|
output = sys.argv[2] if len(sys.argv) > 2 else None
|
||||||
|
md_path = convert_file(path, output)
|
||||||
|
print(f"Converted {path} -> {md_path}")
|
||||||
|
else:
|
||||||
|
print(f"Error: {path} not found")
|
||||||
|
sys.exit(1)
|
||||||
Loading…
Add table
Add a link
Reference in a new issue