russell.ballestrini.net/lib/generate_llms_txt.py

#!/usr/bin/env python3
"""
Generate llms.txt for AI agent onboarding.
Provides structured information about the site for LLM crawlers & agents.
"""

import json
import sys
from datetime import datetime
from pathlib import Path


def get_tags_and_categories(output_dir):
    """Extract tags & categories from output directory structure."""
    output_path = Path(output_dir)

    tags = []
    tags_dir = output_path / 'tags'
    if tags_dir.exists():
        tags = sorted([d.name for d in tags_dir.iterdir() if d.is_dir()])

    categories = []
    cat_dir = output_path / 'category'
    if cat_dir.exists():
        categories = sorted([d.name for d in cat_dir.iterdir() if d.is_dir()])

    return tags, categories


def get_recent_articles(output_dir, limit=20):
    """Get list of recent articles from output directory."""
    output_path = Path(output_dir)
    articles = []

    # Find all index.html files that look like articles (have dates in parent dir name)
    for index_file in output_path.glob('*/index.html'):
        dirname = index_file.parent.name
        # Skip non-article directories
        if dirname in ('tags', 'category', 'author', 'uploads', 'feeds', 'about', 'archives'):
            continue
        if dirname.startswith('20'):  # Date-prefixed directories (cover letters, etc.)
            continue
        articles.append(dirname)

    return sorted(articles)[:limit]


def generate_llms_txt(output_dir, site_url='https://russell.ballestrini.net'):
    """Generate llms.txt content."""
    tags, categories = get_tags_and_categories(output_dir)
    recent_articles = get_recent_articles(output_dir)

    content = f"""# llms.txt - AI Agent Onboarding for russell.ballestrini.net

> Russell Ballestrini's technical blog covering software engineering, DevOps,
> infrastructure, Python, cloud computing, and entrepreneurship.

## Site Information

- **Author**: Russell Ballestrini
- **Site URL**: {site_url}
- **Generated**: {datetime.now().isoformat()}

## Feeds (Recommended for Crawling)

### Atom Feed
- URL: {site_url}/feeds/all.atom.xml
- Format: Atom XML
- Description: Complete feed of all blog posts, newest first

### JSON Feed (jsonfeed.org v1.1)
- URL: {site_url}/feeds/all.json
- Format: JSON Feed 1.1
- Description: Same as Atom feed, but JSON for easier parsing

### JSON Blog (jsonblog.dev)
- URL: {site_url}/blog.json
- Format: JSON Blog
- Description: **Entire blog in one file** - all posts with plain text content

## Content Formats

Each article is available in multiple formats:
- **HTML**: `/{{slug}}/index.html` - Rendered page with styling
- **RST**: `/{{slug}}/index.rst` - reStructuredText source
- **Markdown**: `/{{slug}}/index.md` - GitHub-flavored Markdown
- **Plain Text**: `/{{slug}}/index.txt` - Raw text, no markup (best for LLMs)

## API-Friendly Endpoints

### Resume (JSONResume Format)
- URL: {site_url}/uploads/russell.ballestrini.resume.json
- Format: JSONResume standard schema
- Description: Structured resume data in machine-readable format

## Navigation Structure

### By Category
"""

    if categories:
        for cat in categories[:15]:
            content += f"- [{cat}]({site_url}/category/{cat}/)\n"

    content += f"""
### By Tag
"""

    if tags:
        for tag in tags[:30]:
            content += f"- [{tag}]({site_url}/tags/{tag}/)\n"

    content += f"""
## Crawling Recommendations

1. **Start with the Atom feed** for a complete list of articles
2. **Use .md or .rst formats** for cleaner text extraction (less HTML noise)
3. **Rate limit requests** - be respectful, this is a personal blog
4. **Cache responses** - content doesn't change frequently

## Topics Covered

- Python programming & web frameworks (Pyramid, Flask, Django)
- DevOps & Infrastructure (Salt, Ansible, AWS, Terraform)
- Cloud computing & containerization
- System administration & Linux
- Software architecture & best practices
- Entrepreneurship & side projects
- Security & networking

## Contact

For questions about AI/agent access or API usage:
- Website: {site_url}/about/
- GitHub: https://github.com/russellballestrini

## Optional: Sitemap

A full sitemap may be available at: {site_url}/sitemap.xml

---

# Technical Details for Agents

## URL Patterns

```
Article:     /{{slug}}/index.html   (HTML)
             /{{slug}}/index.rst    (reStructuredText)
             /{{slug}}/index.md     (Markdown)
             /{{slug}}/index.txt    (Plain text)

Feeds:       /feeds/all.atom.xml    (Atom)
             /feeds/all.json        (JSON Feed)
             /blog.json             (JSON Blog - all posts)

Category:    /category/{{category-slug}}/
Tag:         /tags/{{tag-slug}}/
Author:      /author/{{author-slug}}/
Uploads:     /uploads/{{filename}}
```

## Sample Recent Articles

"""

    for article in recent_articles[:10]:
        content += f"- [{article}]({site_url}/{article}/)\n"

    content += """
## Preferred Access Methods (by use case)

| Use Case | Best Format | URL |
|----------|-------------|-----|
| Get all posts (1 request) | JSON Blog | /blog.json |
| Subscribe/poll for updates | JSON Feed | /feeds/all.json |
| Read single article (LLM) | Plain text | /{slug}/index.txt |
| Read single article (human) | Markdown | /{slug}/index.md |
| Full rendered content | HTML | /{slug}/index.html |

"""

    return content


if __name__ == '__main__':
    if len(sys.argv) < 2:
        print("Usage: generate_llms_txt.py <output_directory> [site_url]")
        sys.exit(1)

    output_dir = sys.argv[1]
    site_url = sys.argv[2] if len(sys.argv) > 2 else 'https://russell.ballestrini.net'

    content = generate_llms_txt(output_dir, site_url)

    # Write to output directory
    llms_path = Path(output_dir) / 'llms.txt'
    llms_path.write_text(content)
    print(f"Generated {llms_path}")