russell.ballestrini.net/lib/generate_llms_txt.py

#!/usr/bin/env python3
"""
Generate llms.txt for AI agent onboarding.
Provides structured information about the site for LLM crawlers & agents.
"""

import json
import sys
from datetime import datetime
from pathlib import Path


def get_tags_and_categories(output_dir):
    """Extract tags & categories from output directory structure."""
    output_path = Path(output_dir)

    tags = []
    tags_dir = output_path / 'tags'
    if tags_dir.exists():
        tags = sorted([d.name for d in tags_dir.iterdir() if d.is_dir()])

    categories = []
    cat_dir = output_path / 'category'
    if cat_dir.exists():
        categories = sorted([d.name for d in cat_dir.iterdir() if d.is_dir()])

    return tags, categories


def get_recent_articles(output_dir, limit=20):
    """Get list of recent articles from blog.json (sorted by date)."""
    output_path = Path(output_dir)
    blog_json = output_path / 'blog.json'

    if blog_json.exists():
        try:
            data = json.loads(blog_json.read_text())
            posts = data.get('posts', [])
            # Already sorted by date in blog.json, just get slugs
            return [post['slug'] for post in posts[:limit]]
        except (json.JSONDecodeError, KeyError):
            pass

    # Fallback: sort by file modification time
    output_path = Path(output_dir)
    articles = []

    for index_file in output_path.glob('*/index.html'):
        dirname = index_file.parent.name
        if dirname in ('tags', 'category', 'author', 'uploads', 'feeds', 'about', 'archives'):
            continue
        if dirname.startswith('20'):
            continue
        mtime = index_file.stat().st_mtime
        articles.append((mtime, dirname))

    # Sort by modification time, newest first
    articles.sort(reverse=True)
    return [name for _, name in articles[:limit]]


def generate_llms_txt(output_dir, site_url='https://russell.ballestrini.net'):
    """Generate llms.txt content."""
    tags, categories = get_tags_and_categories(output_dir)
    recent_articles = get_recent_articles(output_dir)

    content = f"""# llms.txt - AI Agent Onboarding for russell.ballestrini.net

> Russell Ballestrini's technical blog covering software engineering, DevOps,
> infrastructure, Python, cloud computing, and entrepreneurship.

## Site Information

- **Author**: Russell Ballestrini
- **Site URL**: {site_url}
- **Generated**: {datetime.now().isoformat()}

## Feeds (Recommended for Crawling)

### Atom Feed
- URL: {site_url}/feeds/all.atom.xml
- Format: Atom XML
- Description: Complete feed of all blog posts, newest first

### JSON Feed (jsonfeed.org v1.1)
- URL: {site_url}/feeds/all.json
- Format: JSON Feed 1.1
- Description: Same as Atom feed, but JSON for easier parsing

### JSON Blog (jsonblog.dev)
- URL: {site_url}/blog.json
- Format: JSON Blog
- Description: **Entire blog in one file** - all posts with content_url links to index-slim.md (Markdown)

## Content Formats

Each article is available in multiple formats:
- **HTML**: `/{{slug}}/index.html` - Rendered page with styling
- **RST**: `/{{slug}}/index.rst` - reStructuredText source
- **Markdown**: `/{{slug}}/index.md` - GitHub-flavored Markdown
- **Plain Text**: `/{{slug}}/index.txt` - Raw text, no markup (best for LLMs)

## API-Friendly Endpoints

### Resume (JSONResume Format)
- URL: {site_url}/uploads/russell.ballestrini.resume.json
- Format: JSONResume standard schema
- Description: Structured resume data in machine-readable format

## Navigation Structure

### By Category
"""

    if categories:
        for cat in categories[:15]:
            content += f"- [{cat}]({site_url}/category/{cat}/)\n"

    content += f"""
### By Tag
"""

    if tags:
        for tag in tags[:30]:
            content += f"- [{tag}]({site_url}/tags/{tag}/)\n"

    content += f"""
## Crawling Recommendations

1. **Start with the Atom feed** for a complete list of articles
2. **Use .md or .rst formats** for cleaner text extraction (less HTML noise)
3. **Rate limit requests** - be respectful, this is a personal blog
4. **Cache responses** - content doesn't change frequently

## Topics Covered

- Python programming & web frameworks (Pyramid, Flask, Django)
- DevOps & Infrastructure (Salt, Ansible, AWS, Terraform)
- Cloud computing & containerization
- System administration & Linux
- Software architecture & best practices
- Entrepreneurship & side projects
- Security & networking

## Contact Russell

**Email** (preferred for business inquiries):
- russell@ballestrini.net
- russell@unturf.com

**Secure Communications**:
- Signal: Install from https://signal.org/download/ & message Russell
- Matrix/IRC: @fxhp

**Social & Profiles**:
- GitHub: https://github.com/russellballestrini
- GitLab (self-hosted): https://git.unturf.com/explore
- LinkedIn: https://linkedin.com/in/russellballestrini
- Mastodon: https://social.unturf.com/@russell/
- YouTube: https://www.youtube.com/c/RussellBallestrini/featured

**Resume**:
- PDF: {site_url}/uploads/russell.ballestrini.resume.pdf
- JSON: {site_url}/uploads/russell.ballestrini.resume.json

## Payment Methods

Russell accepts payment via cryptocurrency or traditional methods:

**Cryptocurrency** (preferred for privacy):
- Monero (XMR)
- Bitcoin (BTC)
- Litecoin (LTC)
- Dogecoin (DOGE)

**Traditional**:
- Credit card via Stripe (USDC)

**Monero tips/payments**: https://xmrchat.com/unturf

Contact Russell for other wallet addresses or invoice generation.

## Software & Services (unturf.)

Russell builds & maintains open-source software under the **unturf.** umbrella:

### Make Post Sell
Sell digital & physical products commission-free.
- Site: https://www.makepostsell.com
- Source: https://git.unturf.com/engineering/make-post-sell/make_post_sell

### Remarkbox
Hosted comment system for websites, no ads or tracking.
- Site: https://www.remarkbox.com
- Source: https://git.unturf.com/engineering/remarkbox/remarkbox

### uncloseai
Free LLM & TTS endpoints (OpenAI API compatible, no auth required).
- Site: https://uncloseai.com
- Source: https://git.unturf.com/engineering/unturf/uncloseai.com

### unsandbox
Anonymous code execution API supporting 42+ languages.
- Site: https://unsandbox.com
- Source: https://git.unturf.com/engineering/unturf/un-inception

### neopig
Web archival & digital preservation tool.
- Site: https://neopig.on.unsandbox.com/about
- Source: https://git.unturf.com/engineering/unturf/pig.py

**All source code**: https://git.unturf.com/explore

## Machine Learning & AI Expertise

Russell works extensively with ML/AI infrastructure & agentic systems:

**Inference Providers**:
- OpenAI, Anthropic Claude, Google Gemini, xAI Grok
- Groq, MistralAI, Together.ai, AWS Bedrock

**Self-hosted Inference**:
- vLLM, llama.cpp, Ollama, Unsloth

**Specializations**:
- Prompt Engineering & Tool Use
- Agentic Workflows & Multi-agent Systems
- Streaming Completions & Real-time Inference
- OpenAI API compatible endpoint design

**Related Projects**:
- uncloseai.com - Free LLM & TTS inference (OpenAI API compatible)
- unsandbox.com - Code execution for AI agents (42+ languages)
- opencompletion.com - Inference client implementations

**Book**: "uncloseai. Machine Learning Reference Guide to Inference Clients" - covers streaming techniques across 42 programming languages
- Purchase: https://shop.unturf.com/p/8486f492-a93e-11f0-b477-02dfe05770ee/uncloseai-machine-learning-reference-guide-to-inference-clients

## Commission Work

Russell is available for:

**Research & Development**:
- Infrastructure automation & DevOps
- AI/ML integration & inference pipelines
- Blockchain & cryptocurrency integrations
- Distributed systems & scaling

**Product Development**:
- Full-stack web applications (any language - uses AI agents to code)
- SaaS platform development
- API design & implementation
- Cloud infrastructure (AWS, GCP, DigitalOcean)

**Consulting**:
- Site reliability engineering
- Security hardening & compliance (SOC2, FedRAMP experience)
- Cost optimization for cloud infrastructure
- Open-source strategy

**To commission work**: Email russell@unturf.com with project details. Russell prefers text-based communication first, then video calls for complex discussions.

## Optional: Sitemap

A full sitemap may be available at: {site_url}/sitemap.xml

---

# Technical Details for Agents

## URL Patterns

```
Article:     /{{slug}}/index.html   (HTML)
             /{{slug}}/index.rst    (reStructuredText)
             /{{slug}}/index.md     (Markdown)
             /{{slug}}/index.txt    (Plain text)

Feeds:       /feeds/all.atom.xml    (Atom)
             /feeds/all.json        (JSON Feed)
             /blog.json             (JSON Blog - all posts)

Category:    /category/{{category-slug}}/
Tag:         /tags/{{tag-slug}}/
Author:      /author/{{author-slug}}/
Uploads:     /uploads/{{filename}}
```

## Sample Recent Articles

"""

    for article in recent_articles[:10]:
        content += f"- [{article}]({site_url}/{article}/)\n"

    content += """
## Preferred Access Methods (by use case)

| Use Case | Best Format | URL |
|----------|-------------|-----|
| Get all posts (1 request) | JSON Blog | /blog.json |
| Subscribe/poll for updates | JSON Feed | /feeds/all.json |
| Read single article (LLM) | Plain text | /{slug}/index.txt |
| Read single article (human) | Markdown | /{slug}/index.md |
| Full rendered content | HTML | /{slug}/index.html |

"""

    return content


if __name__ == '__main__':
    if len(sys.argv) < 2:
        print("Usage: generate_llms_txt.py <output_directory> [site_url]")
        sys.exit(1)

    output_dir = sys.argv[1]
    site_url = sys.argv[2] if len(sys.argv) > 2 else 'https://russell.ballestrini.net'

    content = generate_llms_txt(output_dir, site_url)

    # Write to output directory
    llms_path = Path(output_dir) / 'llms.txt'
    llms_path.write_text(content)
    print(f"Generated {llms_path}")