russell.ballestrini.net/lib/generate_plaintext.py

#!/usr/bin/env python3
"""
Generate plain text versions of articles (index.txt) from HTML.
Strips all markup for clean LLM ingestion.
"""

import re
import sys
from html.parser import HTMLParser
from pathlib import Path


class HTMLToText(HTMLParser):
    """Convert HTML to plain text."""

    def __init__(self):
        super().__init__()
        self.text = []
        self.in_pre = False
        self.in_code = False
        self.skip_tags = {'script', 'style', 'head', 'meta', 'link'}
        self.skip_depth = 0

    def handle_starttag(self, tag, attrs):
        if tag in self.skip_tags:
            self.skip_depth += 1
        elif tag == 'pre':
            self.in_pre = True
            self.text.append('\n```\n')
        elif tag == 'code' and not self.in_pre:
            self.in_code = True
            self.text.append('`')
        elif tag in ('p', 'div', 'article', 'section'):
            self.text.append('\n\n')
        elif tag in ('br',):
            self.text.append('\n')
        elif tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
            level = int(tag[1])
            self.text.append('\n\n' + '#' * level + ' ')
        elif tag == 'li':
            self.text.append('\n- ')
        elif tag in ('ul', 'ol'):
            self.text.append('\n')
        elif tag == 'blockquote':
            self.text.append('\n> ')
        elif tag == 'a':
            pass  # Handle in data
        elif tag == 'img':
            for name, value in attrs:
                if name == 'alt' and value:
                    self.text.append(f'[Image: {value}]')
                    break

    def handle_endtag(self, tag):
        if tag in self.skip_tags:
            self.skip_depth -= 1
        elif tag == 'pre':
            self.in_pre = False
            self.text.append('\n```\n')
        elif tag == 'code' and not self.in_pre:
            self.in_code = False
            self.text.append('`')
        elif tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
            self.text.append('\n')
        elif tag in ('ul', 'ol'):
            self.text.append('\n')

    def handle_data(self, data):
        if self.skip_depth > 0:
            return
        if self.in_pre:
            self.text.append(data)
        else:
            # Collapse whitespace in normal text
            cleaned = re.sub(r'\s+', ' ', data)
            self.text.append(cleaned)

    def handle_entityref(self, name):
        entities = {
            'amp': '&', 'lt': '<', 'gt': '>', 'quot': '"',
            'apos': "'", 'nbsp': ' ', 'mdash': '—', 'ndash': '–',
            'ldquo': '"', 'rdquo': '"', 'lsquo': "'", 'rsquo': "'",
        }
        self.text.append(entities.get(name, f'&{name};'))

    def handle_charref(self, name):
        try:
            if name.startswith('x'):
                char = chr(int(name[1:], 16))
            else:
                char = chr(int(name))
            self.text.append(char)
        except (ValueError, OverflowError):
            self.text.append(f'&#{name};')

    def get_text(self):
        text = ''.join(self.text)
        # Clean up excessive newlines
        text = re.sub(r'\n{3,}', '\n\n', text)
        # Clean up spaces around newlines
        text = re.sub(r' *\n *', '\n', text)
        # Remove leading/trailing whitespace per line
        lines = [line.strip() for line in text.split('\n')]
        text = '\n'.join(lines)
        return text.strip()


def html_to_text(html_content):
    """Convert HTML to plain text."""
    parser = HTMLToText()
    parser.feed(html_content)
    return parser.get_text()


def extract_article_content(html_path):
    """Extract main article content from HTML file."""
    content = html_path.read_text(encoding='utf-8')

    # Try to extract just the article content
    # Look for common article containers
    patterns = [
        r'<article[^>]*>(.*?)</article>',
        r'<div[^>]*class="[^"]*entry-content[^"]*"[^>]*>(.*?)</div>',
        r'<div[^>]*class="[^"]*post-content[^"]*"[^>]*>(.*?)</div>',
        r'<div[^>]*class="[^"]*content[^"]*"[^>]*>(.*?)</div>',
    ]

    for pattern in patterns:
        match = re.search(pattern, content, re.DOTALL | re.IGNORECASE)
        if match:
            return html_to_text(match.group(1))

    # Fallback: try to get body content
    body_match = re.search(r'<body[^>]*>(.*?)</body>', content, re.DOTALL | re.IGNORECASE)
    if body_match:
        return html_to_text(body_match.group(1))

    # Last resort: convert entire HTML
    return html_to_text(content)


def convert_output_directory(output_dir):
    """Convert all index.html files to index.txt."""
    output_path = Path(output_dir)
    count = 0

    for html_file in output_path.rglob('index.html'):
        # Skip non-article directories
        parent_name = html_file.parent.name
        if parent_name in ('feeds', 'uploads', 'theme'):
            continue

        txt_file = html_file.with_suffix('.txt')
        try:
            text_content = extract_article_content(html_file)
            if text_content.strip():
                txt_file.write_text(text_content, encoding='utf-8')
                count += 1
        except Exception as e:
            print(f"Warning: Failed to convert {html_file}: {e}")

    return count


if __name__ == '__main__':
    if len(sys.argv) < 2:
        print("Usage: generate_plaintext.py <output_directory>")
        print("       generate_plaintext.py <file.html> [output.txt]")
        sys.exit(1)

    path = Path(sys.argv[1])

    if path.is_dir():
        count = convert_output_directory(path)
        print(f"Generated {count} plain text files")
    elif path.is_file():
        output = sys.argv[2] if len(sys.argv) > 2 else None
        text = extract_article_content(path)
        if output:
            Path(output).write_text(text, encoding='utf-8')
            print(f"Converted {path} -> {output}")
        else:
            print(text)
    else:
        print(f"Error: {path} not found")
        sys.exit(1)