From 7fae521dbaee105139ea2823505b11be9511cdf4 Mon Sep 17 00:00:00 2001
From: Russell Ballestrini <russell@unturf.com>
Date: Fri, 2 Jan 2026 12:54:30 -0500
Subject: [PATCH] Add code, font, and style file crawling support

- Add CODE_EXTENSIONS for 50+ code file types (.py, .js, .rs, .go, etc.)
- Add FONT_EXTENSIONS (.woff, .woff2, .ttf, .otf, .eot)
- Add STYLE_EXTENSIONS (.css, .scss, .sass, .less)
- Add corresponding MIME type sets for detection
- Extract from <link> tags (stylesheets, preload fonts)
- Extract from <script src> tags for JavaScript
- Extract @font-face URLs from inline style blocks
- Make import resilient to missing tables in source archives
---
 async_web_fetcher.py | 120 +++++++++++++++++++++++-
 serp.py              | 212 +++++++++++++------------------------------
 2 files changed, 179 insertions(+), 153 deletions(-)

diff --git a/async_web_fetcher.py b/async_web_fetcher.py
index ae9c489..d82e397 100644
--- a/async_web_fetcher.py
+++ b/async_web_fetcher.py
@@ -44,10 +44,72 @@ IMAGE_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.bmp', '.
 VIDEO_EXTENSIONS = {'.mp4', '.webm', '.mov', '.avi', '.mkv', '.m4v', '.ogv', '.flv', '.wmv'}
 AUDIO_EXTENSIONS = {'.mp3', '.wav', '.ogg', '.m4a', '.flac', '.aac', '.wma'}
 
+# Code file extensions
+CODE_EXTENSIONS = {
+    '.py', '.js', '.ts', '.jsx', '.tsx', '.mjs', '.cjs',  # Python, JavaScript, TypeScript
+    '.rs', '.go', '.rb', '.php', '.pl', '.pm',  # Rust, Go, Ruby, PHP, Perl
+    '.java', '.kt', '.kts', '.scala', '.groovy',  # JVM languages
+    '.c', '.h', '.cpp', '.hpp', '.cc', '.cxx', '.hxx',  # C/C++
+    '.cs', '.fs', '.fsx',  # .NET
+    '.swift', '.m', '.mm',  # Apple
+    '.lua', '.r', '.R', '.jl',  # Lua, R, Julia
+    '.sh', '.bash', '.zsh', '.fish', '.ps1',  # Shell
+    '.sql', '.graphql', '.gql',  # Query languages
+    '.yaml', '.yml', '.toml', '.json', '.xml',  # Config
+    '.md', '.rst', '.txt',  # Docs
+    '.zig', '.nim', '.d', '.v',  # Modern systems langs
+    '.ex', '.exs', '.erl', '.hrl',  # Erlang/Elixir
+    '.clj', '.cljs', '.cljc', '.edn',  # Clojure
+    '.hs', '.lhs',  # Haskell
+    '.ml', '.mli', '.re', '.rei',  # OCaml/ReasonML
+    '.lisp', '.cl', '.el', '.scm', '.rkt',  # Lisps
+    '.f90', '.f95', '.f03', '.for',  # Fortran
+    '.asm', '.s',  # Assembly
+    '.cob', '.cbl',  # COBOL
+    '.pro',  # Prolog
+    '.tcl',  # Tcl
+    '.dart',  # Dart
+    '.raku', '.p6',  # Raku
+    '.cr',  # Crystal
+    '.vue', '.svelte',  # Frontend frameworks
+    '.tf', '.hcl',  # Terraform
+    '.dockerfile', '.makefile',  # Build files
+}
+
+# Font file extensions
+FONT_EXTENSIONS = {'.woff', '.woff2', '.ttf', '.otf', '.eot', '.sfnt'}
+
+# Style file extensions
+STYLE_EXTENSIONS = {'.css', '.scss', '.sass', '.less', '.styl'}
+
 # MIME types by media type
 IMAGE_MIME_PREFIXES = ('image/',)
 VIDEO_MIME_PREFIXES = ('video/',)
 AUDIO_MIME_PREFIXES = ('audio/',)
+CODE_MIME_TYPES = {
+    'text/x-python', 'application/x-python', 'text/x-python-script',
+    'text/javascript', 'application/javascript', 'application/x-javascript',
+    'text/typescript', 'application/typescript',
+    'text/x-rust', 'text/x-go', 'text/x-ruby', 'application/x-ruby',
+    'text/x-java-source', 'text/x-kotlin', 'text/x-scala',
+    'text/x-c', 'text/x-c++', 'text/x-csrc', 'text/x-c++src',
+    'text/x-csharp', 'text/x-fsharp',
+    'text/x-swift', 'text/x-objective-c',
+    'text/x-lua', 'text/x-r', 'text/x-julia',
+    'text/x-shellscript', 'application/x-sh', 'text/x-bash',
+    'application/sql', 'application/graphql',
+    'application/json', 'application/xml', 'text/xml',
+    'text/yaml', 'application/x-yaml', 'text/x-yaml',
+    'text/markdown', 'text/x-markdown',
+    'text/plain',  # Often used for code
+}
+FONT_MIME_TYPES = {
+    'font/woff', 'font/woff2', 'font/ttf', 'font/otf', 'font/sfnt',
+    'application/font-woff', 'application/font-woff2',
+    'application/x-font-ttf', 'application/x-font-otf',
+    'application/vnd.ms-fontobject',
+}
+STYLE_MIME_TYPES = {'text/css', 'text/x-scss', 'text/x-sass', 'text/x-less'}
 
 
 @dataclass
@@ -376,13 +438,14 @@ def get_media_type_from_extension(url: str) -> Optional[str]:
     Determine media type from URL extension.
 
     Returns:
-        'image', 'video', 'audio', or None
+        'image', 'video', 'audio', 'code', 'font', 'style', or None
     """
     parsed = Uri(url)
     if not parsed.path:
         return None
     path = parsed.path.lower()
 
+    # Check for extension match
     for ext in IMAGE_EXTENSIONS:
         if path.endswith(ext):
             return 'image'
@@ -392,6 +455,15 @@ def get_media_type_from_extension(url: str) -> Optional[str]:
     for ext in AUDIO_EXTENSIONS:
         if path.endswith(ext):
             return 'audio'
+    for ext in CODE_EXTENSIONS:
+        if path.endswith(ext):
+            return 'code'
+    for ext in FONT_EXTENSIONS:
+        if path.endswith(ext):
+            return 'font'
+    for ext in STYLE_EXTENSIONS:
+        if path.endswith(ext):
+            return 'style'
     return None
 
 
@@ -400,18 +472,26 @@ def get_media_type_from_mime(mime_type: str) -> Optional[str]:
     Determine media type from MIME type.
 
     Returns:
-        'image', 'video', 'audio', or None
+        'image', 'video', 'audio', 'code', 'font', 'style', or None
     """
     if not mime_type:
         return None
 
     mime_lower = mime_type.lower()
+    # Check prefixes first
     if mime_lower.startswith(IMAGE_MIME_PREFIXES):
         return 'image'
     if mime_lower.startswith(VIDEO_MIME_PREFIXES):
         return 'video'
     if mime_lower.startswith(AUDIO_MIME_PREFIXES):
         return 'audio'
+    # Check exact matches for code/font/style
+    if mime_lower in CODE_MIME_TYPES:
+        return 'code'
+    if mime_lower in FONT_MIME_TYPES:
+        return 'font'
+    if mime_lower in STYLE_MIME_TYPES:
+        return 'style'
     return None
 
 
@@ -710,16 +790,48 @@ def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMod
                 if src:
                     add_media(src, 'audio', title=title)
 
-    # Extract from <a href="..."> pointing to media files
+    # Extract from <a href="..."> pointing to media/code/font/style files
     for a in soup.find_all('a', href=True):
         href = a['href']
         media_type = get_media_type_from_extension(href)
         if media_type:
             if (media_type == 'image' and collect_images) or \
                (media_type == 'video' and collect_videos) or \
-               (media_type == 'audio' and collect_audio):
+               (media_type == 'audio' and collect_audio) or \
+               media_type in ('code', 'font', 'style'):  # Always collect code/font/style
                 add_media(href, media_type, alt_text=a.get_text(strip=True)[:100])
 
+    # Extract from <link> tags for stylesheets and fonts
+    for link in soup.find_all('link', href=True):
+        href = link.get('href')
+        rel = link.get('rel', [])
+        as_attr = link.get('as', '')
+
+        if 'stylesheet' in rel:
+            add_media(href, 'style', alt_text='stylesheet')
+        elif 'preload' in rel and as_attr == 'font':
+            add_media(href, 'font', alt_text='preload font')
+        elif 'preload' in rel and as_attr == 'style':
+            add_media(href, 'style', alt_text='preload style')
+        else:
+            # Check by extension
+            media_type = get_media_type_from_extension(href)
+            if media_type in ('font', 'style'):
+                add_media(href, media_type, alt_text=f'link {media_type}')
+
+    # Extract from <script src="..."> for JavaScript files
+    for script in soup.find_all('script', src=True):
+        src = script.get('src')
+        if src:
+            add_media(src, 'code', alt_text='script')
+
+    # Extract font URLs from @font-face in <style> blocks
+    font_face_pattern = re.compile(r'@font-face\s*\{[^}]*url\(["\']?([^"\')\s]+)["\']?\)', re.IGNORECASE | re.DOTALL)
+    for style_tag in soup.find_all('style'):
+        if style_tag.string:
+            for match in font_face_pattern.findall(style_tag.string):
+                add_media(match, 'font', alt_text='font-face')
+
     # Extract from CSS background-image: url(...)
     if collect_images:
         bg_pattern = re.compile(r'background(?:-image)?\s*:\s*url\(["\']?([^"\')\s]+)["\']?\)', re.IGNORECASE)
diff --git a/serp.py b/serp.py
index 5fc3fd3..908f72a 100644
--- a/serp.py
+++ b/serp.py
@@ -3119,116 +3119,33 @@ async def phantom_page(lang: str = Cookie(None), accept_language: str = Header(N
     return inject_i18n(html, language)
 
 
-@app.get("/about", response_class=HTMLResponse)
-async def about_page(lang: str = Cookie(None), accept_language: str = Header(None)):
-    """About neopig - the story of pig.py's evolution."""
-    language = get_lang(lang, accept_language)
-    html = f"""
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8">
-    <title>{{about}} neopig</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <style>
-        {VIEW_CSS}
-        .about-content {{
-            max-width: 800px;
-            margin: 0 auto;
-            padding: 40px 20px;
-            line-height: 1.8;
-            font-size: 17px;
-        }}
-        .about-content h1 {{
-            font-size: 2.5em;
-            margin-bottom: 10px;
-            color: #ff6b6b;
-        }}
-        .about-content h2 {{
-            font-size: 1.6em;
-            margin-top: 40px;
-            margin-bottom: 15px;
-            color: #4ecdc4;
-            border-bottom: 2px solid #333;
-            padding-bottom: 10px;
-        }}
-        .about-content p {{
-            margin-bottom: 20px;
-            color: #e0e0e0;
-        }}
-        .about-content a {{
-            color: #ff6b6b;
-            text-decoration: none;
-        }}
-        .about-content a:hover {{
-            text-decoration: underline;
-        }}
-        .about-content code {{
-            background: #2a2a2a;
-            padding: 2px 8px;
-            border-radius: 4px;
-            font-family: 'Fira Code', monospace;
-            font-size: 0.9em;
-        }}
-        .about-content pre {{
-            background: #1a1a1a;
-            padding: 20px;
-            border-radius: 8px;
-            overflow-x: auto;
-            border: 1px solid #333;
-        }}
-        .about-content pre code {{
-            background: none;
-            padding: 0;
-        }}
-        .about-content ul, .about-content ol {{
-            margin-bottom: 20px;
-            padding-left: 30px;
-        }}
-        .about-content li {{
-            margin-bottom: 10px;
-            color: #e0e0e0;
-        }}
-        .feature-grid {{
-            display: grid;
-            grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
-            gap: 20px;
-            margin: 30px 0;
-        }}
-        .feature-card {{
-            background: #1e1e24;
-            padding: 20px;
-            border-radius: 12px;
-            border: 1px solid #333;
-        }}
-        .feature-card h3 {{
-            color: #ff6b6b;
-            margin-bottom: 10px;
-            font-size: 1.1em;
-        }}
-        .feature-card p {{
-            color: #aaa;
-            font-size: 0.95em;
-            margin: 0;
-        }}
-        .origin-quote {{
-            background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%);
-            border-left: 4px solid #ff6b6b;
-            padding: 20px 25px;
-            margin: 30px 0;
-            border-radius: 0 8px 8px 0;
-            font-style: italic;
-        }}
-        .origin-quote cite {{
-            display: block;
-            margin-top: 10px;
-            color: #888;
-            font-style: normal;
-        }}
-    </style>
-</head>
-<body>
-    <!-- NAV -->
+ABOUT_CSS = """
+.about-content {
+    max-width: 800px;
+    margin: 0 auto;
+    padding: 40px 20px;
+    line-height: 1.8;
+    font-size: 17px;
+}
+.about-content h1 { font-size: 2.5em; margin-bottom: 10px; color: #ff6b6b; }
+.about-content h2 { font-size: 1.6em; margin-top: 40px; margin-bottom: 15px; color: #4ecdc4; border-bottom: 2px solid #333; padding-bottom: 10px; }
+.about-content p { margin-bottom: 20px; color: #e0e0e0; }
+.about-content a { color: #ff6b6b; text-decoration: none; }
+.about-content a:hover { text-decoration: underline; }
+.about-content code { background: #2a2a2a; padding: 2px 8px; border-radius: 4px; font-family: 'Fira Code', monospace; font-size: 0.9em; }
+.about-content pre { background: #1a1a1a; padding: 20px; border-radius: 8px; overflow-x: auto; border: 1px solid #333; }
+.about-content pre code { background: none; padding: 0; }
+.about-content ul, .about-content ol { margin-bottom: 20px; padding-left: 30px; }
+.about-content li { margin-bottom: 10px; color: #e0e0e0; }
+.feature-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); gap: 20px; margin: 30px 0; }
+.feature-card { background: #1e1e24; padding: 20px; border-radius: 12px; border: 1px solid #333; }
+.feature-card h3 { color: #ff6b6b; margin-bottom: 10px; font-size: 1.1em; }
+.feature-card p { color: #aaa; font-size: 0.95em; margin: 0; }
+.origin-quote { background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%); border-left: 4px solid #ff6b6b; padding: 20px 25px; margin: 30px 0; border-radius: 0 8px 8px 0; font-style: italic; }
+.origin-quote cite { display: block; margin-top: 10px; color: #888; font-style: normal; }
+"""
+
+ABOUT_CONTENT = """
     <div class="about-content">
         <h1>🐷 neopig</h1>
         <p><strong>Neo Python Image Grabber</strong> &mdash; A full-domain async media crawler with content-addressable storage, full-text search, and page screenshot capture.</p>
@@ -3248,9 +3165,23 @@ async def about_page(lang: str = Cookie(None), accept_language: str = Header(Non
 
         <p>That was it. Point it at a URL, and it would slurp down every image it could find. No configuration, no complexity &mdash; just a hungry little pig gobbling up pixels.</p>
 
+        <h2>The Great Bitbucket Extinction</h2>
+
+        <p>Then came the dark times. The original source lived at <code><a href="https://bitbucket.org/russellballestrini/pig" style="text-decoration:line-through;color:#666;">bitbucket.org/russellballestrini/pig</a></code> &mdash; click it, we dare you. In 2020, Atlassian swallowed Bitbucket whole and spat out a glorious fountain of Mercurial repositories into the void. The original pig.py, nestled in its cozy hg repo, was atomized in the great purge.</p>
+
+        <p>But here's where it gets <em>weird</em>.</p>
+
+        <p>Russell once wrote that <a href="https://russell.ballestrini.net/programming-is-like-alchemy/">"programming is like alchemy &mdash; instead of exchanging matter, we programmers exchange time."</a> Programs are golems, familiar spirits, magical servants performing repetitive tasks. <em>"It is more accurate to group programs with technology than magic, but less fun."</em></p>
+
+        <p>And speaking of alchemy: <a href="https://phys.org/news/2025-07-marathon-fusion-mercury-gold-energy.html">Marathon Fusion</a> discovered that tokamak breeding blankets &mdash; wrapped in Mercury-Lithium alloy, like pigs in a blanket &mdash; can transmute Mercury-198 into Gold-197 through chrysopoeia. Fast neutrons trigger (n, 2n) reactions; unstable mercury decays into stable gold within 64 hours. Two metric tons of gold per gigawatt. The alchemists' dream realized, wrapped in radioactive patience (17.7 years of cooling before you can touch your transmuted treasure).</p>
+
+        <p>A golden goose born from the ashes of deprecated version control. The old pig was archived, but a new creature stirred in the digital depths...</p>
+
         <h2>The Evolution to neopig</h2>
 
-        <p>Over a decade later, neopig emerged as a spiritual successor &mdash; keeping the original's spirit of simplicity while adding the features needed for serious web archival:</p>
+        <p>Like a phoenix rising from dead Bitbucket repos, neopig emerged &mdash; a chimera, a griffin, a more hungry and gluttonous beast than its predecessor ever dreamed of being. Where pig.py sipped politely from single pages, neopig <em>devours entire domains</em>.</p>
+
+        <p>Over a decade later, this spiritual successor keeps the original's spirit of simplicity while adding the features needed for serious web archival:</p>
 
         <div class="feature-grid">
             <div class="feature-card">
@@ -3345,9 +3276,14 @@ python neopig.py --upgrade-neopig ./archives/example.com-20251231.tar.gz</code><
             This project continues that tradition of building useful tools for the community.
         </p>
     </div>
-</body>
-</html>
 """
+
+
+@app.get("/about", response_class=HTMLResponse)
+async def about_page(lang: str = Cookie(None), accept_language: str = Header(None)):
+    """About neopig - the story of pig.py's evolution."""
+    language = get_lang(lang, accept_language)
+    html = layout("{{about}}", ABOUT_CONTENT, extra_css=ABOUT_CSS)
     return inject_i18n(html, language)
 
 
@@ -3827,15 +3763,6 @@ async def complete_resumable_upload(upload_id: str):
     }
 
 
-@app.get("/sandbox", response_class=HTMLResponse)
-async def sandbox_page(lang: str = Cookie(None), accept_language: str = Header(None)):
-    """Sandbox page for uploading archives."""
-    if not SANDBOX_MODE:
-        raise HTTPException(status_code=403, detail="Sandbox mode not enabled (NEOPIG_SANDBOX=1)")
-    language = get_lang(lang, accept_language)
-    return inject_i18n(SANDBOX_HTML, language)
-
-
 @app.post("/api/sandbox/upload")
 async def upload_archive(file: UploadFile = File(...)):
     """Upload a tar.gz archive and import into local database as a job."""
@@ -4030,27 +3957,8 @@ async def sandbox_status():
     }
 
 
-SANDBOX_HTML = """
-<!DOCTYPE html>
-<html>
-<head>
-    <title>neopig Sandbox</title>
-    <style>
-        * { box-sizing: border-box; }
-        body {
-            font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
-            margin: 0; padding: 0;
-            background: #0a0a0a; color: #e0e0e0;
-        }
-        .nav {
-            background: #1a1a1a; padding: 10px 20px;
-            display: flex; gap: 20px; align-items: center;
-            border-bottom: 1px solid #333;
-        }
-        .nav a { color: #ff6b6b; text-decoration: none; }
-        .nav a:hover { text-decoration: underline; }
-        .nav .brand { font-weight: bold; font-size: 18px; }
-        .container { padding: 20px; max-width: 800px; margin: 0 auto; }
+SANDBOX_CSS = """
+        .container { max-width: 800px; margin: 0 auto; }
         h1 { color: #ff6b6b; margin-bottom: 5px; }
         .subtitle { color: #666; margin-bottom: 30px; }
 
@@ -4095,11 +4003,9 @@ SANDBOX_HTML = """
 
         .success { color: #6f6; }
         .error { color: #f66; }
-    </style>
-</head>
-<body>
-    <!-- NAV -->
+"""
 
+SANDBOX_CONTENT = """
     <div class="container">
         <h1>🧪 Sandbox Mode</h1>
         <p class="subtitle">Upload a neopig archive (.tar.gz or .run) to explore it</p>
@@ -4283,11 +4189,19 @@ SANDBOX_HTML = """
 
         loadStatus();
     </script>
-</body>
-</html>
 """
 
 
+@app.get("/sandbox", response_class=HTMLResponse)
+async def sandbox_page(lang: str = Cookie(None), accept_language: str = Header(None)):
+    """Sandbox page for uploading archives."""
+    if not SANDBOX_MODE:
+        raise HTTPException(status_code=403, detail="Sandbox mode not enabled (NEOPIG_SANDBOX=1)")
+    language = get_lang(lang, accept_language)
+    html = layout("Sandbox", SANDBOX_CONTENT, extra_css=SANDBOX_CSS)
+    return inject_i18n(html, language)
+
+
 # ============================================================================
 # Crawler API
 # ============================================================================