engineering/pig.py
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add code, font, and style file crawling support

Browse source 
- Add CODE_EXTENSIONS for 50+ code file types (.py, .js, .rs, .go, etc.)
- Add FONT_EXTENSIONS (.woff, .woff2, .ttf, .otf, .eot)
- Add STYLE_EXTENSIONS (.css, .scss, .sass, .less)
- Add corresponding MIME type sets for detection
- Extract from <link> tags (stylesheets, preload fonts)
- Extract from <script src> tags for JavaScript
- Extract @font-face URLs from inline style blocks
- Make import resilient to missing tables in source archives
This commit is contained in:
Russell Ballestrini 2026-01-02 12:54:30 -05:00
parent e2a556f7a4
commit 7fae521dba
2 changed files with 179 additions and 153 deletions
Download patch file Download diff file Expand all files Collapse all files

120
async_web_fetcher.py
View file

@ -44,10 +44,72 @@ IMAGE_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.bmp', '.
VIDEO_EXTENSIONS = {'.mp4', '.webm', '.mov', '.avi', '.mkv', '.m4v', '.ogv', '.flv', '.wmv'}
AUDIO_EXTENSIONS = {'.mp3', '.wav', '.ogg', '.m4a', '.flac', '.aac', '.wma'}
# Code file extensions
CODE_EXTENSIONS = {
'.py', '.js', '.ts', '.jsx', '.tsx', '.mjs', '.cjs', # Python, JavaScript, TypeScript
'.rs', '.go', '.rb', '.php', '.pl', '.pm', # Rust, Go, Ruby, PHP, Perl
'.java', '.kt', '.kts', '.scala', '.groovy', # JVM languages
'.c', '.h', '.cpp', '.hpp', '.cc', '.cxx', '.hxx', # C/C++
'.cs', '.fs', '.fsx', # .NET
'.swift', '.m', '.mm', # Apple
'.lua', '.r', '.R', '.jl', # Lua, R, Julia
'.sh', '.bash', '.zsh', '.fish', '.ps1', # Shell
'.sql', '.graphql', '.gql', # Query languages
'.yaml', '.yml', '.toml', '.json', '.xml', # Config
'.md', '.rst', '.txt', # Docs
'.zig', '.nim', '.d', '.v', # Modern systems langs
'.ex', '.exs', '.erl', '.hrl', # Erlang/Elixir
'.clj', '.cljs', '.cljc', '.edn', # Clojure
'.hs', '.lhs', # Haskell
'.ml', '.mli', '.re', '.rei', # OCaml/ReasonML
'.lisp', '.cl', '.el', '.scm', '.rkt', # Lisps
'.f90', '.f95', '.f03', '.for', # Fortran
'.asm', '.s', # Assembly
'.cob', '.cbl', # COBOL
'.pro', # Prolog
'.tcl', # Tcl
'.dart', # Dart
'.raku', '.p6', # Raku
'.cr', # Crystal
'.vue', '.svelte', # Frontend frameworks
'.tf', '.hcl', # Terraform
'.dockerfile', '.makefile', # Build files
}
# Font file extensions
FONT_EXTENSIONS = {'.woff', '.woff2', '.ttf', '.otf', '.eot', '.sfnt'}
# Style file extensions
STYLE_EXTENSIONS = {'.css', '.scss', '.sass', '.less', '.styl'}
# MIME types by media type
IMAGE_MIME_PREFIXES = ('image/',)
VIDEO_MIME_PREFIXES = ('video/',)
AUDIO_MIME_PREFIXES = ('audio/',)
CODE_MIME_TYPES = {
'text/x-python', 'application/x-python', 'text/x-python-script',
'text/javascript', 'application/javascript', 'application/x-javascript',
'text/typescript', 'application/typescript',
'text/x-rust', 'text/x-go', 'text/x-ruby', 'application/x-ruby',
'text/x-java-source', 'text/x-kotlin', 'text/x-scala',
'text/x-c', 'text/x-c++', 'text/x-csrc', 'text/x-c++src',
'text/x-csharp', 'text/x-fsharp',
'text/x-swift', 'text/x-objective-c',
'text/x-lua', 'text/x-r', 'text/x-julia',
'text/x-shellscript', 'application/x-sh', 'text/x-bash',
'application/sql', 'application/graphql',
'application/json', 'application/xml', 'text/xml',
'text/yaml', 'application/x-yaml', 'text/x-yaml',
'text/markdown', 'text/x-markdown',
'text/plain', # Often used for code
}
FONT_MIME_TYPES = {
'font/woff', 'font/woff2', 'font/ttf', 'font/otf', 'font/sfnt',
'application/font-woff', 'application/font-woff2',
'application/x-font-ttf', 'application/x-font-otf',
'application/vnd.ms-fontobject',
}
STYLE_MIME_TYPES = {'text/css', 'text/x-scss', 'text/x-sass', 'text/x-less'}
@dataclass
@ -376,13 +438,14 @@ def get_media_type_from_extension(url: str) -> Optional[str]:
Determine media type from URL extension.
Returns:
'image', 'video', 'audio', or None
'image', 'video', 'audio', 'code', 'font', 'style', or None
"""
parsed = Uri(url)
if not parsed.path:
return None
path = parsed.path.lower()
# Check for extension match
for ext in IMAGE_EXTENSIONS:
if path.endswith(ext):
return 'image'
@ -392,6 +455,15 @@ def get_media_type_from_extension(url: str) -> Optional[str]:
for ext in AUDIO_EXTENSIONS:
if path.endswith(ext):
return 'audio'
for ext in CODE_EXTENSIONS:
if path.endswith(ext):
return 'code'
for ext in FONT_EXTENSIONS:
if path.endswith(ext):
return 'font'
for ext in STYLE_EXTENSIONS:
if path.endswith(ext):
return 'style'
return None
@ -400,18 +472,26 @@ def get_media_type_from_mime(mime_type: str) -> Optional[str]:
Determine media type from MIME type.
Returns:
'image', 'video', 'audio', or None
'image', 'video', 'audio', 'code', 'font', 'style', or None
"""
if not mime_type:
return None
mime_lower = mime_type.lower()
# Check prefixes first
if mime_lower.startswith(IMAGE_MIME_PREFIXES):
return 'image'
if mime_lower.startswith(VIDEO_MIME_PREFIXES):
return 'video'
if mime_lower.startswith(AUDIO_MIME_PREFIXES):
return 'audio'
# Check exact matches for code/font/style
if mime_lower in CODE_MIME_TYPES:
return 'code'
if mime_lower in FONT_MIME_TYPES:
return 'font'
if mime_lower in STYLE_MIME_TYPES:
return 'style'
return None
@ -710,16 +790,48 @@ def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMod
if src:
add_media(src, 'audio', title=title)
# Extract from <a href="..."> pointing to media files
# Extract from <a href="..."> pointing to media/code/font/style files
for a in soup.find_all('a', href=True):
href = a['href']
media_type = get_media_type_from_extension(href)
if media_type:
if (media_type == 'image' and collect_images) or \
(media_type == 'video' and collect_videos) or \
(media_type == 'audio' and collect_audio):
(media_type == 'audio' and collect_audio) or \
media_type in ('code', 'font', 'style'): # Always collect code/font/style
add_media(href, media_type, alt_text=a.get_text(strip=True)[:100])
# Extract from <link> tags for stylesheets and fonts
for link in soup.find_all('link', href=True):
href = link.get('href')
rel = link.get('rel', [])
as_attr = link.get('as', '')
if 'stylesheet' in rel:
add_media(href, 'style', alt_text='stylesheet')
elif 'preload' in rel and as_attr == 'font':
add_media(href, 'font', alt_text='preload font')
elif 'preload' in rel and as_attr == 'style':
add_media(href, 'style', alt_text='preload style')
else:
# Check by extension
media_type = get_media_type_from_extension(href)
if media_type in ('font', 'style'):
add_media(href, media_type, alt_text=f'link {media_type}')
# Extract from <script src="..."> for JavaScript files
for script in soup.find_all('script', src=True):
src = script.get('src')
if src:
add_media(src, 'code', alt_text='script')
# Extract font URLs from @font-face in <style> blocks
font_face_pattern = re.compile(r'@font-face\s*\{[^}]*url\(["\']?([^"\')\s]+)["\']?\)', re.IGNORECASE | re.DOTALL)
for style_tag in soup.find_all('style'):
if style_tag.string:
for match in font_face_pattern.findall(style_tag.string):
add_media(match, 'font', alt_text='font-face')
# Extract from CSS background-image: url(...)
if collect_images:
bg_pattern = re.compile(r'background(?:-image)?\s*:\s*url\(["\']?([^"\')\s]+)["\']?\)', re.IGNORECASE)

212
serp.py
View file

@ -3119,116 +3119,33 @@ async def phantom_page(lang: str = Cookie(None), accept_language: str = Header(N
return inject_i18n(html, language)
@app.get("/about", response_class=HTMLResponse)
async def about_page(lang: str = Cookie(None), accept_language: str = Header(None)):
"""About neopig - the story of pig.py's evolution."""
language = get_lang(lang, accept_language)
html = f"""
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>{{about}} neopig</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<style>
{VIEW_CSS}
.about-content {{
max-width: 800px;
margin: 0 auto;
padding: 40px 20px;
line-height: 1.8;
font-size: 17px;
}}
.about-content h1 {{
font-size: 2.5em;
margin-bottom: 10px;
color: #ff6b6b;
}}
.about-content h2 {{
font-size: 1.6em;
margin-top: 40px;
margin-bottom: 15px;
color: #4ecdc4;
border-bottom: 2px solid #333;
padding-bottom: 10px;
}}
.about-content p {{
margin-bottom: 20px;
color: #e0e0e0;
}}
.about-content a {{
color: #ff6b6b;
text-decoration: none;
}}
.about-content a:hover {{
text-decoration: underline;
}}
.about-content code {{
background: #2a2a2a;
padding: 2px 8px;
border-radius: 4px;
font-family: 'Fira Code', monospace;
font-size: 0.9em;
}}
.about-content pre {{
background: #1a1a1a;
padding: 20px;
border-radius: 8px;
overflow-x: auto;
border: 1px solid #333;
}}
.about-content pre code {{
background: none;
padding: 0;
}}
.about-content ul, .about-content ol {{
margin-bottom: 20px;
padding-left: 30px;
}}
.about-content li {{
margin-bottom: 10px;
color: #e0e0e0;
}}
.feature-grid {{
display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
gap: 20px;
margin: 30px 0;
}}
.feature-card {{
background: #1e1e24;
padding: 20px;
border-radius: 12px;
border: 1px solid #333;
}}
.feature-card h3 {{
color: #ff6b6b;
margin-bottom: 10px;
font-size: 1.1em;
}}
.feature-card p {{
color: #aaa;
font-size: 0.95em;
margin: 0;
}}
.origin-quote {{
background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%);
border-left: 4px solid #ff6b6b;
padding: 20px 25px;
margin: 30px 0;
border-radius: 0 8px 8px 0;
font-style: italic;
}}
.origin-quote cite {{
display: block;
margin-top: 10px;
color: #888;
font-style: normal;
}}
</style>
</head>
<body>
<!-- NAV -->
ABOUT_CSS = """
.about-content {
max-width: 800px;
margin: 0 auto;
padding: 40px 20px;
line-height: 1.8;
font-size: 17px;
}
.about-content h1 { font-size: 2.5em; margin-bottom: 10px; color: #ff6b6b; }
.about-content h2 { font-size: 1.6em; margin-top: 40px; margin-bottom: 15px; color: #4ecdc4; border-bottom: 2px solid #333; padding-bottom: 10px; }
.about-content p { margin-bottom: 20px; color: #e0e0e0; }
.about-content a { color: #ff6b6b; text-decoration: none; }
.about-content a:hover { text-decoration: underline; }
.about-content code { background: #2a2a2a; padding: 2px 8px; border-radius: 4px; font-family: 'Fira Code', monospace; font-size: 0.9em; }
.about-content pre { background: #1a1a1a; padding: 20px; border-radius: 8px; overflow-x: auto; border: 1px solid #333; }
.about-content pre code { background: none; padding: 0; }
.about-content ul, .about-content ol { margin-bottom: 20px; padding-left: 30px; }
.about-content li { margin-bottom: 10px; color: #e0e0e0; }
.feature-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); gap: 20px; margin: 30px 0; }
.feature-card { background: #1e1e24; padding: 20px; border-radius: 12px; border: 1px solid #333; }
.feature-card h3 { color: #ff6b6b; margin-bottom: 10px; font-size: 1.1em; }
.feature-card p { color: #aaa; font-size: 0.95em; margin: 0; }
.origin-quote { background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%); border-left: 4px solid #ff6b6b; padding: 20px 25px; margin: 30px 0; border-radius: 0 8px 8px 0; font-style: italic; }
.origin-quote cite { display: block; margin-top: 10px; color: #888; font-style: normal; }
"""
ABOUT_CONTENT = """
<div class="about-content">
<h1>🐷 neopig</h1>
<p><strong>Neo Python Image Grabber</strong> &mdash; A full-domain async media crawler with content-addressable storage, full-text search, and page screenshot capture.</p>
@ -3248,9 +3165,23 @@ async def about_page(lang: str = Cookie(None), accept_language: str = Header(Non
<p>That was it. Point it at a URL, and it would slurp down every image it could find. No configuration, no complexity &mdash; just a hungry little pig gobbling up pixels.</p>
<h2>The Great Bitbucket Extinction</h2>
<p>Then came the dark times. The original source lived at <code><a href="https://bitbucket.org/russellballestrini/pig" style="text-decoration:line-through;color:#666;">bitbucket.org/russellballestrini/pig</a></code> &mdash; click it, we dare you. In 2020, Atlassian swallowed Bitbucket whole and spat out a glorious fountain of Mercurial repositories into the void. The original pig.py, nestled in its cozy hg repo, was atomized in the great purge.</p>
<p>But here's where it gets <em>weird</em>.</p>
<p>Russell once wrote that <a href="https://russell.ballestrini.net/programming-is-like-alchemy/">"programming is like alchemy &mdash; instead of exchanging matter, we programmers exchange time."</a> Programs are golems, familiar spirits, magical servants performing repetitive tasks. <em>"It is more accurate to group programs with technology than magic, but less fun."</em></p>
<p>And speaking of alchemy: <a href="https://phys.org/news/2025-07-marathon-fusion-mercury-gold-energy.html">Marathon Fusion</a> discovered that tokamak breeding blankets &mdash; wrapped in Mercury-Lithium alloy, like pigs in a blanket &mdash; can transmute Mercury-198 into Gold-197 through chrysopoeia. Fast neutrons trigger (n, 2n) reactions; unstable mercury decays into stable gold within 64 hours. Two metric tons of gold per gigawatt. The alchemists' dream realized, wrapped in radioactive patience (17.7 years of cooling before you can touch your transmuted treasure).</p>
<p>A golden goose born from the ashes of deprecated version control. The old pig was archived, but a new creature stirred in the digital depths...</p>
<h2>The Evolution to neopig</h2>
<p>Over a decade later, neopig emerged as a spiritual successor &mdash; keeping the original's spirit of simplicity while adding the features needed for serious web archival:</p>
<p>Like a phoenix rising from dead Bitbucket repos, neopig emerged &mdash; a chimera, a griffin, a more hungry and gluttonous beast than its predecessor ever dreamed of being. Where pig.py sipped politely from single pages, neopig <em>devours entire domains</em>.</p>
<p>Over a decade later, this spiritual successor keeps the original's spirit of simplicity while adding the features needed for serious web archival:</p>
<div class="feature-grid">
<div class="feature-card">
@ -3345,9 +3276,14 @@ python neopig.py --upgrade-neopig ./archives/example.com-20251231.tar.gz</code><
This project continues that tradition of building useful tools for the community.
</p>
</div>
</body>
</html>
"""
@app.get("/about", response_class=HTMLResponse)
async def about_page(lang: str = Cookie(None), accept_language: str = Header(None)):
"""About neopig - the story of pig.py's evolution."""
language = get_lang(lang, accept_language)
html = layout("{{about}}", ABOUT_CONTENT, extra_css=ABOUT_CSS)
return inject_i18n(html, language)
@ -3827,15 +3763,6 @@ async def complete_resumable_upload(upload_id: str):
}
@app.get("/sandbox", response_class=HTMLResponse)
async def sandbox_page(lang: str = Cookie(None), accept_language: str = Header(None)):
"""Sandbox page for uploading archives."""
if not SANDBOX_MODE:
raise HTTPException(status_code=403, detail="Sandbox mode not enabled (NEOPIG_SANDBOX=1)")
language = get_lang(lang, accept_language)
return inject_i18n(SANDBOX_HTML, language)
@app.post("/api/sandbox/upload")
async def upload_archive(file: UploadFile = File(...)):
"""Upload a tar.gz archive and import into local database as a job."""
@ -4030,27 +3957,8 @@ async def sandbox_status():
}
SANDBOX_HTML = """
<!DOCTYPE html>
<html>
<head>
<title>neopig Sandbox</title>
<style>
* { box-sizing: border-box; }
body {
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
margin: 0; padding: 0;
background: #0a0a0a; color: #e0e0e0;
}
.nav {
background: #1a1a1a; padding: 10px 20px;
display: flex; gap: 20px; align-items: center;
border-bottom: 1px solid #333;
}
.nav a { color: #ff6b6b; text-decoration: none; }
.nav a:hover { text-decoration: underline; }
.nav .brand { font-weight: bold; font-size: 18px; }
.container { padding: 20px; max-width: 800px; margin: 0 auto; }
SANDBOX_CSS = """
.container { max-width: 800px; margin: 0 auto; }
h1 { color: #ff6b6b; margin-bottom: 5px; }
.subtitle { color: #666; margin-bottom: 30px; }
@ -4095,11 +4003,9 @@ SANDBOX_HTML = """
.success { color: #6f6; }
.error { color: #f66; }
</style>
</head>
<body>
<!-- NAV -->
"""
SANDBOX_CONTENT = """
<div class="container">
<h1>🧪 Sandbox Mode</h1>
<p class="subtitle">Upload a neopig archive (.tar.gz or .run) to explore it</p>
@ -4283,11 +4189,19 @@ SANDBOX_HTML = """
loadStatus();
</script>
</body>
</html>
"""
@app.get("/sandbox", response_class=HTMLResponse)
async def sandbox_page(lang: str = Cookie(None), accept_language: str = Header(None)):
"""Sandbox page for uploading archives."""
if not SANDBOX_MODE:
raise HTTPException(status_code=403, detail="Sandbox mode not enabled (NEOPIG_SANDBOX=1)")
language = get_lang(lang, accept_language)
html = layout("Sandbox", SANDBOX_CONTENT, extra_css=SANDBOX_CSS)
return inject_i18n(html, language)
# ============================================================================
# Crawler API
# ============================================================================