engineering/pig.py
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Port 31337 default, code file display fixes

Browse source 
- Change default port from 8000 to 31337 across all files
- About page: and → &, beast → pig
- Search results: SVG placeholder for code files (shows extension)
- Download button: remove .bin fallback, use original extension
This commit is contained in:
Russell Ballestrini 2026-01-02 15:35:39 -05:00
parent 47c0068cf1
commit 66d2f4d1d9
6 changed files with 171 additions and 93 deletions
Download patch file Download diff file Expand all files Collapse all files

6
CLAUDE.md
View file

@ -31,7 +31,7 @@ pip install -r requirements.txt
python neopig.py https://example.com --mode images
# Run SERP web interface
python serp.py --host 0.0.0.0 --port 8000
python serp.py --host 0.0.0.0 --port 31337
# Run tests
pytest tests/ -v
@ -193,7 +193,7 @@ tar -xzf example.com-20251229.tar.gz
cd example.com-20251229
pip install -r requirements.txt
python neopig/serp.py
# Open http://localhost:8000
# Open http://localhost:31337
# Option 2: Serve directly from tar.gz (no extraction)
python neopig/serp.py example.com-20251229.tar.gz
@ -206,7 +206,7 @@ make run TARBALL=example.com-20251229.tar.gz
# Run it (just needs python3 + fastapi/uvicorn)
./example.com-20251229.run
# Opens http://localhost:8000
# Opens http://localhost:31337
```
The `.run` file is a single executable containing:

4
Makefile
View file

@ -40,7 +40,7 @@ endif
echo "Created: $$OUTNAME ($$(stat -c%s $$OUTNAME 2>/dev/null || stat -f%z $$OUTNAME) bytes)"
serp: install
$(PYTHON) serp.py --host 0.0.0.0 --port 8000
$(PYTHON) serp.py --host 0.0.0.0 --port 31337
clean:
rm -rf $(VENV) __pycache__ *.pyc
@ -63,7 +63,7 @@ vendor-install: vendor-uri2png venv
# Combined server (SERP + screenshot)
server: vendor-install
$(PYTHON) serp.py --host 0.0.0.0 --port 8000
$(PYTHON) serp.py --host 0.0.0.0 --port 31337
# Test tarball extraction and run embedded neopig server
# Usage: make test-alpha TARBALL=example.tar.gz

8
README.md
View file

@ -65,7 +65,7 @@ pip install -r requirements.txt
python neopig.py https://example.com --mode images
# Run SERP web interface
python serp.py --host 0.0.0.0 --port 8000
python serp.py --host 0.0.0.0 --port 31337
# Archive a site into a distributable package
python archive.py https://discourse-urho3d.github.io/
@ -618,7 +618,7 @@ python neopig.py https://example.com --screenshot-engine wkhtmltoimage
python neopig.py --list-engines
# Run with live SERP viewer
python neopig.py https://example.com --serve --port 8000
python neopig.py https://example.com --serve --port 31337
# Backfill markdown for existing pages
python neopig.py --backfill-markdown example.com --trim-wrapper
@ -646,7 +646,7 @@ python archive.py https://example.com --depth 5 --max-pages 500
python archive.py https://example.com --no-screenshot
# Watch live while archiving
python archive.py https://example.com --serve --port 8000
python archive.py https://example.com --serve --port 31337
# Upgrade neopig in existing archive
python archive.py --upgrade-neopig example-20251231.tar.gz
@ -656,7 +656,7 @@ python archive.py --upgrade-neopig example-20251231.tar.gz
```bash
# Start web interface
python serp.py --host 0.0.0.0 --port 8000
python serp.py --host 0.0.0.0 --port 31337
# Custom database/vault
python serp.py --db data/neopig.db --vault data/vault

10
archive.py
View file

@ -279,6 +279,11 @@ class SiteArchiver:
# Write index.html
self._write_index_html(tmpdir_path, sitemap, domain)
# Close database BEFORE copying to ensure WAL is checkpointed
# (SQLite WAL mode keeps data in -wal file until close)
await pig.db.close()
logger.info("Database closed, WAL checkpointed")
# Step 3: Stream everything to tar.gz in one pass
logger.info("Streaming to archive...")
@ -378,9 +383,6 @@ class SiteArchiver:
# Clear html_contents to free memory
html_contents.clear()
# Close database connection
await pig.db.close()
final_size = tar_path.stat().st_size
logger.info(f"Archive complete: {tar_path} ({final_size / 1024 / 1024:.1f} MB)")
@ -570,7 +572,7 @@ async def main():
parser.add_argument("--fast", action="store_true", help="Fast mode: no crawl delay (for sites without robots.txt)")
parser.add_argument("--package-only", action="store_true", help="Skip crawling, just package existing data from vault")
parser.add_argument("--serve", action="store_true", help="Start SERP server to watch crawl live")
parser.add_argument("--port", type=int, default=8000, help="Port for SERP server (default: 8000)")
parser.add_argument("--port", type=int, default=31337, help="Port for SERP server (default: 31337)")
parser.add_argument("--backfill-markdown", metavar="DOMAIN", help="Re-process stored HTML for DOMAIN to regenerate markdown with absolute URLs")
parser.add_argument("--trim-wrapper", action="store_true", help="With --backfill-markdown: strip nav/header/footer/logo before conversion")
parser.add_argument("--db", default="data/neopig.db", help="Database path (for --backfill-markdown)")

4
neopig.py
View file

@ -2540,8 +2540,8 @@ async def main():
parser.add_argument(
"--port",
type=int,
default=8000,
help="Port for SERP server (default: 8000)"
default=31337,
help="Port for SERP server (default: 31337)"
)
parser.add_argument(

232
serp.py
View file

@ -1364,9 +1364,42 @@ SEARCH_CONTENT = """
} else {
mediaContainer.innerHTML = countDisplay + mediaResults.map(r => {
const isVideo = r.media_type === 'video';
const mediaEl = isVideo
? `<video src="/media/${r.md5_hash}" muted loop preload="metadata" onmouseenter="this.play()" onmouseleave="this.pause()"></video>`
: `<img src="/media/${r.md5_hash}" alt="${r.alt_text || ''}" loading="lazy">`;
const isCode = r.media_type === 'code';
// Get file extension from mime_type for code files
const getCodeExt = (mime) => {
const mimeToExt = {
'text/javascript': '.js', 'application/javascript': '.js',
'text/x-python': '.py', 'text/python': '.py',
'text/x-c': '.c', 'text/x-csrc': '.c',
'text/x-c++': '.cpp', 'text/x-c++src': '.cpp',
'text/x-java': '.java', 'text/java': '.java',
'text/x-ruby': '.rb', 'text/ruby': '.rb',
'text/x-go': '.go', 'text/go': '.go',
'text/x-rust': '.rs', 'text/rust': '.rs',
'text/x-php': '.php', 'text/php': '.php',
'text/x-shellscript': '.sh', 'text/x-sh': '.sh',
'text/css': '.css', 'text/html': '.html',
'text/xml': '.xml', 'application/xml': '.xml',
'application/json': '.json', 'text/json': '.json',
'text/x-yaml': '.yaml', 'text/yaml': '.yaml',
'text/markdown': '.md', 'text/x-markdown': '.md',
'text/plain': '.txt', 'text/x-typescript': '.ts',
};
return mimeToExt[mime] || (r.alt_text ? '.' + r.alt_text : '');
};
let mediaEl;
if (isVideo) {
mediaEl = `<video src="/media/${r.md5_hash}" muted loop preload="metadata" onmouseenter="this.play()" onmouseleave="this.pause()"></video>`;
} else if (isCode) {
const ext = getCodeExt(r.mime_type);
mediaEl = `<svg viewBox="0 0 100 100" style="width:100%;height:100%;background:#1a1a1a;border-radius:4px;">
<text x="50" y="55" text-anchor="middle" fill="#888" font-family="monospace" font-size="16" font-weight="bold">${ext}</text>
</svg>`;
} else {
mediaEl = `<img src="/media/${r.md5_hash}" alt="${r.alt_text || ''}" loading="lazy">`;
}
const keywords = JSON.parse(r.keywords || '[]');
const tagsHtml = keywords.map(k => `<span class="tag">${k}</span>`).join('');
@ -2544,10 +2577,27 @@ async def view_media_page(md5_hash: str, noai: bool = Query(False), lang: str =
# Hero: media element
is_video = media['media_type'] == 'video'
is_audio = media['media_type'] == 'audio'
is_code = media['media_type'] == 'code'
if is_video:
hero_html = f'<a href="/media/{md5_hash}" target="_blank"><video src="/media/{md5_hash}" controls muted loop style="max-width:100%;max-height:70vh;"></video></a>'
elif is_audio:
hero_html = f'<audio src="/media/{md5_hash}" controls></audio>'
elif is_code:
# Fetch code content and display with syntax highlighting
code_content = ""
lang_class = media.get('alt_text') or '' # language stored in alt_text
try:
vault_path = VAULT_PATH / hash_to_path(md5_hash)
# Find file with this hash
subdir = vault_path.parent
for f in subdir.iterdir():
if f.stem == md5_hash:
code_content = f.read_text(errors='replace')[:100000] # Limit size
break
except Exception:
code_content = "(Unable to load file content)"
escaped_code = html_module.escape(code_content)
hero_html = f'<pre style="max-height:70vh;overflow:auto;background:#1e1e1e;padding:15px;border-radius:8px;"><code class="language-{lang_class}">{escaped_code}</code></pre>'
else:
hero_html = f'<a href="/media/{md5_hash}" target="_blank"><img src="/media/{md5_hash}" alt="{html_module.escape(media.get("alt_text") or "")}" style="max-width:100%;max-height:70vh;"></a>'
@ -2571,7 +2621,7 @@ async def view_media_page(md5_hash: str, noai: bool = Query(False), lang: str =
("{{keywords_label}}", keywords_html),
]
# Download button
# Download button - get actual extension from vault file
name_source = media.get('alt_text') or media.get('title')
if not name_source and sources:
pt = sources[0].get('page_title', '')
@ -2579,8 +2629,26 @@ async def view_media_page(md5_hash: str, noai: bool = Query(False), lang: str =
name_source = f"{pt}-{media_idx}" if pt else f"media-{media_idx}"
download_name = slugify(name_source or f"media-{md5_hash[:8]}")
ext_map = {'image/jpeg': '.jpg', 'image/png': '.png', 'image/gif': '.gif', 'image/webp': '.webp',
'video/mp4': '.mp4', 'video/webm': '.webm', 'audio/mpeg': '.mp3', 'audio/wav': '.wav'}
ext = ext_map.get(media.get('mime_type', ''), '.bin')
'video/mp4': '.mp4', 'video/webm': '.webm', 'audio/mpeg': '.mp3', 'audio/wav': '.wav',
'text/javascript': '.js', 'text/python': '.py', 'text/css': '.css', 'text/html': '.html',
'text/markdown': '.md', 'text/rust': '.rs', 'text/go': '.go', 'text/c': '.c', 'text/cpp': '.cpp'}
ext = ext_map.get(media.get('mime_type', ''), '')
# If no extension from MIME, try to get from vault file
if not ext:
try:
subdir = VAULT_PATH / hash_to_path(md5_hash).parent
for f in subdir.iterdir():
if f.stem == md5_hash:
ext = f.suffix
break
except Exception:
pass
# Try original media_uri if still no extension
if not ext and media.get('media_uri'):
from pathlib import Path as P
orig_ext = P(media['media_uri'].split('?')[0]).suffix
if orig_ext and len(orig_ext) <= 5: # Reasonable extension length
ext = orig_ext
download_btn = f'<a href="/media/{md5_hash}?download=1" style="display:block;padding:12px 20px;background:#4a9eff;color:#fff;text-decoration:none;border-radius:6px;text-align:center;font-weight:500;margin-top:15px;">Download ({download_name}{ext})</a>'
# Get page media items (siblings)
@ -3493,7 +3561,7 @@ ABOUT_CONTENT = """
<h1 class="hero-title">neopig</h1>
<p class="hero-subtitle">Neo Python Image Grabber</p>
<p class="hero-tagline">
A chimera born from dead repositories. A beast that devours domains whole.
A chimera born from dead repositories. A pig that devours domains whole.
<br>Where others archive pages, we <em>consume realities</em>.
</p>
</section>
@ -3605,11 +3673,11 @@ ABOUT_CONTENT = """
</div>
</section>
<!-- CHAPTER IV: THE BEAST AWAKENS -->
<!-- CHAPTER IV: THE PIG AWAKENS -->
<section class="chapter chapter-beast">
<div class="chapter-number">IV</div>
<div class="chapter-content">
<h2 class="chapter-title">The Beast Awakens</h2>
<h2 class="chapter-title">The Pig Awakens</h2>
<p class="narrative">
Like a phoenix rising from dead Bitbucket repos, <strong>neopig</strong> emerged.
@ -3620,7 +3688,7 @@ ABOUT_CONTENT = """
</p>
<div class="dramatic-quote">
<p>A beast with an appetite like Gluttony from Fullmetal Alchemist &mdash; a homunculus that devours everything in its path, absorbing knowledge, power & form.</p>
<p>A pig with an appetite like Gluttony from Fullmetal Alchemist &mdash; a homunculus that devours everything in its path, absorbing knowledge, power & form.</p>
</div>
<p class="narrative">
@ -3632,7 +3700,7 @@ ABOUT_CONTENT = """
</p>
<p class="narrative">
<em>We are adding the ability to backup entire codebases.</em> The beast grows hungrier.
<em>We are adding the ability to backup entire codebases.</em> The pig grows hungrier.
</p>
</div>
</section>
@ -3656,15 +3724,15 @@ ABOUT_CONTENT = """
</p>
<div class="dramatic-quote">
<p>Our spider-pig gets out of the danger of the mirror void &mdash; where days are years &mdash; and exits absorbing everything like a griffin, like a chimera, like a beast that feeds on the bones of dead platforms.</p>
<p>Our spider-pig gets out of the danger of the mirror void &mdash; where days are years &mdash; and exits absorbing everything like a griffin, like a chimera, like a pig that feeds on the bones of dead platforms.</p>
</div>
<p class="narrative">
When a code hosting platform dies (and they all die eventually &mdash; Google Code, Gitorious, BitBucket's hg repos), neopig ensures the knowledge survives. We are the ark. We are the vault. We are the beast that remembers.
When a code hosting platform dies (and they all die eventually &mdash; Google Code, Gitorious, BitBucket's hg repos), neopig ensures the knowledge survives. We are the ark. We are the vault. We are the pig that remembers.
</p>
<p class="narrative">
The archives become self-extracting executables. <code>.run</code> files that contain their own server, their own search engine, their own reality. Drop one on a machine with Python, and it <em>wakes up</em>. A sleeping beast, carrying an entire website in its belly, ready to serve it to anyone who asks.
The archives become self-extracting executables. <code>.run</code> files that contain their own server, their own search engine, their own reality. Drop one on a machine with Python, and it <em>wakes up</em>. A sleeping pig, carrying an entire website in its belly, ready to serve it to anyone who asks.
</p>
</div>
</section>
@ -3705,7 +3773,7 @@ ABOUT_CONTENT = """
</div>
<div class="feature-card">
<h3>Self-Extracting Archives</h3>
<p>Create <code>.run</code> executables that contain entire websites. Drop on any machine, execute, browse. The beast travels light.</p>
<p>Create <code>.run</code> executables that contain entire websites. Drop on any machine, execute, browse. The pig travels light.</p>
</div>
<div class="feature-card">
<h3>Code File Support</h3>
@ -3806,7 +3874,7 @@ ABOUT_CONTENT = """
</p>
<p class="narrative">
The beast is imbued with <em>righteous fire</em>. It does not crawl politely because it fears the powerful &mdash; it respects robots.txt because it honors the wishes of creators. It does not hoard in darkness &mdash; it preserves in light, making archives searchable, shareable, <em>alive</em>.
The pig is imbued with <em>righteous fire</em>. It does not crawl politely because it fears the powerful &mdash; it respects robots.txt because it honors the wishes of creators. It does not hoard in darkness &mdash; it preserves in light, making archives searchable, shareable, <em>alive</em>.
</p>
<p class="narrative">
@ -3914,14 +3982,14 @@ ABOUT_CONTENT = """
</div>
</section>
<!-- CHAPTER X: INVOKE THE BEAST -->
<!-- CHAPTER X: INVOKE THE PIG -->
<section class="chapter chapter-beast">
<div class="chapter-number">X</div>
<div class="chapter-content">
<h2 class="chapter-title">Invoke the Beast</h2>
<h2 class="chapter-title">Invoke the Pig</h2>
<div class="code-section">
<pre><code># Archive a site & watch it live as the beast feeds
<pre><code># Archive a site & watch it live as the pig feeds
python neopig.py https://discourse-urho3d.github.io/ --serve
# Fast mode for static sites (no crawl delay, for sites without robots.txt)
@ -3936,7 +4004,7 @@ python neopig.py https://example.com --hydra
# Create a self-extracting executable
make run TARBALL=example.com-20260101.tar.gz
# Run the beast (just needs python3 + fastapi/uvicorn)
# Run the pig (just needs python3 + fastapi/uvicorn)
./example.com-20260101.run
# Opens http://localhost:31337 with full search</code></pre>
</div>
@ -3954,7 +4022,7 @@ make run TARBALL=example.com-20260101.tar.gz
<!-- FOOTER -->
<footer class="about-footer">
<p>neopig is open source, donated into the public domain.</p>
<p>The beast lives at:</p>
<p>The pig lives at:</p>
<a href="https://git.unturf.com/engineering/unturf/pig.py" target="_blank">git.unturf.com/engineering/unturf/pig.py</a>
<p style="margin-top: 40px; color: #444; font-size: 0.9em;">
In memory of all the repositories Atlassian killed.<br>
@ -4544,66 +4612,74 @@ async def import_archive_to_db(archive_path: Path, job_id: int) -> dict:
break
# Merge database
if temp_db.exists():
src = sqlite3.connect(temp_db)
src.row_factory = sqlite3.Row
try:
# Check what tables exist in source
tables = [r[0] for r in src.execute("SELECT name FROM sqlite_master WHERE type='table'")]
logger.info(f"Import: source database has tables: {tables}")
if not temp_db.exists():
raise ValueError("Malformed archive: missing neopig.db")
# Import media (if table exists)
if 'media' in tables:
for row in src.execute("SELECT * FROM media"):
try:
async with db.session() as session:
await session.execute(text(
"""INSERT OR IGNORE INTO media
(md5_hash, media_type, mime_type, file_size, keywords, alt_text, title,
first_seen_at, last_seen_at, score)
VALUES (:md5_hash, :media_type, :mime_type, :file_size, :keywords, :alt_text, :title,
:first_seen_at, :last_seen_at, :score)"""), dict(row))
await session.commit()
stats["media_imported"] += 1
except Exception as e:
logger.debug(f"Skip media row: {e}")
else:
logger.warning("Import: source has no 'media' table")
src = sqlite3.connect(temp_db)
src.row_factory = sqlite3.Row
try:
# Check what tables exist in source
tables = [r[0] for r in src.execute("SELECT name FROM sqlite_master WHERE type='table'")]
logger.info(f"Import: source database has tables: {tables}")
# Import media_sources (if table exists)
if 'media_sources' in tables:
for row in src.execute("SELECT * FROM media_sources"):
try:
async with db.session() as session:
await session.execute(text(
"""INSERT OR IGNORE INTO media_sources
(md5_hash, media_uri, page_uri, page_title, alt_text, searchable_text, discovered_at)
VALUES (:md5_hash, :media_uri, :page_uri, :page_title, :alt_text, :searchable_text, :discovered_at)"""), dict(row))
await session.commit()
stats["sources_imported"] += 1
except Exception as e:
logger.debug(f"Skip media_source row: {e}")
else:
logger.warning("Import: source has no 'media_sources' table")
# Validate archive has required tables
required_tables = {'pages', 'media', 'media_sources'}
if not required_tables.intersection(tables):
raise ValueError(f"Malformed archive: neopig.db has no data tables (found: {tables})")
# Import pages (if table exists)
if 'pages' in tables:
for row in src.execute("SELECT * FROM pages"):
try:
async with db.session() as session:
await session.execute(text(
"""INSERT OR IGNORE INTO pages
(uri, uri_hash, domain, title, content, markdown, crawled_at)
VALUES (:uri, :uri_hash, :domain, :title, :content, :markdown, :crawled_at)"""), dict(row))
await session.commit()
stats["pages_imported"] += 1
except Exception as e:
logger.debug(f"Skip page row: {e}")
else:
logger.warning("Import: source has no 'pages' table")
finally:
src.close()
temp_db.unlink(missing_ok=True)
# Import media (if table exists)
if 'media' in tables:
for row in src.execute("SELECT * FROM media"):
try:
async with db.session() as session:
await session.execute(text(
"""INSERT OR IGNORE INTO media
(md5_hash, media_type, mime_type, file_size, keywords, alt_text, title,
first_seen_at, last_seen_at, score)
VALUES (:md5_hash, :media_type, :mime_type, :file_size, :keywords, :alt_text, :title,
:first_seen_at, :last_seen_at, :score)"""), dict(row))
await session.commit()
stats["media_imported"] += 1
except Exception as e:
logger.debug(f"Skip media row: {e}")
else:
logger.warning("Import: source has no 'media' table")
# Import media_sources (if table exists)
if 'media_sources' in tables:
for row in src.execute("SELECT * FROM media_sources"):
try:
async with db.session() as session:
await session.execute(text(
"""INSERT OR IGNORE INTO media_sources
(md5_hash, media_uri, page_uri, page_title, alt_text, searchable_text, discovered_at)
VALUES (:md5_hash, :media_uri, :page_uri, :page_title, :alt_text, :searchable_text, :discovered_at)"""), dict(row))
await session.commit()
stats["sources_imported"] += 1
except Exception as e:
logger.debug(f"Skip media_source row: {e}")
else:
logger.warning("Import: source has no 'media_sources' table")
# Import pages (if table exists)
if 'pages' in tables:
for row in src.execute("SELECT * FROM pages"):
try:
row_dict = dict(row)
async with db.session() as session:
await session.execute(text(
"""INSERT OR REPLACE INTO pages
(uri, uri_hash, path, title, description, keywords, content, markdown, raw_html, crawled_at)
VALUES (:uri, :uri_hash, :path, :title, :description, :keywords, :content, :markdown, :raw_html, :crawled_at)"""), row_dict)
await session.commit()
stats["pages_imported"] += 1
except Exception as e:
logger.debug(f"Skip page row: {e}")
else:
logger.warning("Import: source has no 'pages' table")
finally:
src.close()
temp_db.unlink(missing_ok=True)
# Extract vault files
vault_prefix = f"{archive_root}/vault/"