diff --git a/CLAUDE.md b/CLAUDE.md index 170809d..8403dee 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,6 +48,8 @@ pytest tests/ -v Based on [pig.py](http://russell.ballestrini.net/python-image-grabber-pig-py/) by Russell Ballestrini. +**Source:** [git.unturf.com/engineering/unturf/pig.py](https://git.unturf.com/engineering/unturf/pig.py) (public domain) + ## Architecture ``` diff --git a/README.md b/README.md index 667d13b..1e5d9bb 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ Based on [pig.py](http://russell.ballestrini.net/python-image-grabber-pig-py/) by Russell Ballestrini +**Source:** [git.unturf.com/engineering/unturf/pig.py](https://git.unturf.com/engineering/unturf/pig.py) (public domain) + --- ## The Origin Story @@ -21,7 +23,21 @@ python pig.py https://www.foxhop.net That was it. Point it at a URL, and it would slurp down every image it could find. No configuration, no complexity — just a hungry little pig gobbling up pixels. -Over a decade later, **neopig** emerged as a spiritual successor — keeping the original's spirit of simplicity while adding the features needed for serious web archival. +### The Great Bitbucket Extinction + +Then came the dark times. The original source lived at ~~[bitbucket.org/russellballestrini/pig](https://bitbucket.org/russellballestrini/pig)~~ — click it, we dare you. In 2020, Atlassian swallowed Bitbucket whole and spat out a glorious fountain of Mercurial repositories into the void. The original pig.py, nestled in its cozy hg repo, was atomized in the great purge. + +But here's where it gets *weird*. + +Russell once wrote that ["programming is like alchemy — instead of exchanging matter, we programmers exchange time."](https://russell.ballestrini.net/programming-is-like-alchemy/) Programs are golems, familiar spirits, magical servants performing repetitive tasks. *"It is more accurate to group programs with technology than magic, but less fun."* + +And speaking of alchemy: [Marathon Fusion](https://phys.org/news/2025-07-marathon-fusion-mercury-gold-energy.html) discovered that tokamak breeding blankets — wrapped in Mercury-Lithium alloy, like pigs in a blanket — can transmute Mercury-198 into Gold-197 through chrysopoeia. Fast neutrons trigger (n, 2n) reactions; unstable mercury decays into stable gold within 64 hours. Two metric tons of gold per gigawatt. The alchemists' dream realized, wrapped in radioactive patience (17.7 years of cooling before you can touch your transmuted treasure). + +A golden goose born from the ashes of deprecated version control. The old pig was archived, but a new creature stirred in the digital depths... + +### The Evolution to neopig + +Like a phoenix rising from dead Bitbucket repos, **neopig** emerged — a chimera, a griffin, a more hungry and gluttonous beast than its predecessor ever dreamed of being. Where pig.py sipped politely from single pages, neopig *devours entire domains*. --- diff --git a/serp.py b/serp.py index 908f72a..95366e5 100644 --- a/serp.py +++ b/serp.py @@ -3120,162 +3120,853 @@ async def phantom_page(lang: str = Cookie(None), accept_language: str = Header(N ABOUT_CSS = """ -.about-content { - max-width: 800px; - margin: 0 auto; - padding: 40px 20px; - line-height: 1.8; - font-size: 17px; +.about-page { width: 100%; min-height: 100vh; background: #0a0a0f; } + +/* Hero Section */ +.hero-section { + width: 100%; + min-height: 100vh; + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + background: linear-gradient(180deg, #0a0a0f 0%, #1a0a1a 50%, #0f0a1a 100%); + position: relative; + overflow: hidden; + padding: 60px 20px; +} +.hero-section::before { + content: ''; + position: absolute; + top: 0; left: 0; right: 0; bottom: 0; + background: radial-gradient(ellipse at 50% 0%, rgba(255,50,100,0.15) 0%, transparent 60%), + radial-gradient(ellipse at 20% 80%, rgba(100,50,255,0.1) 0%, transparent 40%), + radial-gradient(ellipse at 80% 60%, rgba(255,100,50,0.08) 0%, transparent 40%); + pointer-events: none; +} +.hero-pig { + width: 280px; + height: auto; + filter: drop-shadow(0 0 60px rgba(255,100,100,0.5)) drop-shadow(0 0 120px rgba(255,50,100,0.3)); + animation: pulse-glow 4s ease-in-out infinite; + margin-bottom: 40px; +} +@keyframes pulse-glow { + 0%, 100% { filter: drop-shadow(0 0 60px rgba(255,100,100,0.5)) drop-shadow(0 0 120px rgba(255,50,100,0.3)); } + 50% { filter: drop-shadow(0 0 80px rgba(255,100,100,0.7)) drop-shadow(0 0 160px rgba(255,50,100,0.5)); } +} +.hero-title { + font-size: 6em; + font-weight: 900; + letter-spacing: -0.02em; + background: linear-gradient(135deg, #ff6b6b 0%, #ff8e53 50%, #ff6b9d 100%); + -webkit-background-clip: text; + -webkit-text-fill-color: transparent; + background-clip: text; + text-shadow: 0 0 80px rgba(255,100,100,0.5); + margin: 0; + text-align: center; +} +.hero-subtitle { + font-size: 1.4em; + color: #888; + margin-top: 20px; + font-weight: 300; + letter-spacing: 0.3em; + text-transform: uppercase; +} +.hero-tagline { + font-size: 1.1em; + color: #666; + margin-top: 30px; + max-width: 600px; + text-align: center; + line-height: 1.8; +} + +/* Chapter Sections */ +.chapter { + width: 100%; + padding: 120px 40px; + position: relative; +} +.chapter-content { + max-width: 900px; + margin: 0 auto; +} +.chapter-number { + font-size: 8em; + font-weight: 900; + color: rgba(255,100,100,0.08); + position: absolute; + top: 40px; + left: 40px; + line-height: 1; + font-family: 'Georgia', serif; +} +.chapter-title { + font-size: 3em; + font-weight: 800; + color: #ff6b6b; + margin-bottom: 40px; + position: relative; +} +.chapter-title::after { + content: ''; + display: block; + width: 100px; + height: 4px; + background: linear-gradient(90deg, #ff6b6b, transparent); + margin-top: 20px; +} + +/* Narrative Text */ +.narrative { + font-size: 1.3em; + line-height: 2; + color: #c0c0c0; + margin-bottom: 30px; +} +.narrative em { color: #ff8e53; font-style: italic; } +.narrative strong { color: #fff; font-weight: 600; } +.narrative a { color: #ff6b6b; text-decoration: none; border-bottom: 1px solid rgba(255,107,107,0.3); transition: all 0.3s; } +.narrative a:hover { color: #ff8e53; border-bottom-color: #ff8e53; } + +/* Dramatic Quote */ +.dramatic-quote { + padding: 60px; + margin: 60px 0; + background: linear-gradient(135deg, rgba(255,50,100,0.1) 0%, rgba(100,50,200,0.1) 100%); + border-left: 6px solid #ff6b6b; + position: relative; +} +.dramatic-quote::before { + content: '"'; + font-size: 12em; + color: rgba(255,100,100,0.15); + position: absolute; + top: -40px; + left: 20px; + font-family: 'Georgia', serif; + line-height: 1; +} +.dramatic-quote p { + font-size: 1.8em; + font-style: italic; + color: #e0e0e0; + margin: 0; + position: relative; + z-index: 1; +} +.dramatic-quote cite { + display: block; + margin-top: 30px; + font-size: 1em; + color: #888; + font-style: normal; +} + +/* Dead Link Styling */ +.dead-link { + text-decoration: line-through; + color: #666 !important; + cursor: not-allowed; + position: relative; +} +.dead-link::after { + content: ' [DEAD]'; + color: #ff4444; + font-size: 0.7em; + font-weight: bold; + text-decoration: none; +} + +/* Dark Chapter */ +.chapter-dark { + background: linear-gradient(180deg, #0a0a0f 0%, #150810 50%, #0a0a0f 100%); +} +.chapter-dark .chapter-number { color: rgba(255,50,50,0.1); } + +/* Rebirth Chapter - TempleOS cyan */ +.chapter-rebirth { + background: linear-gradient(180deg, #0a0a0f 0%, #0a1515 50%, #0a0a0f 100%); +} +.chapter-rebirth .chapter-title { color: #00FFFF; /* TempleOS cyan */ } +.chapter-rebirth .chapter-title::after { background: linear-gradient(90deg, #00FFFF, transparent); } +.chapter-rebirth .chapter-number { color: rgba(0,255,255,0.08); } + +/* Beast Section */ +.chapter-beast { + background: linear-gradient(180deg, #0a0a0f 0%, #1a0a05 50%, #0a0a0f 100%); +} +.chapter-beast .chapter-title { color: #ff8e53; } +.chapter-beast .chapter-title::after { background: linear-gradient(90deg, #ff8e53, transparent); } + +/* Feature Cards - Full Width Grid */ +.feature-section { + width: 100%; + padding: 100px 40px; + background: linear-gradient(180deg, #0a0a0f 0%, #0f0f18 50%, #0a0a0f 100%); +} +.feature-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); + gap: 30px; + max-width: 1400px; + margin: 0 auto; +} +.feature-card { + background: linear-gradient(145deg, rgba(30,30,40,0.8) 0%, rgba(20,20,30,0.9) 100%); + padding: 40px; + border-radius: 16px; + border: 1px solid rgba(255,100,100,0.1); + transition: all 0.4s ease; + position: relative; + overflow: hidden; +} +.feature-card::before { + content: ''; + position: absolute; + top: 0; left: 0; + width: 100%; height: 4px; + background: linear-gradient(90deg, #ff6b6b, #ff8e53, #4ecdc4); + opacity: 0; + transition: opacity 0.4s; +} +.feature-card:hover { + transform: translateY(-5px); + border-color: rgba(255,100,100,0.3); + box-shadow: 0 20px 60px rgba(0,0,0,0.4), 0 0 40px rgba(255,100,100,0.1); +} +.feature-card:hover::before { opacity: 1; } +.feature-card h3 { + font-size: 1.4em; + color: #ff6b6b; + margin-bottom: 15px; + font-weight: 700; +} +.feature-card p { + color: #999; + line-height: 1.7; + font-size: 1.05em; +} +.feature-card a { color: #4ecdc4; } + +/* Code Blocks */ +.code-section { + background: #0d0d12; + padding: 40px; + border-radius: 12px; + margin: 40px 0; + overflow-x: auto; + border: 1px solid #222; +} +.code-section pre { + margin: 0; + font-family: 'Fira Code', 'Monaco', monospace; + font-size: 1em; + line-height: 1.8; + color: #e0e0e0; +} +.code-section code { background: none; } + +/* Inline Code */ +.narrative code { + background: rgba(255,100,100,0.1); + padding: 3px 10px; + border-radius: 4px; + font-family: 'Fira Code', monospace; + font-size: 0.9em; + color: #ff8e53; +} + +/* Mirror Section - Special */ +.chapter-mirror { + background: linear-gradient(180deg, #0a0a0f 0%, #0a0a1f 30%, #100a1a 70%, #0a0a0f 100%); + position: relative; +} +.chapter-mirror::before { + content: ''; + position: absolute; + top: 0; left: 0; right: 0; bottom: 0; + background: repeating-linear-gradient( + 90deg, + transparent 0px, + transparent 100px, + rgba(100,100,255,0.02) 100px, + rgba(100,100,255,0.02) 101px + ); + pointer-events: none; +} + +/* The Forge - TempleOS background */ +.chapter-forge { + background: url('/static/vendor/templeos.png') center center no-repeat, + linear-gradient(180deg, #0a0a0f 0%, #0a1515 30%, #0a1515 70%, #0a0a0f 100%); + background-size: contain, cover; + position: relative; +} +.chapter-forge::before { + content: ''; + position: absolute; + top: 0; left: 0; right: 0; bottom: 0; + background: rgba(0, 0, 0, 0.75); + pointer-events: none; +} +.chapter-forge .chapter-content { + position: relative; + z-index: 1; +} +.chapter-forge .chapter-title { color: #00FFFF; /* TempleOS cyan */ } +.chapter-forge .chapter-title::after { background: linear-gradient(90deg, #00FFFF, transparent); } +.chapter-forge .chapter-number { color: rgba(0,255,255,0.1); } + +/* The Rule - 72 Benedictine principles */ +.rule-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(400px, 1fr)); + gap: 12px; + margin: 40px 0; +} +.rule-item { + font-size: 1.05em; + line-height: 1.6; + color: #b0b0b0; + padding: 12px 16px; + background: rgba(255,255,255,0.02); + border-radius: 6px; + border-left: 3px solid rgba(255,107,107,0.3); +} +.rule-item strong { + color: #ff6b6b; + font-weight: 700; +} +@media (max-width: 768px) { + .rule-grid { grid-template-columns: 1fr; } +} + +.chapter-mirror .chapter-title { color: #8b5cf6; } +.chapter-mirror .chapter-title::after { background: linear-gradient(90deg, #8b5cf6, transparent); } +.chapter-mirror .chapter-number { color: rgba(139,92,246,0.08); } + +/* Footer */ +.about-footer { + width: 100%; + padding: 80px 40px; + background: #050508; + text-align: center; +} +.about-footer p { + color: #666; + font-size: 1.1em; + margin-bottom: 20px; +} +.about-footer a { + color: #ff6b6b; + font-size: 1.2em; + text-decoration: none; + border-bottom: 2px solid rgba(255,107,107,0.3); + padding-bottom: 4px; + transition: all 0.3s; +} +.about-footer a:hover { + color: #ff8e53; + border-bottom-color: #ff8e53; +} + +/* Responsive */ +@media (max-width: 768px) { + .hero-title { font-size: 3.5em; } + .hero-pig { width: 200px; } + .chapter { padding: 80px 20px; } + .chapter-number { font-size: 5em; top: 20px; left: 20px; } + .chapter-title { font-size: 2em; } + .narrative { font-size: 1.1em; } + .dramatic-quote { padding: 30px; } + .dramatic-quote p { font-size: 1.3em; } + .feature-section { padding: 60px 20px; } } -.about-content h1 { font-size: 2.5em; margin-bottom: 10px; color: #ff6b6b; } -.about-content h2 { font-size: 1.6em; margin-top: 40px; margin-bottom: 15px; color: #4ecdc4; border-bottom: 2px solid #333; padding-bottom: 10px; } -.about-content p { margin-bottom: 20px; color: #e0e0e0; } -.about-content a { color: #ff6b6b; text-decoration: none; } -.about-content a:hover { text-decoration: underline; } -.about-content code { background: #2a2a2a; padding: 2px 8px; border-radius: 4px; font-family: 'Fira Code', monospace; font-size: 0.9em; } -.about-content pre { background: #1a1a1a; padding: 20px; border-radius: 8px; overflow-x: auto; border: 1px solid #333; } -.about-content pre code { background: none; padding: 0; } -.about-content ul, .about-content ol { margin-bottom: 20px; padding-left: 30px; } -.about-content li { margin-bottom: 10px; color: #e0e0e0; } -.feature-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); gap: 20px; margin: 30px 0; } -.feature-card { background: #1e1e24; padding: 20px; border-radius: 12px; border: 1px solid #333; } -.feature-card h3 { color: #ff6b6b; margin-bottom: 10px; font-size: 1.1em; } -.feature-card p { color: #aaa; font-size: 0.95em; margin: 0; } -.origin-quote { background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%); border-left: 4px solid #ff6b6b; padding: 20px 25px; margin: 30px 0; border-radius: 0 8px 8px 0; font-style: italic; } -.origin-quote cite { display: block; margin-top: 10px; color: #888; font-style: normal; } """ ABOUT_CONTENT = """ -
-

🐷 neopig

-

Neo Python Image Grabber — A full-domain async media crawler with content-addressable storage, full-text search, and page screenshot capture.

+
-

The Origin Story

+ +
+ pig.py +

neopig

+

Neo Python Image Grabber

+

+ A chimera born from dead repositories. A beast that devours domains whole. +
Where others archive pages, we consume realities. +

+
-
- "pig.py is a very simple python command line tool to download all the images from a given uri." - — Russell Ballestrini, August 22, 2011 + +
+
I
+
+

Genesis

+ +
+

pig.py is a very simple python command line tool to download all the images from a given uri.

+ — Russell Ballestrini, August 22, 2011 +
+ +

+ In the beginning, there was pig.py. A simple creature. Innocent. Hungry. +

+ +

+ Russell Ballestrini crafted it in the summer of 2011 — a humble Python script with a singular appetite: images. Point it at a webpage, and it would slurp down every pixel it could find. No configuration. No complexity. Just a hungry little pig gobbling up the visual fabric of the web. +

+ +
+ 
python pig.py https://www.foxhop.net
+
+ +

+ That was it. The entire interface. The pig asked only for a target, and it fed. +

+ +

+ Russell released it into the public domain — a gift to anyone who needed to harvest images from the wild web. The source lived at bitbucket.org/russellballestrini/pig, nestled safely in a Mercurial repository. The pig slept soundly in its pen, unaware of the extinction event approaching. +

+
-

Back in 2011, Russell Ballestrini created pig.py — a delightfully simple Python script that did one thing well: download all the images from a webpage. It was placed in the public domain, a gift to anyone who needed to grab images from the web.

+ +
+
II
+
+

The Great Bitbucket Extinction

-

The original pig.py was elegant in its simplicity:

+

+ Then came the dark times. +

- 
python pig.py https://www.foxhop.net
+

+ In 2020, Atlassian — the corporate leviathan that had swallowed Bitbucket whole — decreed that Mercurial repositories would be purged. Millions of hg repos, atomized. Scattered like digital ash across the void. +

-

That was it. Point it at a URL, and it would slurp down every image it could find. No configuration, no complexity — just a hungry little pig gobbling up pixels.

+

+ The original pig.py, that innocent image-gobbling creature, was among the casualties. Its home at bitbucket.org/russellballestrini/pig became a tombstone. Click it. We dare you. There's nothing there but the echo of corporate indifference. +

-

The Great Bitbucket Extinction

- -

Then came the dark times. The original source lived at bitbucket.org/russellballestrini/pig — click it, we dare you. In 2020, Atlassian swallowed Bitbucket whole and spat out a glorious fountain of Mercurial repositories into the void. The original pig.py, nestled in its cozy hg repo, was atomized in the great purge.

- -

But here's where it gets weird.

- -

Russell once wrote that "programming is like alchemy — instead of exchanging matter, we programmers exchange time." Programs are golems, familiar spirits, magical servants performing repetitive tasks. "It is more accurate to group programs with technology than magic, but less fun."

- -

And speaking of alchemy: Marathon Fusion discovered that tokamak breeding blankets — wrapped in Mercury-Lithium alloy, like pigs in a blanket — can transmute Mercury-198 into Gold-197 through chrysopoeia. Fast neutrons trigger (n, 2n) reactions; unstable mercury decays into stable gold within 64 hours. Two metric tons of gold per gigawatt. The alchemists' dream realized, wrapped in radioactive patience (17.7 years of cooling before you can touch your transmuted treasure).

- -

A golden goose born from the ashes of deprecated version control. The old pig was archived, but a new creature stirred in the digital depths...

- -

The Evolution to neopig

- -

Like a phoenix rising from dead Bitbucket repos, neopig emerged — a chimera, a griffin, a more hungry and gluttonous beast than its predecessor ever dreamed of being. Where pig.py sipped politely from single pages, neopig devours entire domains.

- -

Over a decade later, this spiritual successor keeps the original's spirit of simplicity while adding the features needed for serious web archival:

- -
-
-

🔄 Async Crawling

-

Full-domain recursive crawling with configurable depth, respecting robots.txt and crawl delays.

+
+

They literally killed pig.py and sent him to bitbucket.

+ — The cruel irony of naming your graveyard after a slang term for death
-
-

💾 Content-Addressed Storage

-

MD5-based deduplication in a 9-deep vault. Same image from 100 pages? Stored once.

+ +

+ "Sent to the bitbucket" — programmer slang from the age of punch cards. When you discarded bad data, you threw it in the bit bucket. Atlassian named their service after a trash can, then proved the prophecy true by throwing away everyone's code. +

+ +

+ The pig was dead. Its bones scattered across cached search results and archived blog posts. A ghost in the machine, referenced but unreachable. +

+ +

+ But here's where it gets weird. +

+
+
+ + +
+
III
+
+

The Alchemy

+ +

+ Russell once wrote that "programming is like alchemy — instead of exchanging matter, we programmers exchange time." +

+ +

+ Programs are golems. Familiar spirits. Magical servants performing repetitive tasks so we don't have to. "It is more accurate to group programs with technology than magic, but less fun." +

+ +

+ Years passed. The web continued its relentless churn — sites going dark, forums shutting down, communities scattering like startled birds. Somewhere, a very angry ex-Ruby developer (Python too) watched the digital decay and remembered the pig. +

+ +
+

What if we could transmute that dead code into gold?

+ — The thought that started everything
-
-

🔍 Full-Text Search

-

Every image indexed with its surrounding context — page title, alt text, captions, nearby headings.

+ +

+ And speaking of transmutation: Marathon Fusion discovered that tokamak breeding blankets — wrapped in Mercury-Lithium alloy, like pigs in a blanket — can transmute Mercury-198 into Gold-197 through chrysopoeia. Fast neutrons trigger (n, 2n) reactions; unstable mercury decays into stable gold within 64 hours. +

+ +

+ Two metric tons of gold per gigawatt. The alchemists' dream realized, wrapped in radioactive patience (17.7 years of cooling before you can touch your transmuted treasure). +

+ +

+ A golden goose born from the ashes of deprecated version control. The old pig was archived, but a new creature stirred in the digital depths... +

+
+
+ + +
+
IV
+
+

The Beast Awakens

+ +

+ Like a phoenix rising from dead Bitbucket repos, neopig emerged. +

+ +

+ Not a resurrection. Not a mere fork. Something else. A chimera. A griffin. A creature stitched together from the DNA of the original pig and the fevered dreams of someone who had watched too many sites die. +

+ +
+

A beast with an appetite like Gluttony from Fullmetal Alchemist — a homunculus that devours everything in its path, absorbing knowledge, power, and form.

-
-

📸 Page Screenshots

-

LinkPeek reborn! Full-page captures via uri2png (wkhtmltoimage, Playwright, etc).

+ +

+ Where pig.py sipped politely from single pages, neopig devours entire domains. It doesn't just grab images — it consumes HTML, screenshots pages, converts content to markdown, indexes every scrap of text, and stores it all in content-addressed vaults that will outlast the heat death of the original servers. +

+ +

+ The code lives now at git.unturf.com/engineering/unturf/pig.py. And it keeps evolving. Every crawl adds new capabilities. Every archived site teaches it new tricks. +

+ +

+ We are adding the ability to backup entire codebases. The beast grows hungrier. +

+
+
+ + +
+
V
+
+

The Mirror Dimension

+ +

+ We are about to invert the git tree into itself. +

+ +

+ Like Dr. Strange versus Spider-Man in the mirror dimension — that impossible space where geometry folds back on itself, where buildings become M.C. Escher nightmares and reality reflects endlessly into its own depths. +

+ +

+ neopig doesn't just archive websites. It can archive itself. It can archive the repositories that contain the code that does the archiving. It can crawl git forges and preserve the source code of tools that preserve source code. +

+ +
+

Our spider-pig gets out of the danger of the mirror void — where days are years — and exits absorbing everything like a griffin, like a chimera, like a beast that feeds on the bones of dead platforms.

-
-

🐙 Hydra Mode

-

Auto-discover RSS/Atom/Sitemap feeds. Persists feed URLs, checks for new content on every crawl.

-
-
-

📊 Media Provenance

-

Track first_seen, last_seen, and when each page first linked to media. See reuse across the web.

-
-
-

📝 Markdown Conversion

-

Intelligent HTML-to-Markdown with forum post detection, noise removal, and content extraction.

-
-
-

📦 Distributable Archives

-

Create self-contained tar.gz packages with embedded search servers for offline browsing.

+ +

+ When a code hosting platform dies (and they all die eventually — Google Code, Gitorious, BitBucket's hg repos), neopig ensures the knowledge survives. We are the ark. We are the vault. We are the beast that remembers. +

+ +

+ The archives become self-extracting executables. .run files that contain their own server, their own search engine, their own reality. Drop one on a machine with Python and Pyramid, and it wakes up. A sleeping beast, carrying an entire website in its belly, ready to serve it to anyone who asks. +

+
+
+ + +
+
+

The Arsenal

+ +
+
+

Async Domain Crawling

+

Full-domain recursive crawling with configurable depth. Respects robots.txt and crawl delays. Polite but relentless.

+
+
+

Content-Addressed Vault

+

MD5-based deduplication in a 9-deep hex directory structure. Same image from 100 pages? Stored once. Space is memory.

+
+
+

Full-Text Search

+

Every image indexed with its surrounding context — page title, alt text, captions, nearby headings. Find images by meaning, not filename.

+
+
+

Page Screenshots

+

LinkPeek reborn! Full-page captures via uri2png. wkhtmltoimage, cutycapt, or Playwright backends.

+
+
+

Hydra Mode

+

Auto-discover RSS/Atom/Sitemap feeds. Persists feed URLs, checks for new content on every crawl. Self-healing archive maintenance.

+
+
+

Media Provenance

+

Track first_seen, last_seen, and when each page first linked to media. Watch content spread across the web over time.

+
+
+

Markdown Conversion

+

Intelligent HTML-to-Markdown with forum post detection, noise removal, and content extraction. Human-readable archives.

+
+
+

Self-Extracting Archives

+

Create .run executables that contain entire websites. Drop on any machine, execute, browse. The beast travels light.

+
+
+

Code File Support

+

Archive source code, stylesheets, fonts, and scripts alongside media. Complete preservation for developer communities.

+
+
-

The Skeleton Key Approach

+ +
+
VI
+
+

The Skeleton Key

-

neopig uses what we call the "skeleton key" approach to media indexing. For every image, we capture all the text that might help you find it later:

+

+ neopig uses what we call the "skeleton key" approach to media indexing. For every image, we capture all the text that might help you find it later: +

-
    -
  • Page context: Title, description, headings near the image
    • -
  • Image attributes: Alt text, title, surrounding captions
    • -
  • Link context: The text of links pointing to the image
    • -
  • Detail pages: For gallery sites, we follow through to detail pages
    • -
+

+ Page context: Title, description, headings near the image.
+ Image attributes: Alt text, title, surrounding captions.
+ Link context: The text of links pointing to the image.
+ Detail pages: For gallery sites, we follow through to detail pages and harvest their metadata too. +

-

The result? You can search for "sunset over mountains" and find that image even if it was named IMG_4372.jpg with no alt text — because the page title mentioned it, or someone linked to it with descriptive text.

+

+ The result? You can search for "sunset over mountains" and find that image even if it was named IMG_4372.jpg with no alt text — because the page title mentioned it, or someone linked to it with descriptive text, or a caption three divs away contained the words. +

-

Preserving the Web

+

+ Every door opens. Every search succeeds. The skeleton key fits every lock because it carries every possible key inside it. +

+
+
-

The web is ephemeral. Sites go dark. Forums shut down. Communities scatter. neopig is built for preservation — capturing not just the media, but the context that gives it meaning.

+ +
+
VII
+
+

Why We Archive

-

When you archive a site with neopig, you get:

+

+ The web is ephemeral. Sites go dark. Forums shut down. Communities scatter. Platforms get acquired and gutted. Executives decide that Mercurial isn't profitable and delete millions of repositories with a single corporate memo. +

-
    -
  • Original HTML with rewritten media links pointing to your local vault
    • -
  • Full-text searchable markdown versions of every page
    • -
  • Screenshots showing exactly how pages looked
    • -
  • A SQLite database you can query, backup, and migrate
    • -
  • Self-extracting archives that work offline forever
    • -
+

+ We have watched beloved communities vanish. Game modding forums. Technical documentation. Art galleries. Fan wikis. Personal blogs with decades of writing. Gone. Not archived by the Wayback Machine. Not cached by Google. Just gone. +

-

The Name

+

+ neopig is built for preservation — capturing not just the media, but the context that gives it meaning. When you archive a site with neopig, you get: +

-

neopig = Neo (new) + Python Image Grabber

+

+ • Original HTML with rewritten media links pointing to your local vault
+ • Full-text searchable markdown versions of every page
+ • Screenshots showing exactly how pages looked
+ • A SQLite database you can query, backup, and migrate forever
+ • Self-extracting archives that work offline until the heat death of the universe +

+
+
-

A tip of the hat to the original pig.py, with a nod to the Matrix's Neo — seeing through the surface of the web to the underlying content within.

+ +
+
VIII
+
+

The Forge

-

Get Started

+

+ This tool was not built for profit. It was forged. +

- 
# Archive a site and serve it live as you crawl
-python neopig.py https://discourse-urho3d.github.io --archive --serve
+            

+                Forged in the fires of permacomputer.com — the belief that computing should be permanent. That knowledge should not evaporate when a company pivots. That communities should not lose their histories when a platform gets acquired. That the digital commons belongs to everyone, forever.
+            

 
-# Fast mode for static sites (no crawl delay)
-python neopig.py https://russell.ballestrini.net --archive --fast -o ./archives/
+            

+                
Truth. Harmony. Freedom. Love.

+                — The four pillars of the permacomputer
+            

 
-# Hydra mode: discover RSS/Atom/Sitemap feeds for auto-updates
+            

+                Truth — We preserve what actually existed, not sanitized versions. The original HTML. The exact timestamps. The unedited conversations. History as it happened, not as someone wishes it had.
+            

+
+            

+                Harmony — Every piece connects. Media links to pages. Pages link to context. Context links to meaning. The archive is not a graveyard of disconnected files but a living web of relationships, searchable, traversable, whole.
+            

+
+            

+                Freedom — Public domain. No licenses to parse. No corporate terms of service. Take it. Fork it. Run it. Share it. The code is yours. The archives you create are yours. No one can revoke your access to your own preserved knowledge.
+            

+
+            

+                Love — We archive because we care. About the communities that built these spaces. About the people who poured years into forum posts and wiki pages and blog entries. About the future researchers who will want to understand how we lived and thought and created. This is an act of love for human knowledge.
+            

+
+            

+                The beast is imbued with righteous fire. It does not crawl politely because it fears the powerful — it respects robots.txt because it honors the wishes of creators. It does not hoard in darkness — it preserves in light, making archives searchable, shareable, alive.
+            

+
+            

+                When the platforms fall (and they will fall), when the corporate servers go dark (and they will go dark), when the executives decide your community isn't profitable enough to maintain (and they will decide this) — the permacomputer endures. The archives persist. The knowledge survives.
+            

+
+            

+                This is what we were forged to do.
+            

+
+
+ + +
+
IX
+
+

The Rule

+ +

+ neopig is built on SQLite — the most deployed database in existence. SQLite adopted the Rule of St. Benedict as their Code of Ethics. 72 principles from a 6th-century monastery, governing software in the 21st. +

+ +

+ We inherit this lineage. All 72 rules, unabridged: +

+ +
+
1. Love the Lord God with your whole heart, soul, and strength.
+
2. Love your neighbor as yourself.
+
3. Do not murder.
+
4. Do not commit adultery.
+
5. Do not steal.
+
6. Do not covet.
+
7. Do not bear false witness.
+
8. Honor all people.
+
9. Do not do to another what you would not have done to yourself.
+
10. Deny oneself in order to follow Christ.
+
11. Chastise the body.
+
12. Do not become attached to pleasures.
+
13. Love fasting.
+
14. Relieve the poor.
+
15. Clothe the naked.
+
16. Visit the sick.
+
17. Bury the dead.
+
18. Be a help in times of trouble.
+
19. Console the sorrowing.
+
20. Be a stranger to the world's ways.
+
21. Prefer nothing more than the love of Christ.
+
22. Do not give way to anger.
+
23. Do not nurse a grudge.
+
24. Do not entertain deceit in your heart.
+
25. Do not give a false peace.
+
26. Do not forsake charity.
+
27. Do not swear, for fear of perjuring yourself.
+
28. Utter only truth from heart and mouth.
+
29. Do not return evil for evil.
+
30. Do no wrong to anyone, and bear patiently wrongs done to yourself.
+
31. Love your enemies.
+
32. Do not curse those who curse you, but rather bless them.
+
33. Bear persecution for justice's sake.
+
34. Be not proud.
+
35. Be not addicted to wine.
+
36. Be not a great eater.
+
37. Be not drowsy.
+
38. Be not lazy.
+
39. Be not a grumbler.
+
40. Be not a detractor.
+
41. Put your hope in God.
+
42. Attribute to God, and not to self, whatever good you see in yourself.
+
43. Recognize always that evil is your own doing, and to impute it to yourself.
+
44. Fear the Day of Judgment.
+
45. Be in dread of hell.
+
46. Desire eternal life with all the passion of the spirit.
+
47. Keep death daily before your eyes.
+
48. Keep constant guard over the actions of your life.
+
49. Know for certain that God sees you everywhere.
+
50. When wrongful thoughts come into your heart, dash them against Christ immediately.
+
51. Disclose wrongful thoughts to your spiritual mentor.
+
52. Guard your tongue against evil and depraved speech.
+
53. Do not love much talking.
+
54. Speak no useless words or words that move to laughter.
+
55. Do not love much or boisterous laughter.
+
56. Listen willingly to holy reading.
+
57. Devote yourself frequently to prayer.
+
58. Daily in your prayers, with tears and sighs, confess your past sins to God, and amend them for the future.
+
59. Fulfill not the desires of the flesh; hate your own will.
+
60. Obey in all things the commands of those whom God has placed in authority over you even though they should act otherwise, mindful of the Lord's precept, "Do what they say, but not what they do."
+
61. Do not wish to be called holy before one is holy; but first to be holy, that you may be truly so called.
+
62. Fulfill God's commandments daily in your deeds.
+
63. Love chastity.
+
64. Hate no one.
+
65. Be not jealous, nor harbor envy.
+
66. Do not love quarreling.
+
67. Shun arrogance.
+
68. Respect your seniors.
+
69. Love your juniors.
+
70. Pray for your enemies in the love of Christ.
+
71. Make peace with your adversary before the sun sets.
+
72. Never despair of God's mercy.
+
+ +

+ We archive not to hoard, but to give. We preserve not to control, but to liberate. The database that holds your memories runs on a foundation of ancient wisdom — and so do we. +

+
+
+ + +
+
X
+
+

Invoke the Beast

+ +
+ 
# Archive a site and watch it live as the beast feeds
+python archive.py https://discourse-urho3d.github.io/ --serve
+
+# Fast mode for static sites (no crawl delay, for sites without robots.txt)
+python archive.py https://russell.ballestrini.net --fast -o ./archives/
+
+# Full domain slurp with unlimited depth
+python neopig.py https://example.com --mode all --depth -1
+
+# Hydra mode: discover and persist RSS/Atom/Sitemap feeds
 python neopig.py https://example.com --hydra
 
-# Re-run with fresh markdown conversion
-python neopig.py --fresh --backfill-markdown example.com
+# Create a self-extracting executable
+make run TARBALL=example.com-20260101.tar.gz
 
-# Purge a job and ALL its data (media, screenshots, pages)
-python neopig.py --purge-job 123
+# Run the beast (just needs python3 + pyramid)
+./example.com-20260101.run
+# Opens http://localhost:6543 with full search
+
-# Upgrade an existing archive with latest neopig/serp -python neopig.py --upgrade-neopig ./archives/example.com-20251231.tar.gz +

+ neopig = Neo (new) + Python Image Grabber +

-

- neopig is open source. The original pig.py was placed in the public domain by Russell Ballestrini. - This project continues that tradition of building useful tools for the community. +

+ A tip of the hat to the original pig.py. A nod to the Matrix's Neo — seeing through the surface of the web to the underlying content within. And perhaps a warning: this pig has grown teeth. +

+
+
+ + +
+ + +
""" diff --git a/tests/unit/test_async_web_fetcher.py b/tests/unit/test_async_web_fetcher.py index 59df9d6..b117dcc 100644 --- a/tests/unit/test_async_web_fetcher.py +++ b/tests/unit/test_async_web_fetcher.py @@ -140,10 +140,16 @@ class TestMediaTypeDetection: assert get_media_type_from_extension("https://example.com/audio.wav") == "audio" assert get_media_type_from_extension("https://example.com/audio.ogg") == "audio" + def test_get_media_type_from_extension_code(self): + """Test detecting code from extension.""" + assert get_media_type_from_extension("https://example.com/file.txt") == "code" + assert get_media_type_from_extension("https://example.com/script.py") == "code" + assert get_media_type_from_extension("https://example.com/app.js") == "code" + def test_get_media_type_from_extension_unknown(self): """Test unknown extension.""" - assert get_media_type_from_extension("https://example.com/file.txt") is None assert get_media_type_from_extension("https://example.com/page.html") is None + assert get_media_type_from_extension("https://example.com/file.xyz") is None def test_get_media_type_from_mime_image(self): """Test detecting image from MIME type.""" @@ -161,10 +167,16 @@ class TestMediaTypeDetection: assert get_media_type_from_mime("audio/mpeg") == "audio" assert get_media_type_from_mime("audio/wav") == "audio" + def test_get_media_type_from_mime_code(self): + """Test detecting code from MIME type.""" + assert get_media_type_from_mime("application/json") == "code" + assert get_media_type_from_mime("text/plain") == "code" + assert get_media_type_from_mime("application/javascript") == "code" + def test_get_media_type_from_mime_unknown(self): """Test unknown MIME type.""" assert get_media_type_from_mime("text/html") is None - assert get_media_type_from_mime("application/json") is None + assert get_media_type_from_mime("application/octet-stream") is None class TestMediaExtraction: