engineering/pig.py
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add self-extracting archives with bundled neopig

Browse source 
- Bootstrap C program extracts serve.py and runs from tarball
- Archives now include neopig source files for self-contained crawling
- --upgrade-neopig flag to update neopig in existing archives
- html2md.py: smart HTML-to-markdown converter for forums/blogs/Q&A
- Fix vault path defaults (data/vault instead of vault)
- Streaming tar.gz creation without temp copies
- URL rewriting for local media references in archives
This commit is contained in:
Russell Ballestrini 2025-12-30 06:55:20 -05:00
parent a1a58fbb50
commit 416b3cb760
10 changed files with 2980 additions and 1185 deletions
Download patch file Download diff file Expand all files Collapse all files

4
CLAUDE.md
View file

@ -8,6 +8,10 @@
**Always use `uri`, never `url`.** This applies to variable names, function names, column names, and comments. URI is the correct term (Uniform Resource Identifier).
## Database Rules
**NEVER write raw SQL strings.** Always use SQLAlchemy ORM with proper model queries. No `text()`, no f-strings with SQL, no string concatenation for queries. Only exception: comments explicitly stating raw SQL is allowed (e.g., for FTS5 virtual tables).
## Quick Start
```bash

438
archive.py
View file

@ -78,10 +78,14 @@ def url_to_path(url: str) -> str:
return f"{path}/index.html"
def html_to_markdown(html: str, base_url: str = '') -> str:
def html_to_markdown(html: str, base_url: str = '', trim_wrapper: bool = False) -> str:
"""Convert HTML to markdown."""
if not HAS_HTML2TEXT:
return html
# Optionally strip nav/header/footer/logo before conversion
if trim_wrapper:
from neopig import trim_html_wrapper
html = trim_html_wrapper(html)
h = html2text.HTML2Text()
h.ignore_links = False
h.ignore_images = False
@ -106,12 +110,14 @@ class SiteArchiver:
include_markdown: bool = True,
screenshot_config: ScreenshotConfig = None,
fast_mode: bool = False,
trim_wrapper: bool = False,
):
self.output_dir = Path(output_dir)
self.include_screenshots = include_screenshots
self.include_markdown = include_markdown and HAS_HTML2TEXT
self.screenshot_config = screenshot_config or ScreenshotConfig(enabled=include_screenshots)
self.fast_mode = fast_mode
self.trim_wrapper = trim_wrapper
async def archive(
self,
@ -147,9 +153,20 @@ class SiteArchiver:
vault_path=vault_path,
screenshot_config=self.screenshot_config,
fast_mode=self.fast_mode,
trim_wrapper=self.trim_wrapper,
)
await pig.init()
# Load previously crawled media/screenshots from DB (source of truth for resume)
crawled_media = await pig.db.get_crawled_media_uris()
crawled_screenshots = await pig.db.get_crawled_screenshot_uris()
logger.info(f"Resume state: {len(crawled_media)} media URIs, {len(crawled_screenshots)} screenshotted pages in DB")
if crawled_media:
pig.seen_media = crawled_media
if crawled_screenshots:
pig.seen_screenshots = crawled_screenshots
logger.info(f"Screenshots enabled: {pig.screenshot_config.enabled}")
# Run the crawl
stats = await pig.crawl(
target_uri=target_url,
@ -160,107 +177,66 @@ class SiteArchiver:
)
# Now package the results
logger.info("Packaging archive...")
logger.info("Packaging archive (streaming mode)...")
with tempfile.TemporaryDirectory() as tmpdir:
archive_root = Path(tmpdir) / archive_name
archive_root.mkdir(parents=True)
tar_path = self.output_dir / f"{archive_name}.tar.gz"
html_vault_path = Path(vault_path) / 'html_vault' / domain
media_vault_path = Path(vault_path) / 'media_vault' / domain
linkpeek_vault_path = Path(vault_path) / 'linkpeek_vault' / domain
# Create subdirectories
html_dir = archive_root / 'html'
media_dir = archive_root / 'media'
html_dir.mkdir()
media_dir.mkdir()
# Get URL-to-hash mapping for rewriting external URLs to local copies
logger.info("Loading media URL mappings...")
url_to_hash = await pig.db.get_all_media_uri_mappings()
logger.info(f"Loaded {len(url_to_hash)} URL mappings for rewriting")
if self.include_markdown:
md_dir = archive_root / 'markdown'
md_dir.mkdir()
def rewrite_urls(html_content: str) -> str:
"""Rewrite external image/media URLs to local archive paths."""
import re
def replace_url(match):
url = match.group(1)
if url in url_to_hash:
md5 = url_to_hash[url]
# Get extension from original URL
ext = Path(url.split('?')[0]).suffix or '.bin'
return match.group(0).replace(url, f'../media/{md5}{ext}')
return match.group(0)
if self.include_screenshots:
screenshots_dir = archive_root / 'screenshots'
screenshots_dir.mkdir()
# Replace src="url" and href="url" patterns
html_content = re.sub(r'src=["\']([^"\']+)["\']', replace_url, html_content)
html_content = re.sub(r'href=["\']([^"\']+\.(jpg|jpeg|png|gif|webp|mp4|webm|svg))["\']',
replace_url, html_content, flags=re.IGNORECASE)
return html_content
# Read pages from neopig's html vault
html_vault_path = Path(vault_path) / 'html_vault' / domain
sitemap = []
# Step 1: Scan HTML files to build sitemap and search index (no copying)
logger.info("Building search index...")
sitemap = []
html_contents = {}
if html_vault_path.exists():
for html_file in html_vault_path.rglob('*.html'):
rel_path = html_file.relative_to(html_vault_path)
html_content = html_file.read_text(encoding='utf-8', errors='replace')
if html_vault_path.exists():
for html_file in html_vault_path.rglob('*.html'):
rel_path = html_file.relative_to(html_vault_path)
try:
content = html_file.read_text(encoding='utf-8', errors='replace')
# Rewrite external URLs to local copies
content = rewrite_urls(content)
title = self._extract_title(content) or str(rel_path)
sitemap.append({'path': f'html/{rel_path}', 'title': title})
html_contents[str(rel_path)] = content
except Exception:
pass
# Write HTML
dest_path = html_dir / rel_path
await aiofiles.os.makedirs(dest_path.parent, exist_ok=True)
async with aiofiles.open(dest_path, 'w', encoding='utf-8') as f:
await f.write(html_content)
# Step 2: Create generated files in small temp dir
local_tmpdir = self.output_dir / '.tmp'
local_tmpdir.mkdir(parents=True, exist_ok=True)
# Write markdown
if self.include_markdown:
md_path = md_dir / str(rel_path).replace('.html', '.md')
await aiofiles.os.makedirs(md_path.parent, exist_ok=True)
md_content = html_to_markdown(html_content)
async with aiofiles.open(md_path, 'w', encoding='utf-8') as f:
await f.write(md_content)
# Extract title for sitemap
title = self._extract_title(html_content) or str(rel_path)
sitemap.append({
'path': f'html/{rel_path}',
'title': title,
})
# Copy media from neopig's media vault (follows symlinks to hash vault)
media_vault_path = Path(vault_path) / 'media_vault' / domain
if media_vault_path.exists():
for media_file in media_vault_path.rglob('*'):
if media_file.is_file() or media_file.is_symlink():
try:
# Follow symlinks to get actual content
if media_file.is_symlink():
target = media_file.resolve()
if not target.exists():
logger.debug(f"Skipping broken symlink: {media_file}")
continue
content = target.read_bytes()
else:
content = media_file.read_bytes()
rel_path = media_file.relative_to(media_vault_path)
dest_path = media_dir / rel_path
await aiofiles.os.makedirs(dest_path.parent, exist_ok=True)
async with aiofiles.open(dest_path, 'wb') as f:
await f.write(content)
except Exception as e:
logger.debug(f"Error copying media {media_file}: {e}")
# Copy screenshots from neopig's linkpeek vault (follows symlinks to hash vault)
if self.include_screenshots:
linkpeek_vault_path = Path(vault_path) / 'linkpeek_vault' / domain
if linkpeek_vault_path.exists():
for screenshot_file in linkpeek_vault_path.rglob('*.png'):
try:
# Follow symlinks to get actual content
if screenshot_file.is_symlink():
target = screenshot_file.resolve()
if not target.exists():
logger.debug(f"Skipping broken symlink: {screenshot_file}")
continue
content = target.read_bytes()
else:
content = screenshot_file.read_bytes()
rel_path = screenshot_file.relative_to(linkpeek_vault_path)
dest_path = screenshots_dir / rel_path
await aiofiles.os.makedirs(dest_path.parent, exist_ok=True)
async with aiofiles.open(dest_path, 'wb') as f:
await f.write(content)
except Exception as e:
logger.debug(f"Error copying screenshot {screenshot_file}: {e}")
with tempfile.TemporaryDirectory(dir=local_tmpdir) as tmpdir:
tmpdir_path = Path(tmpdir)
# Create search database
await self._create_search_database(archive_root, sitemap, domain, html_dir)
await self._create_search_database_streaming(tmpdir_path, sitemap, domain, html_contents)
# Write embedded serve.py
self._write_serve_py(archive_root)
# Write serve.py
self._write_serve_py(tmpdir_path)
# Write metadata
metadata = {
@ -271,29 +247,195 @@ class SiteArchiver:
'include_screenshots': self.include_screenshots,
'include_markdown': self.include_markdown,
}
async with aiofiles.open(archive_root / 'metadata.json', 'w') as f:
await f.write(json.dumps(metadata, indent=2))
(tmpdir_path / 'metadata.json').write_text(json.dumps(metadata, indent=2))
# Copy state file into archive for future delta crawls
# Copy state file
state_domain = domain.replace('.', '-').replace(':', '-')
state_file = Path("data") / f"crawl-state-{state_domain}.json"
if state_file.exists():
shutil.copy(state_file, archive_root / 'crawl_state.json')
shutil.copy(state_file, tmpdir_path / 'crawl_state.json')
logger.info(f"Included crawl state for future delta crawls")
# Create tar.gz
tar_path = self.output_dir / f"{archive_name}.tar.gz"
# Write index.html
self._write_index_html(tmpdir_path, sitemap, domain)
def create_tarball():
# Step 3: Stream everything to tar.gz in one pass
logger.info("Streaming to archive...")
def stream_to_tar():
with tarfile.open(tar_path, 'w:gz') as tar:
tar.add(archive_root, arcname=archive_name)
# Add generated files first (from temp)
for f in tmpdir_path.iterdir():
tar.add(f, arcname=f"{archive_name}/{f.name}")
await asyncio.to_thread(create_tarball)
# Stream HTML files (with rewritten URLs)
for rel_path_str, content in html_contents.items():
try:
html_bytes = content.encode('utf-8')
arcname = f"{archive_name}/html/{rel_path_str}"
info = tarfile.TarInfo(name=arcname)
info.size = len(html_bytes)
tar.addfile(info, io.BytesIO(html_bytes))
final_size = tar_path.stat().st_size
logger.info(f"Archive complete: {tar_path} ({final_size / 1024 / 1024:.1f} MB)")
# Generate markdown on the fly
if self.include_markdown:
md_content = html_to_markdown(content, trim_wrapper=self.trim_wrapper)
md_bytes = md_content.encode('utf-8')
md_rel = rel_path_str.replace('.html', '.md')
md_arcname = f"{archive_name}/markdown/{md_rel}"
md_info = tarfile.TarInfo(name=md_arcname)
md_info.size = len(md_bytes)
tar.addfile(md_info, io.BytesIO(md_bytes))
except Exception as e:
logger.debug(f"Error adding HTML {rel_path_str}: {e}")
return tar_path
# Stream media files by hash (matching rewritten URLs)
# Build hash -> file path mapping from hash vault
hash_vault = Path(vault_path)
added_hashes = set()
for url, md5 in url_to_hash.items():
if md5 in added_hashes:
continue
# Find file in hash vault: vault/xx/hash.ext
bucket = md5[:2]
bucket_dir = hash_vault / bucket
if bucket_dir.exists():
for f in bucket_dir.iterdir():
if f.stem == md5:
try:
arcname = f"{archive_name}/media/{f.name}"
tar.add(f, arcname=arcname)
added_hashes.add(md5)
except Exception as e:
logger.debug(f"Error adding media {f}: {e}")
break
# Stream screenshots (follow symlinks)
if self.include_screenshots and linkpeek_vault_path.exists():
for screenshot_file in linkpeek_vault_path.rglob('*.png'):
try:
rel_path = screenshot_file.relative_to(linkpeek_vault_path)
arcname = f"{archive_name}/screenshots/{rel_path}"
if screenshot_file.is_symlink():
target = screenshot_file.resolve()
if target.exists():
tar.add(target, arcname=arcname)
else:
tar.add(screenshot_file, arcname=arcname)
except Exception as e:
logger.debug(f"Error adding screenshot {screenshot_file}: {e}")
# Bundle neopig source files for self-contained crawling
neopig_src = Path(__file__).parent
neopig_files = [
'neopig.py', 'database.py', 'async_web_fetcher.py',
'storage.py', 'domain_vault.py', 'screenshot.py',
'html2md.py', 'serp.py', 'filevault.py', 'async_filevault.py',
]
for pyfile in neopig_files:
src_path = neopig_src / pyfile
if src_path.exists():
tar.add(src_path, arcname=f"{archive_name}/neopig/{pyfile}")
# Add requirements.txt for neopig dependencies
req_path = neopig_src / 'requirements.txt'
if req_path.exists():
tar.add(req_path, arcname=f"{archive_name}/requirements.txt")
await asyncio.to_thread(stream_to_tar)
# Clear html_contents to free memory
html_contents.clear()
final_size = tar_path.stat().st_size
logger.info(f"Archive complete: {tar_path} ({final_size / 1024 / 1024:.1f} MB)")
return tar_path
async def _create_search_database_streaming(self, tmpdir: Path, sitemap: list, domain: str, html_contents: dict):
"""Create FTS5 search database from collected html contents."""
import sqlite3
db_path = tmpdir / 'archive.db'
def create_db():
conn = sqlite3.connect(db_path)
c = conn.cursor()
c.execute('''
CREATE TABLE pages (
id INTEGER PRIMARY KEY,
path TEXT UNIQUE,
title TEXT,
content TEXT
)
''')
c.execute('''
CREATE VIRTUAL TABLE pages_fts USING fts5(
title, content, path,
content='pages',
content_rowid='id'
)
''')
for item in sitemap:
html_path = item['path'].replace('html/', '', 1)
content = html_contents.get(html_path, '')
if content:
try:
soup = BeautifulSoup(content, 'html.parser')
for tag in soup(['script', 'style', 'nav', 'header', 'footer']):
tag.decompose()
text = soup.get_text(separator=' ', strip=True)[:50000]
except Exception:
text = ''
else:
text = ''
try:
c.execute('INSERT INTO pages (path, title, content) VALUES (?, ?, ?)',
(item['path'], item['title'], text))
except Exception:
pass
c.execute('''
INSERT INTO pages_fts(rowid, title, content, path)
SELECT id, title, content, path FROM pages
''')
conn.commit()
conn.close()
await asyncio.to_thread(create_db)
def _write_index_html(self, tmpdir: Path, sitemap: list, domain: str):
"""Write index.html with sitemap."""
html = f'''<!DOCTYPE html>
<html>
<head>
<title>{domain} Archive</title>
<style>
body {{ font-family: sans-serif; max-width: 800px; margin: 40px auto; padding: 0 20px; }}
h1 {{ color: #333; }}
ul {{ list-style: none; padding: 0; }}
li {{ padding: 8px 0; border-bottom: 1px solid #eee; }}
a {{ color: #0066cc; text-decoration: none; }}
a:hover {{ text-decoration: underline; }}
</style>
</head>
<body>
<h1>{domain} Archive</h1>
<p>{len(sitemap)} pages archived</p>
<ul>
'''
for item in sitemap[:1000]: # Limit to 1000 in index
html += f' <li><a href="{item["path"]}">{item["title"]}</a></li>\n'
if len(sitemap) > 1000:
html += f' <li>... and {len(sitemap) - 1000} more pages</li>\n'
html += ''' </ul>
</body>
</html>'''
(tmpdir / 'index.html').write_text(html)
def _extract_title(self, html: str) -> Optional[str]:
"""Extract title from HTML."""
@ -434,12 +576,24 @@ def get_archive_source():
except Exception:
pass
# Check command line for tarball argument
# Check command line for tarball or .run argument
for arg in sys.argv[1:]:
if not arg.startswith('-'):
p = Path(arg)
if p.exists() and p.suffix in ('.gz', '.tar', '.tgz'):
return ('tarball', p, 0)
if p.exists():
if p.suffix in ('.gz', '.tar', '.tgz'):
return ('tarball', p, 0)
# Check if it's a .run file with NEOPIG trailer
if p.suffix == '.run' or p.stat().st_size > 100000:
try:
with open(p, 'rb') as f:
f.seek(-22, 2)
trailer = f.read(22)
if trailer[:6] == b'NEOPIG':
offset = int(trailer[6:22].decode(), 16)
return ('run', p, offset)
except Exception:
pass
# Must be extracted directory
return ('directory', Path(__file__).parent, 0)
@ -1333,13 +1487,66 @@ if __name__ == '__main__':
(archive_root / 'serve.py').write_text(serve_py)
def upgrade_neopig_in_archive(archive_path: Path) -> Path:
"""Replace neopig/ directory in existing archive with current source."""
import shutil
if not archive_path.exists():
raise FileNotFoundError(f"Archive not found: {archive_path}")
neopig_src = Path(__file__).parent
neopig_files = [
'neopig.py', 'database.py', 'async_web_fetcher.py',
'storage.py', 'domain_vault.py', 'screenshot.py',
'html2md.py', 'serp.py', 'filevault.py', 'async_filevault.py',
]
# Create new archive with updated neopig
output_path = archive_path.with_suffix('.upgraded.tar.gz')
with tarfile.open(archive_path, 'r:gz') as old_tar:
with tarfile.open(output_path, 'w:gz') as new_tar:
# Get archive name from first member
members = old_tar.getmembers()
archive_name = members[0].name.split('/')[0]
# Copy all members except neopig/ and requirements.txt
for member in members:
parts = member.name.split('/')
if len(parts) > 1 and parts[1] == 'neopig':
continue # Skip old neopig files
if len(parts) > 1 and parts[1] == 'requirements.txt':
continue # Skip old requirements
if member.isfile():
f = old_tar.extractfile(member)
if f:
new_tar.addfile(member, f)
else:
new_tar.addfile(member)
# Add new neopig files
for pyfile in neopig_files:
src_path = neopig_src / pyfile
if src_path.exists():
new_tar.add(src_path, arcname=f"{archive_name}/neopig/{pyfile}")
# Add requirements.txt
req_path = neopig_src / 'requirements.txt'
if req_path.exists():
new_tar.add(req_path, arcname=f"{archive_name}/requirements.txt")
# Replace original with upgraded
shutil.move(output_path, archive_path)
return archive_path
async def main():
parser = argparse.ArgumentParser(
description="Archive a website for preservation (uses neopig for crawling)",
epilog="Example: python archive.py https://discourse-urho3d.github.io/"
)
parser.add_argument("url", help="URL of the site to archive")
parser.add_argument("url", nargs='?', help="URL of the site to archive")
parser.add_argument("-o", "--output", default=".", help="Output directory for tar.gz")
parser.add_argument("-d", "--depth", type=int, default=-1, help="Crawl depth (-1 = unlimited)")
parser.add_argument("-p", "--max-pages", type=int, default=-1, help="Max pages (-1 = unlimited)")
@ -1354,18 +1561,32 @@ async def main():
parser.add_argument("--serve", action="store_true", help="Start SERP server to watch crawl live")
parser.add_argument("--port", type=int, default=8000, help="Port for SERP server (default: 8000)")
parser.add_argument("--backfill-markdown", metavar="DOMAIN", help="Re-process stored HTML for DOMAIN to regenerate markdown with absolute URLs")
parser.add_argument("--trim-wrapper", action="store_true", help="With --backfill-markdown: strip nav/header/footer/logo before conversion")
parser.add_argument("--db", default="data/neopig.db", help="Database path (for --backfill-markdown)")
parser.add_argument("--upgrade-neopig", metavar="TARBALL", help="Upgrade neopig inside an existing archive")
args = parser.parse_args()
setup_logging(level=logging.DEBUG if args.verbose else logging.INFO)
# Handle --upgrade-neopig
if args.upgrade_neopig:
archive_path = Path(args.upgrade_neopig)
logger.info(f"Upgrading neopig in {archive_path}...")
upgrade_neopig_in_archive(archive_path)
logger.info(f"Done! Archive updated: {archive_path}")
return
# Handle --backfill-markdown
if args.backfill_markdown:
from neopig import backfill_markdown
await backfill_markdown(args.db, domain_filter=args.backfill_markdown)
await backfill_markdown(args.db, domain_filter=args.backfill_markdown, trim_wrapper=args.trim_wrapper)
return
# URL required for archiving
if not args.url:
parser.error("URL required (use --upgrade-neopig or --backfill-markdown for other operations)")
if args.no_markdown and not HAS_HTML2TEXT:
pass
elif not HAS_HTML2TEXT:
@ -1385,6 +1606,7 @@ async def main():
include_markdown=not args.no_markdown,
screenshot_config=screenshot_config,
fast_mode=args.fast,
trim_wrapper=args.trim_wrapper,
)
# Start SERP server if requested

5
async_web_fetcher.py
View file

@ -2198,6 +2198,11 @@ class AsyncWebFetcher:
logger.info(f"Depth {current_depth} complete: crawled {pages_at_this_depth} pages")
# Stop if no pages were crawled at this depth (all links exhausted or skipped)
if pages_at_this_depth == 0:
logger.info("No pages crawled at this depth - stopping crawl")
break
return self._finalize_results(all_pages)
def _finalize_results(self, all_pages: List[Dict]) -> List[Dict]:

269
bootstrap.c Normal file
View file

@ -0,0 +1,269 @@
/*
* bootstrap.c - Self-extracting archive bootstrap
*
* A small C program that boots an embedded Python archive server.
* When compiled and concatenated with a tar.gz archive, it:
* 1. Extracts serve.py from the embedded tarball
* 2. Runs: python3 serve.py <self>
* 3. The Python script serves directly from the tarball portion
*
* Build:
* gcc -O2 -o bootstrap bootstrap.c -lz
*
* Create self-extracting archive:
* cat bootstrap archive.tar.gz > archive.run
* chmod +x archive.run
* echo -n "NEOPIG" >> archive.run # Magic marker
* printf '%016x' $(stat -c%s bootstrap) >> archive.run # Offset (hex)
*
* Or use the provided make-executable.sh script.
*/
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <fcntl.h>
#include <errno.h>
#include <zlib.h>
#define MAGIC "NEOPIG"
#define MAGIC_LEN 6
#define OFFSET_LEN 16
#define TRAILER_LEN (MAGIC_LEN + OFFSET_LEN)
#define SERVE_PY_NAME "serve.py"
#define CHUNK_SIZE 16384
/* Find python3 interpreter */
static const char *find_python(void) {
static const char *pythons[] = {
"/usr/bin/python3",
"/usr/local/bin/python3",
"/opt/homebrew/bin/python3",
"python3",
NULL
};
for (int i = 0; pythons[i]; i++) {
if (access(pythons[i], X_OK) == 0) {
return pythons[i];
}
}
/* Try PATH lookup */
return "python3";
}
/* Read trailer to get tarball offset */
static long read_trailer(const char *self_path) {
FILE *f = fopen(self_path, "rb");
if (!f) {
perror("fopen self");
return -1;
}
/* Seek to trailer */
if (fseek(f, -TRAILER_LEN, SEEK_END) != 0) {
perror("fseek trailer");
fclose(f);
return -1;
}
char trailer[TRAILER_LEN + 1];
if (fread(trailer, 1, TRAILER_LEN, f) != TRAILER_LEN) {
perror("fread trailer");
fclose(f);
return -1;
}
trailer[TRAILER_LEN] = '\0';
fclose(f);
/* Check magic */
if (memcmp(trailer, MAGIC, MAGIC_LEN) != 0) {
fprintf(stderr, "Error: Invalid archive (missing NEOPIG magic)\n");
fprintf(stderr, "This executable must be created with make-executable.sh\n");
return -1;
}
/* Parse hex offset */
long offset = strtol(trailer + MAGIC_LEN, NULL, 16);
return offset;
}
/* Extract serve.py from gzipped tarball */
static char *extract_serve_py(const char *self_path, long tar_offset) {
/* Create temp file for serve.py */
char *temp_path = strdup("/tmp/neopig_serve_XXXXXX.py");
if (!temp_path) return NULL;
/* mkstemps for .py suffix */
int fd = mkstemps(temp_path, 3);
if (fd < 0) {
perror("mkstemps");
free(temp_path);
return NULL;
}
/* Open self and seek to tarball */
FILE *self = fopen(self_path, "rb");
if (!self) {
perror("fopen self for tar");
close(fd);
unlink(temp_path);
free(temp_path);
return NULL;
}
if (fseek(self, tar_offset, SEEK_SET) != 0) {
perror("fseek to tar");
fclose(self);
close(fd);
unlink(temp_path);
free(temp_path);
return NULL;
}
/* Open gzip stream */
gzFile gz = gzdopen(fileno(self), "rb");
if (!gz) {
fprintf(stderr, "gzdopen failed\n");
fclose(self);
close(fd);
unlink(temp_path);
free(temp_path);
return NULL;
}
/* Read tar headers looking for serve.py */
unsigned char header[512];
int found = 0;
while (gzread(gz, header, 512) == 512) {
/* Check for end of archive (all zeros) */
int all_zero = 1;
for (int i = 0; i < 512 && all_zero; i++) {
if (header[i] != 0) all_zero = 0;
}
if (all_zero) break;
/* Get filename (first 100 bytes) */
char filename[101];
memcpy(filename, header, 100);
filename[100] = '\0';
/* Get file size (octal, bytes 124-135) */
char size_str[13];
memcpy(size_str, header + 124, 12);
size_str[12] = '\0';
long filesize = strtol(size_str, NULL, 8);
/* Check if this is serve.py */
char *basename = strrchr(filename, '/');
basename = basename ? basename + 1 : filename;
if (strcmp(basename, SERVE_PY_NAME) == 0) {
/* Extract this file */
unsigned char buf[CHUNK_SIZE];
long remaining = filesize;
while (remaining > 0) {
int to_read = remaining > CHUNK_SIZE ? CHUNK_SIZE : remaining;
int got = gzread(gz, buf, to_read);
if (got <= 0) break;
write(fd, buf, got);
remaining -= got;
}
found = 1;
break;
} else {
/* Skip this file's content (padded to 512 bytes) */
long blocks = (filesize + 511) / 512;
for (long i = 0; i < blocks; i++) {
if (gzread(gz, header, 512) != 512) break;
}
}
}
gzclose(gz);
close(fd);
if (!found) {
fprintf(stderr, "Error: serve.py not found in archive\n");
unlink(temp_path);
free(temp_path);
return NULL;
}
return temp_path;
}
int main(int argc, char *argv[]) {
/* Get path to self */
char self_path[4096];
ssize_t len = readlink("/proc/self/exe", self_path, sizeof(self_path) - 1);
if (len < 0) {
/* Fallback to argv[0] */
if (argv[0][0] == '/') {
strncpy(self_path, argv[0], sizeof(self_path) - 1);
} else {
char *cwd = getcwd(NULL, 0);
snprintf(self_path, sizeof(self_path), "%s/%s", cwd, argv[0]);
free(cwd);
}
} else {
self_path[len] = '\0';
}
/* Read trailer to get tarball offset */
long tar_offset = read_trailer(self_path);
if (tar_offset < 0) {
return 1;
}
printf("Archive offset: %ld bytes\n", tar_offset);
/* Extract serve.py */
char *serve_py_path = extract_serve_py(self_path, tar_offset);
if (!serve_py_path) {
return 1;
}
printf("Extracted: %s\n", serve_py_path);
/* Find python */
const char *python = find_python();
/* Build command: python3 serve.py <self_path> */
printf("Starting: %s %s %s\n", python, serve_py_path, self_path);
/* Fork and exec */
pid_t pid = fork();
if (pid < 0) {
perror("fork");
unlink(serve_py_path);
free(serve_py_path);
return 1;
}
if (pid == 0) {
/* Child - exec python */
execl(python, "python3", serve_py_path, self_path, NULL);
perror("execl python3");
_exit(1);
}
/* Parent - wait for child and cleanup */
int status;
waitpid(pid, &status, 0);
/* Cleanup temp file */
unlink(serve_py_path);
free(serve_py_path);
return WIFEXITED(status) ? WEXITSTATUS(status) : 1;
}

843
database.py
View file

@ -1,11 +1,11 @@
#!/usr/bin/env python3
"""
SQLite database for neopig metadata.
SQLAlchemy async database for neopig metadata.
Stores:
- Crawl jobs (target, keywords, timestamps, stats)
- Media records (md5_hash, source URLs, metadata)
- Analysis results (Qwen 3 VL outputs)
- Pages for full-text search
"""
import json
@ -13,18 +13,124 @@ import logging
from datetime import datetime, timezone
from typing import List, Dict, Any, Optional, Set
import aiosqlite
from sqlalchemy import (
Column, Integer, String, Text, ForeignKey, Index, UniqueConstraint,
create_engine, event, text, select, update, func, or_, and_
)
from sqlalchemy.orm import declarative_base, relationship, sessionmaker
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
logger = logging.getLogger(__name__)
Base = declarative_base()
# =============================================================================
# Models
# =============================================================================
class CrawlJob(Base):
__tablename__ = 'crawl_jobs'
id = Column(Integer, primary_key=True, autoincrement=True)
target_uri = Column(Text, nullable=False)
keywords = Column(Text) # JSON array
mode = Column(String(50), default='images')
status = Column(String(50), default='running')
started_at = Column(Text, nullable=False)
completed_at = Column(Text)
stats = Column(Text) # JSON object
media_sources = relationship('MediaSource', back_populates='crawl_job')
pages = relationship('Page', back_populates='crawl_job')
class Media(Base):
__tablename__ = 'media'
md5_hash = Column(String(32), primary_key=True)
media_type = Column(String(20)) # 'image', 'video', 'audio'
mime_type = Column(String(100))
file_size = Column(Integer)
keywords = Column(Text) # JSON array
alt_text = Column(Text)
title = Column(Text)
first_seen_at = Column(Text, nullable=False)
analysis_status = Column(String(20), default='pending')
analysis_result = Column(Text) # JSON
sources = relationship('MediaSource', back_populates='media')
__table_args__ = (
Index('idx_media_type', 'media_type'),
Index('idx_media_analysis', 'analysis_status'),
)
class MediaSource(Base):
__tablename__ = 'media_sources'
id = Column(Integer, primary_key=True, autoincrement=True)
md5_hash = Column(String(32), ForeignKey('media.md5_hash'), nullable=False)
media_uri = Column(Text, nullable=False)
page_uri = Column(Text)
page_title = Column(Text)
page_description = Column(Text)
page_keywords = Column(Text)
page_content = Column(Text)
alt_text = Column(Text)
link_text = Column(Text)
detail_page_uri = Column(Text)
detail_title = Column(Text)
detail_content = Column(Text)
searchable_text = Column(Text)
crawl_job_id = Column(Integer, ForeignKey('crawl_jobs.id'))
discovered_at = Column(Text, nullable=False)
media = relationship('Media', back_populates='sources')
crawl_job = relationship('CrawlJob', back_populates='media_sources')
__table_args__ = (
UniqueConstraint('md5_hash', 'media_uri', 'page_uri', name='uq_media_source'),
Index('idx_sources_hash', 'md5_hash'),
Index('idx_sources_job', 'crawl_job_id'),
Index('idx_sources_media_uri', 'media_uri'),
Index('idx_sources_page_uri', 'page_uri'),
)
class Page(Base):
__tablename__ = 'pages'
id = Column(Integer, primary_key=True, autoincrement=True)
uri = Column(Text, nullable=False, unique=True)
path = Column(Text)
title = Column(Text)
content = Column(Text)
markdown = Column(Text)
raw_html = Column(Text)
crawl_job_id = Column(Integer, ForeignKey('crawl_jobs.id'))
crawled_at = Column(Text, nullable=False)
crawl_job = relationship('CrawlJob', back_populates='pages')
__table_args__ = (
Index('idx_pages_uri', 'uri'),
)
# =============================================================================
# Database Class
# =============================================================================
class Database:
"""
Async SQLite database for neopig metadata.
"""
"""Async SQLAlchemy database for neopig metadata."""
def __init__(self, db_path: str = "neopig.db"):
self.db_path = db_path
self._engine = None
self._session_factory = None
self._initialized = False
async def init(self) -> None:
@ -32,132 +138,71 @@ class Database:
if self._initialized:
return
async with aiosqlite.connect(self.db_path) as db:
# Crawl jobs table
await db.execute("""
CREATE TABLE IF NOT EXISTS crawl_jobs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
target_uri TEXT NOT NULL,
keywords TEXT, -- JSON array
mode TEXT DEFAULT 'images',
status TEXT DEFAULT 'running',
started_at TEXT NOT NULL,
completed_at TEXT,
stats TEXT -- JSON object
)
""")
# Create async engine with WAL mode for concurrent access
self._engine = create_async_engine(
f"sqlite+aiosqlite:///{self.db_path}",
echo=False,
)
# Media records table (main deduped storage)
await db.execute("""
CREATE TABLE IF NOT EXISTS media (
md5_hash TEXT PRIMARY KEY,
media_type TEXT, -- 'image', 'video', 'audio'
mime_type TEXT,
file_size INTEGER,
keywords TEXT, -- JSON array
alt_text TEXT,
title TEXT,
first_seen_at TEXT NOT NULL,
analysis_status TEXT DEFAULT 'pending', -- 'pending', 'analyzed', 'invalid', 'error'
analysis_result TEXT -- JSON from Qwen 3 VL
)
""")
# Enable WAL mode for concurrent reads/writes
@event.listens_for(self._engine.sync_engine, "connect")
def set_sqlite_pragma(dbapi_conn, connection_record):
cursor = dbapi_conn.cursor()
cursor.execute("PRAGMA journal_mode=WAL")
cursor.execute("PRAGMA busy_timeout=30000") # 30 second timeout
cursor.close()
# Media sources table (tracks all contexts where media was found)
await db.execute("""
CREATE TABLE IF NOT EXISTS media_sources (
id INTEGER PRIMARY KEY AUTOINCREMENT,
md5_hash TEXT NOT NULL,
media_uri TEXT NOT NULL,
page_uri TEXT,
page_title TEXT,
page_description TEXT,
page_keywords TEXT,
page_content TEXT,
alt_text TEXT,
link_text TEXT,
detail_page_uri TEXT,
detail_title TEXT,
detail_content TEXT,
searchable_text TEXT, -- Combined metadata for full-text search
crawl_job_id INTEGER,
discovered_at TEXT NOT NULL,
FOREIGN KEY (md5_hash) REFERENCES media(md5_hash),
FOREIGN KEY (crawl_job_id) REFERENCES crawl_jobs(id),
UNIQUE(md5_hash, media_uri, page_uri)
)
""")
# Create session factory
self._session_factory = async_sessionmaker(
self._engine,
class_=AsyncSession,
expire_on_commit=False
)
# Pages table for full-text search
await db.execute("""
CREATE TABLE IF NOT EXISTS pages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
uri TEXT NOT NULL UNIQUE,
path TEXT,
title TEXT,
content TEXT,
markdown TEXT,
raw_html TEXT,
crawl_job_id INTEGER,
crawled_at TEXT NOT NULL,
FOREIGN KEY (crawl_job_id) REFERENCES crawl_jobs(id)
)
""")
# Create tables
async with self._engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
# Add columns if they don't exist (migration for existing DBs)
for col in ['raw_html', 'markdown']:
try:
await db.execute(f"ALTER TABLE pages ADD COLUMN {col} TEXT")
except Exception:
pass # Column already exists
# FTS5 virtual table for page search
await db.execute("""
# Create FTS5 virtual table (SQLAlchemy doesn't handle virtual tables)
await conn.execute(text("""
CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5(
title, content, uri, path,
content='pages',
content_rowid='id'
)
""")
"""))
# Triggers to keep FTS in sync
await db.execute("""
# FTS triggers
await conn.execute(text("""
CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN
INSERT INTO pages_fts(rowid, title, content, uri, path)
VALUES (new.id, new.title, new.content, new.uri, new.path);
END
""")
"""))
await db.execute("""
await conn.execute(text("""
CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN
INSERT INTO pages_fts(pages_fts, rowid, title, content, uri, path)
VALUES ('delete', old.id, old.title, old.content, old.uri, old.path);
END
""")
"""))
await db.execute("""
await conn.execute(text("""
CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN
INSERT INTO pages_fts(pages_fts, rowid, title, content, uri, path)
VALUES ('delete', old.id, old.title, old.content, old.uri, old.path);
INSERT INTO pages_fts(rowid, title, content, uri, path)
VALUES (new.id, new.title, new.content, new.uri, new.path);
END
""")
# Indexes
await db.execute("CREATE INDEX IF NOT EXISTS idx_media_type ON media(media_type)")
await db.execute("CREATE INDEX IF NOT EXISTS idx_media_analysis ON media(analysis_status)")
await db.execute("CREATE INDEX IF NOT EXISTS idx_sources_hash ON media_sources(md5_hash)")
await db.execute("CREATE INDEX IF NOT EXISTS idx_sources_job ON media_sources(crawl_job_id)")
await db.execute("CREATE INDEX IF NOT EXISTS idx_sources_media_uri ON media_sources(media_uri)")
await db.execute("CREATE INDEX IF NOT EXISTS idx_sources_page_uri ON media_sources(page_uri)")
await db.execute("CREATE INDEX IF NOT EXISTS idx_pages_uri ON pages(uri)")
await db.commit()
"""))
self._initialized = True
logger.info(f"Database initialized: {self.db_path}")
def session(self) -> AsyncSession:
"""Get a new async session."""
return self._session_factory()
async def create_crawl_job(
self,
target_uri: str,
@ -165,38 +210,31 @@ class Database:
mode: str = "images"
) -> int:
"""Create a new crawl job and return its ID."""
async with aiosqlite.connect(self.db_path) as db:
cursor = await db.execute(
"""
INSERT INTO crawl_jobs (target_uri, keywords, mode, started_at)
VALUES (?, ?, ?, ?)
""",
(
target_uri,
json.dumps(keywords or []),
mode,
datetime.now(timezone.utc).isoformat()
)
async with self.session() as session:
job = CrawlJob(
target_uri=target_uri,
keywords=json.dumps(keywords or []),
mode=mode,
started_at=datetime.now(timezone.utc).isoformat()
)
await db.commit()
return cursor.lastrowid
session.add(job)
await session.commit()
return job.id
async def complete_crawl_job(self, job_id: int, stats: Dict[str, Any]) -> None:
"""Mark a crawl job as complete."""
async with aiosqlite.connect(self.db_path) as db:
await db.execute(
"""
UPDATE crawl_jobs
SET status = 'completed', completed_at = ?, stats = ?
WHERE id = ?
""",
(
datetime.now(timezone.utc).isoformat(),
json.dumps(stats),
job_id
async with self.session() as session:
stmt = (
update(CrawlJob)
.where(CrawlJob.id == job_id)
.values(
status='completed',
completed_at=datetime.now(timezone.utc).isoformat(),
stats=json.dumps(stats)
)
)
await db.commit()
await session.execute(stmt)
await session.commit()
async def create_media_record(
self,
@ -218,38 +256,40 @@ class Database:
detail_content: str = "",
searchable_text: str = "",
) -> None:
"""Create a new media record and add source context.
Skeleton key approach: stores both embedding context (page_title, page_content from listing)
and detail context (detail_title, detail_content from detail page) for maximum searchability.
For blogs: page_content contains the post text so images are searchable by post content.
For galleries: detail_content contains the detail page text for richer metadata.
"""
"""Create a new media record and add source context."""
now = datetime.now(timezone.utc).isoformat()
async with aiosqlite.connect(self.db_path) as db:
# Insert or ignore media record (just the hash and basic info)
await db.execute(
"""
INSERT OR IGNORE INTO media
(md5_hash, media_type, mime_type, file_size, first_seen_at)
VALUES (?, ?, ?, ?, ?)
""",
(md5_hash, media_type, mime_type, file_size, now)
)
async with self.session() as session:
# Insert or ignore media record using SQLite upsert
media_stmt = sqlite_insert(Media).values(
md5_hash=md5_hash,
media_type=media_type,
mime_type=mime_type,
file_size=file_size,
first_seen_at=now
).on_conflict_do_nothing(index_elements=['md5_hash'])
await session.execute(media_stmt)
# Always add source with full context (unique on media_uri + page_uri)
await db.execute(
"""
INSERT OR IGNORE INTO media_sources
(md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, page_content, alt_text, link_text, detail_page_uri, detail_title, detail_content, searchable_text, crawl_job_id, discovered_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, page_content, alt_text, link_text, detail_page_uri, detail_title, detail_content, searchable_text, crawl_job_id, now)
)
await db.commit()
# Insert or ignore source context
source_stmt = sqlite_insert(MediaSource).values(
md5_hash=md5_hash,
media_uri=media_uri,
page_uri=page_uri,
page_title=page_title,
page_description=page_description,
page_keywords=page_keywords,
page_content=page_content,
alt_text=alt_text,
link_text=link_text,
detail_page_uri=detail_page_uri,
detail_title=detail_title,
detail_content=detail_content,
searchable_text=searchable_text,
crawl_job_id=crawl_job_id,
discovered_at=now
).on_conflict_do_nothing()
await session.execute(source_stmt)
await session.commit()
async def add_media_source(
self,
@ -268,40 +308,39 @@ class Database:
searchable_text: str = "",
crawl_job_id: int = None
) -> None:
"""Add another source context for an existing media hash.
Skeleton key approach: stores both embedding context (page_title, page_content from listing)
and detail context (detail_title, detail_content from detail page) for maximum searchability.
For blogs: page_content contains the post text so images are searchable by post content.
For galleries: detail_content contains the detail page text for richer metadata.
"""
async with aiosqlite.connect(self.db_path) as db:
await db.execute(
"""
INSERT OR IGNORE INTO media_sources
(md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, page_content, alt_text, link_text, detail_page_uri, detail_title, detail_content, searchable_text, crawl_job_id, discovered_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, page_content, alt_text, link_text, detail_page_uri, detail_title, detail_content, searchable_text, crawl_job_id, datetime.now(timezone.utc).isoformat())
)
await db.commit()
"""Add another source context for an existing media hash."""
async with self.session() as session:
stmt = sqlite_insert(MediaSource).values(
md5_hash=md5_hash,
media_uri=media_uri,
page_uri=page_uri,
page_title=page_title,
page_description=page_description,
page_keywords=page_keywords,
page_content=page_content,
alt_text=alt_text,
link_text=link_text,
detail_page_uri=detail_page_uri,
detail_title=detail_title,
detail_content=detail_content,
searchable_text=searchable_text,
crawl_job_id=crawl_job_id,
discovered_at=datetime.now(timezone.utc).isoformat()
).on_conflict_do_nothing()
await session.execute(stmt)
await session.commit()
async def get_pending_analysis(self, limit: int = 100) -> List[Dict[str, Any]]:
"""Get media items pending analysis."""
async with aiosqlite.connect(self.db_path) as db:
db.row_factory = aiosqlite.Row
cursor = await db.execute(
"""
SELECT md5_hash, media_type, mime_type, keywords, alt_text, title
FROM media
WHERE analysis_status = 'pending'
LIMIT ?
""",
(limit,)
async with self.session() as session:
stmt = (
select(Media.md5_hash, Media.media_type, Media.mime_type,
Media.keywords, Media.alt_text, Media.title)
.where(Media.analysis_status == 'pending')
.limit(limit)
)
rows = await cursor.fetchall()
return [dict(row) for row in rows]
result = await session.execute(stmt)
return [dict(row._mapping) for row in result.fetchall()]
async def update_analysis(
self,
@ -310,61 +349,57 @@ class Database:
result: Dict[str, Any] = None
) -> None:
"""Update analysis status and result for a media item."""
async with aiosqlite.connect(self.db_path) as db:
await db.execute(
"""
UPDATE media
SET analysis_status = ?, analysis_result = ?
WHERE md5_hash = ?
""",
(status, json.dumps(result) if result else None, md5_hash)
async with self.session() as session:
stmt = (
update(Media)
.where(Media.md5_hash == md5_hash)
.values(
analysis_status=status,
analysis_result=json.dumps(result) if result else None
)
)
await db.commit()
await session.execute(stmt)
await session.commit()
async def get_media_by_keyword(self, keyword: str, limit: int = 100) -> List[Dict[str, Any]]:
"""Search media by keyword."""
async with aiosqlite.connect(self.db_path) as db:
db.row_factory = aiosqlite.Row
# Search in keywords JSON array and alt_text/title
cursor = await db.execute(
"""
SELECT m.*, GROUP_CONCAT(ms.source_url) as source_urls
FROM media m
LEFT JOIN media_sources ms ON m.md5_hash = ms.md5_hash
WHERE m.keywords LIKE ?
OR m.alt_text LIKE ?
OR m.title LIKE ?
GROUP BY m.md5_hash
LIMIT ?
""",
(f'%{keyword}%', f'%{keyword}%', f'%{keyword}%', limit)
async with self.session() as session:
like_pattern = f'%{keyword}%'
stmt = (
select(Media)
.where(or_(
Media.keywords.like(like_pattern),
Media.alt_text.like(like_pattern),
Media.title.like(like_pattern)
))
.limit(limit)
)
rows = await cursor.fetchall()
return [dict(row) for row in rows]
result = await session.execute(stmt)
return [
{**dict(row._mapping), 'source_urls': None}
for row in result.fetchall()
]
async def check_media_uri_exists(self, media_uri: str, page_uri: str) -> Optional[str]:
"""
Check if a media URI from a specific page has already been crawled.
Returns:
The md5_hash if exists, None otherwise
"""
async with aiosqlite.connect(self.db_path) as db:
cursor = await db.execute(
"SELECT md5_hash FROM media_sources WHERE media_uri = ? AND page_uri = ?",
(media_uri, page_uri)
"""Check if a media URI from a specific page has already been crawled."""
async with self.session() as session:
stmt = (
select(MediaSource.md5_hash)
.where(and_(
MediaSource.media_uri == media_uri,
MediaSource.page_uri == page_uri
))
)
row = await cursor.fetchone()
result = await session.execute(stmt)
row = result.fetchone()
return row[0] if row else None
async def get_crawled_media_uris(self) -> Set[str]:
"""Get all media URIs that have been crawled."""
async with aiosqlite.connect(self.db_path) as db:
cursor = await db.execute(
"SELECT DISTINCT media_uri FROM media_sources"
)
rows = await cursor.fetchall()
return {row[0] for row in rows}
async with self.session() as session:
stmt = select(MediaSource.media_uri).distinct()
result = await session.execute(stmt)
return {row[0] for row in result.fetchall()}
async def store_page(
self,
@ -376,83 +411,323 @@ class Database:
raw_html: str = "",
crawl_job_id: int = None
) -> None:
"""Store a page for full-text search and phantom site recreation."""
async with aiosqlite.connect(self.db_path) as db:
await db.execute(
"""
INSERT OR REPLACE INTO pages (uri, path, title, content, markdown, raw_html, crawl_job_id, crawled_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""",
(uri, path, title, content[:100000], markdown, raw_html, crawl_job_id, datetime.now(timezone.utc).isoformat())
"""Store a page for full-text search."""
async with self.session() as session:
# Use SQLite upsert (INSERT OR REPLACE)
stmt = sqlite_insert(Page).values(
uri=uri,
path=path,
title=title,
content=content[:100000],
markdown=markdown,
raw_html=raw_html,
crawl_job_id=crawl_job_id,
crawled_at=datetime.now(timezone.utc).isoformat()
)
await db.commit()
# On conflict with uri, update all fields
stmt = stmt.on_conflict_do_update(
index_elements=['uri'],
set_={
'path': stmt.excluded.path,
'title': stmt.excluded.title,
'content': stmt.excluded.content,
'markdown': stmt.excluded.markdown,
'raw_html': stmt.excluded.raw_html,
'crawl_job_id': stmt.excluded.crawl_job_id,
'crawled_at': stmt.excluded.crawled_at,
}
)
await session.execute(stmt)
await session.commit()
async def search_pages(self, query: str, limit: int = 50) -> List[Dict[str, Any]]:
"""Search pages using FTS5 with LIKE fallback."""
async with aiosqlite.connect(self.db_path) as db:
db.row_factory = aiosqlite.Row
"""Search pages using FTS5 with LIKE fallback.
Note: FTS5 virtual table requires raw SQL (allowed per CLAUDE.md).
"""
async with self.session() as session:
results = []
# Try FTS5 with prefix matching
# Try FTS5 (raw SQL required for virtual table)
try:
fts_query = ' '.join(f'"{word}"*' for word in query.split())
cursor = await db.execute("""
SELECT p.uri, p.path, p.title,
snippet(pages_fts, 1, '<mark>', '</mark>', '...', 40) as snippet
FROM pages_fts
JOIN pages p ON pages_fts.rowid = p.id
WHERE pages_fts MATCH ?
ORDER BY rank
LIMIT ?
""", (fts_query, limit))
results = [dict(row) for row in await cursor.fetchall()]
result = await session.execute(
text("""
SELECT p.uri, p.path, p.title,
snippet(pages_fts, 1, '<mark>', '</mark>', '...', 40) as snippet
FROM pages_fts
JOIN pages p ON pages_fts.rowid = p.id
WHERE pages_fts MATCH :query
ORDER BY rank LIMIT :limit
"""),
{'query': fts_query, 'limit': limit}
)
results = [dict(row._mapping) for row in result.fetchall()]
except Exception:
pass
# Fallback to LIKE
# Fallback to ORM LIKE
if not results:
try:
like_q = f'%{query}%'
cursor = await db.execute("""
SELECT uri, path, title, substr(content, 1, 200) as snippet
FROM pages
WHERE title LIKE ? COLLATE NOCASE
OR content LIKE ? COLLATE NOCASE
LIMIT ?
""", (like_q, like_q, limit))
results = [dict(row) for row in await cursor.fetchall()]
except Exception:
pass
like_q = f'%{query}%'
stmt = (
select(Page.uri, Page.path, Page.title,
func.substr(Page.content, 1, 200).label('snippet'))
.where(or_(
Page.title.ilike(like_q),
Page.content.ilike(like_q)
))
.limit(limit)
)
result = await session.execute(stmt)
results = [dict(row._mapping) for row in result.fetchall()]
return results
async def get_stats(self) -> Dict[str, Any]:
"""Get database statistics."""
async with aiosqlite.connect(self.db_path) as db:
async with self.session() as session:
stats = {}
# Total media
cursor = await db.execute("SELECT COUNT(*) FROM media")
stats['total_media'] = (await cursor.fetchone())[0]
result = await session.execute(select(func.count()).select_from(Media))
stats['total_media'] = result.scalar()
# By type
cursor = await db.execute(
"SELECT media_type, COUNT(*) FROM media GROUP BY media_type"
result = await session.execute(
select(Media.media_type, func.count())
.group_by(Media.media_type)
)
stats['by_type'] = {row[0]: row[1] for row in await cursor.fetchall()}
stats['by_type'] = {row[0]: row[1] for row in result.fetchall()}
# By analysis status
cursor = await db.execute(
"SELECT analysis_status, COUNT(*) FROM media GROUP BY analysis_status"
result = await session.execute(
select(Media.analysis_status, func.count())
.group_by(Media.analysis_status)
)
stats['by_analysis'] = {row[0]: row[1] for row in await cursor.fetchall()}
stats['by_analysis'] = {row[0]: row[1] for row in result.fetchall()}
# Total sources
cursor = await db.execute("SELECT COUNT(*) FROM media_sources")
stats['total_sources'] = (await cursor.fetchone())[0]
result = await session.execute(select(func.count()).select_from(MediaSource))
stats['total_sources'] = result.scalar()
# Crawl jobs
cursor = await db.execute("SELECT COUNT(*) FROM crawl_jobs")
stats['total_jobs'] = (await cursor.fetchone())[0]
result = await session.execute(select(func.count()).select_from(CrawlJob))
stats['total_jobs'] = result.scalar()
return stats
# =============================================================================
# Query methods for SERP
# =============================================================================
def _model_to_dict(self, obj) -> Dict[str, Any]:
"""Convert a SQLAlchemy model instance to dict."""
return {c.name: getattr(obj, c.name) for c in obj.__table__.columns}
async def get_media_by_hash(self, md5_hash: str) -> Optional[Dict[str, Any]]:
"""Get media record by MD5 hash."""
async with self.session() as session:
stmt = select(Media).where(Media.md5_hash == md5_hash)
result = await session.execute(stmt)
row = result.scalar_one_or_none()
return self._model_to_dict(row) if row else None
async def get_media_sources(self, md5_hash: str) -> List[Dict[str, Any]]:
"""Get all sources for a media item."""
async with self.session() as session:
stmt = (
select(MediaSource.media_uri, MediaSource.page_uri, MediaSource.page_title,
MediaSource.page_description, MediaSource.page_content, MediaSource.alt_text,
MediaSource.link_text, MediaSource.detail_page_uri, MediaSource.detail_title,
MediaSource.detail_content, MediaSource.discovered_at)
.where(MediaSource.md5_hash == md5_hash)
)
result = await session.execute(stmt)
return [dict(row._mapping) for row in result.fetchall()]
async def get_page_by_uri(self, uri: str) -> Optional[Dict[str, Any]]:
"""Get page by URI."""
async with self.session() as session:
stmt = select(Page).where(Page.uri == uri)
result = await session.execute(stmt)
row = result.scalar_one_or_none()
return self._model_to_dict(row) if row else None
async def get_crawl_jobs(self, limit: int = 50) -> List[Dict[str, Any]]:
"""Get recent crawl jobs."""
async with self.session() as session:
stmt = (
select(CrawlJob)
.order_by(CrawlJob.started_at.desc())
.limit(limit)
)
result = await session.execute(stmt)
return [self._model_to_dict(row) for row in result.scalars()]
async def get_crawl_job(self, job_id: int) -> Optional[Dict[str, Any]]:
"""Get a specific crawl job."""
async with self.session() as session:
stmt = select(CrawlJob).where(CrawlJob.id == job_id)
result = await session.execute(stmt)
row = result.scalar_one_or_none()
return self._model_to_dict(row) if row else None
async def search_media_advanced(
self,
q: str = None,
media_type: str = None,
limit: int = 100,
offset: int = 0
) -> List[Dict[str, Any]]:
"""Search media with filters."""
async with self.session() as session:
stmt = (
select(Media.md5_hash, Media.media_type, Media.mime_type, Media.file_size,
Media.alt_text, Media.title, Media.first_seen_at,
MediaSource.page_uri, MediaSource.page_title)
.outerjoin(MediaSource, Media.md5_hash == MediaSource.md5_hash)
.distinct()
)
conditions = []
if q:
q_mid = f'% {q} %'
q_start = f'{q} %'
q_end = f'% {q}'
conditions.append(or_(
MediaSource.searchable_text.like(q_mid),
MediaSource.searchable_text.like(q_start),
MediaSource.searchable_text.like(q_end),
MediaSource.searchable_text.like(q),
Media.alt_text.like(q_mid),
Media.title.like(q_mid)
))
if media_type:
conditions.append(Media.media_type == media_type)
if conditions:
stmt = stmt.where(and_(*conditions))
stmt = stmt.order_by(Media.first_seen_at.desc()).limit(limit).offset(offset)
result = await session.execute(stmt)
return [dict(row._mapping) for row in result.fetchall()]
async def get_domains_with_pages(self) -> List[tuple]:
"""Get domains that have pages with raw_html.
Note: Uses raw SQL for SQLite-specific substr/instr functions.
"""
async with self.session() as session:
# Raw SQL needed for SQLite string functions
result = await session.execute(
text("""SELECT DISTINCT
substr(uri, instr(uri, '://') + 3,
instr(substr(uri, instr(uri, '://') + 3), '/') - 1) as domain,
COUNT(*) as cnt
FROM pages WHERE raw_html IS NOT NULL
GROUP BY domain""")
)
return [(row[0], row[1]) for row in result.fetchall()]
async def lookup_media_by_uris(self, uris: List[str]) -> Dict[str, str]:
"""Look up media hashes by URIs. Returns {uri: md5_hash}."""
if not uris:
return {}
async with self.session() as session:
stmt = (
select(MediaSource.media_uri, MediaSource.md5_hash)
.where(MediaSource.media_uri.in_(uris))
)
result = await session.execute(stmt)
return {row[0]: row[1] for row in result.fetchall()}
async def lookup_media_by_filename(self, filename: str) -> Optional[tuple]:
"""Look up media by filename pattern. Returns (media_uri, md5_hash) or None."""
async with self.session() as session:
stmt = (
select(MediaSource.media_uri, MediaSource.md5_hash)
.where(MediaSource.media_uri.like(f'%{filename}%'))
.limit(1)
)
result = await session.execute(stmt)
row = result.fetchone()
return (row[0], row[1]) if row else None
async def get_page_screenshot(self, page_uri: str, exclude_hash: str = None) -> Optional[str]:
"""Get screenshot hash for a page. Returns md5_hash or None."""
async with self.session() as session:
stmt = (
select(Media.md5_hash)
.join(MediaSource, Media.md5_hash == MediaSource.md5_hash)
.where(and_(
MediaSource.page_uri == page_uri,
Media.media_type == 'screenshot' # Must be a screenshot, not just any PNG
))
.order_by(Media.file_size.desc())
.limit(1)
)
if exclude_hash:
stmt = stmt.where(Media.md5_hash != exclude_hash)
result = await session.execute(stmt)
row = result.fetchone()
return row[0] if row else None
async def get_recent_media(self, limit: int = 50, media_type: str = None) -> List[Dict[str, Any]]:
"""Get recently discovered media."""
async with self.session() as session:
stmt = (
select(Media.md5_hash, Media.media_type, Media.mime_type, Media.file_size,
Media.alt_text, Media.title, Media.first_seen_at)
.order_by(Media.first_seen_at.desc())
.limit(limit)
)
if media_type:
stmt = stmt.where(Media.media_type == media_type)
result = await session.execute(stmt)
return [dict(row._mapping) for row in result.fetchall()]
async def get_pages_by_domain(self, domain: str = None) -> List[Dict[str, Any]]:
"""Get pages, optionally filtered by domain."""
async with self.session() as session:
stmt = select(Page.uri, Page.path, Page.title, Page.raw_html, Page.markdown)
if domain:
stmt = stmt.where(Page.uri.like(f'%{domain}%'))
else:
stmt = stmt.where(Page.raw_html.isnot(None))
result = await session.execute(stmt)
return [dict(row._mapping) for row in result.fetchall()]
async def get_page_media(self, page_uri: str, limit: int = 50) -> List[Dict[str, Any]]:
"""Get media items from a specific page (excluding screenshots)."""
async with self.session() as session:
stmt = (
select(Media.md5_hash, Media.media_type, Media.alt_text, Media.file_size)
.join(MediaSource, Media.md5_hash == MediaSource.md5_hash)
.where(and_(
MediaSource.page_uri == page_uri,
Media.mime_type != 'image/png'
))
.distinct()
.limit(limit)
)
result = await session.execute(stmt)
return [dict(row._mapping) for row in result.fetchall()]
async def get_all_media_uri_mappings(self) -> Dict[str, str]:
"""Get all media URI to hash mappings."""
async with self.session() as session:
stmt = select(MediaSource.media_uri, MediaSource.md5_hash)
result = await session.execute(stmt)
return {row[0]: row[1] for row in result.fetchall()}
async def get_crawled_screenshot_uris(self) -> Set[str]:
"""Get page URIs that have been screenshotted (for resume support)."""
async with self.session() as session:
# Screenshots are stored with media_uri = 'screenshot:{page_uri}'
stmt = (
select(MediaSource.page_uri)
.join(Media, MediaSource.md5_hash == Media.md5_hash)
.where(Media.media_type == 'screenshot')
.distinct()
)
result = await session.execute(stmt)
return {row[0] for row in result.fetchall()}

645
html2md.py Normal file
View file

@ -0,0 +1,645 @@
#!/usr/bin/env python3
"""
Smart HTML to Markdown converter.
Detects content structure (forums, blogs, Q&A, e-commerce) and generates
clean markdown that preserves semantic meaning.
"""
from bs4 import BeautifulSoup, Tag
from typing import Optional, List, Dict, Tuple
from urllib.parse import urljoin
import re
class SmartMarkdownConverter:
"""Convert HTML to markdown with semantic structure preservation."""
def __init__(self, base_url: str = ''):
self.base_url = base_url
def convert(self, html: str) -> str:
"""Convert HTML to markdown, auto-detecting content type."""
soup = BeautifulSoup(html, 'html.parser')
# Remove noise
self._remove_noise(soup)
# Detect content type and extract accordingly
content_type, content = self._detect_and_extract(soup)
if content_type == 'forum':
return self._render_forum(content)
elif content_type == 'blog':
return self._render_blog(content)
elif content_type == 'qa':
return self._render_qa(content)
elif content_type == 'ecommerce':
return self._render_ecommerce(content)
else:
# Fallback: clean conversion of main content
return self._render_generic(soup)
def _remove_noise(self, soup: BeautifulSoup):
"""Remove navigation, scripts, styles, and other noise."""
# Remove script, style, and other non-content elements
for tag in soup(['script', 'style', 'noscript', 'iframe', 'svg']):
tag.decompose()
# Remove common navigation/chrome elements
noise_selectors = [
'nav', 'header', 'footer', 'aside',
'.sidebar', '.nav', '.navigation', '.menu', '.breadcrumb',
'.header', '.footer', '.logo', '.site-logo',
'#header', '#footer', '#nav', '#sidebar',
'.d-header', '.d-footer', # Discourse
'.pagination', '.pager',
'[role="banner"]', '[role="navigation"]', '[role="contentinfo"]',
'.cookie-banner', '.modal', '.popup', '.overlay',
'.ad', '.ads', '.advertisement', '.sponsored',
'.social-share', '.share-buttons',
]
for selector in noise_selectors:
for tag in soup.select(selector):
tag.decompose()
def _detect_and_extract(self, soup: BeautifulSoup) -> Tuple[str, any]:
"""Detect content type and extract structured content."""
# Forum detection (Discourse, phpBB, vBulletin, etc.)
forum_indicators = [
'.post_container', # Discourse archived (has avatar_container + post)
'.topic-post', # Discourse live
'.post-stream > article', # Discourse alt
'.message', # XenForo
'.postcontainer', # vBulletin
'article.boxed', # Generic
]
for post_sel in forum_indicators:
posts = soup.select(post_sel)
if len(posts) >= 2: # At least 2 posts to be a forum thread
return 'forum', self._extract_forum_posts(soup, posts)
# Q&A detection (Stack Exchange, etc.)
qa_indicators = [
('.question', '.answer'),
('#question', '.answer'),
('.question-page', '.answercell'),
]
for q_sel, a_sel in qa_indicators:
questions = soup.select(q_sel)
answers = soup.select(a_sel)
if questions or answers:
return 'qa', self._extract_qa(soup, questions, answers)
# Blog detection
blog_indicators = [
'article', '.post', '.entry', '.blog-post',
'.hentry', '[itemtype*="BlogPosting"]',
]
for selector in blog_indicators:
articles = soup.select(selector)
if articles and len(articles) <= 10: # Not a listing page
return 'blog', self._extract_blog(soup, articles)
# E-commerce detection
ecom_indicators = [
'.product', '.product-detail', '[itemtype*="Product"]',
'.product-info', '.pdp-main',
]
for selector in ecom_indicators:
products = soup.select(selector)
if products:
return 'ecommerce', self._extract_ecommerce(soup, products)
return 'generic', soup
def _extract_forum_posts(self, soup: BeautifulSoup, posts: List[Tag]) -> List[Dict]:
"""Extract forum posts with avatar, username, content, timestamp."""
extracted = []
for post in posts:
post_data = {
'avatar': None,
'username': None,
'timestamp': None,
'content': None,
'quotes': [],
}
# Avatar - look for common patterns (may be sibling or child)
avatar = (
post.select_one('.avatar_container img.avatar') or # Discourse archived
post.select_one('.avatar_container img') or
post.select_one('.avatar-container img') or
post.select_one('img.avatar') or
post.select_one('.avatar img') or
post.select_one('.avatar-flair img') or
post.select_one('.user-avatar img') or
post.select_one('.postprofile img') or
post.select_one('.author img')
)
if avatar:
src = avatar.get('src', '')
# Skip placeholder avatars
if src and '{size}' not in src:
post_data['avatar'] = self._resolve_url(src)
# Username - Discourse archived uses .user_name, live uses .username
username_el = (
post.select_one('.user_name') or # Discourse archived
post.select_one('.username') or
post.select_one('.author') or
post.select_one('.creator a') or
post.select_one('.user-card-name') or
post.select_one('a[data-user-card]') or
post.select_one('.names .name') or
post.select_one('.postprofile dt') or
post.select_one('strong.username')
)
if username_el:
post_data['username'] = username_el.get_text(strip=True)
# Timestamp
time_el = (
post.select_one('time') or
post.select_one('.post-date') or
post.select_one('.timestamp') or
post.select_one('.date') or
post.select_one('.relative-date')
)
if time_el:
post_data['timestamp'] = time_el.get('title') or time_el.get_text(strip=True)
# Content - the main post body
content_el = (
post.select_one('.post_content') or # Discourse archived
post.select_one('.cooked') or # Discourse live
post.select_one('.post-content') or
post.select_one('.message-body') or
post.select_one('.postcontent') or
post.select_one('.post_body') or
post.select_one('.entry-content') or
post.select_one('.content') or
post.select_one('.post') # Fallback to inner .post div
)
if content_el:
post_data['content'] = self._element_to_markdown(content_el)
else:
# Fallback: use whole post but try to exclude metadata
clone = BeautifulSoup(str(post), 'html.parser')
for sel in ['.avatar', '.avatar_container', '.user_name', '.username', '.author', '.date', '.post-actions', '.post-menu']:
for el in clone.select(sel):
el.decompose()
post_data['content'] = self._element_to_markdown(clone)
if post_data['content']:
extracted.append(post_data)
return extracted
def _extract_qa(self, soup: BeautifulSoup, questions: List[Tag], answers: List[Tag]) -> Dict:
"""Extract Q&A content with votes, user info."""
qa_data = {'question': None, 'answers': []}
if questions:
q = questions[0]
qa_data['question'] = {
'title': self._get_text(q, '.question-title, h1'),
'votes': self._get_text(q, '.vote-count, .js-vote-count'),
'content': self._element_to_markdown(q.select_one('.post-text, .s-prose, .question-body')),
'author': self._get_text(q, '.user-info .user-details a, .author'),
}
for a in answers:
answer_data = {
'votes': self._get_text(a, '.vote-count, .js-vote-count'),
'content': self._element_to_markdown(a.select_one('.post-text, .s-prose, .answer-body')),
'author': self._get_text(a, '.user-info .user-details a, .author'),
'accepted': bool(a.select_one('.accepted-answer, .is-accepted')),
}
if answer_data['content']:
qa_data['answers'].append(answer_data)
return qa_data
def _extract_blog(self, soup: BeautifulSoup, articles: List[Tag]) -> List[Dict]:
"""Extract blog articles with metadata."""
extracted = []
for article in articles:
article_data = {
'title': self._get_text(article, 'h1, h2, .entry-title, .post-title'),
'author': self._get_text(article, '.author, .byline, [rel="author"]'),
'date': self._get_text(article, 'time, .date, .published, .post-date'),
'content': None,
}
content_el = (
article.select_one('.entry-content') or
article.select_one('.post-content') or
article.select_one('.article-body') or
article.select_one('.content') or
article
)
article_data['content'] = self._element_to_markdown(content_el)
if article_data['content']:
extracted.append(article_data)
return extracted
def _extract_ecommerce(self, soup: BeautifulSoup, products: List[Tag]) -> List[Dict]:
"""Extract product information."""
extracted = []
for product in products:
product_data = {
'name': self._get_text(product, '.product-name, .product-title, h1, h2'),
'price': self._get_text(product, '.price, .product-price, [itemprop="price"]'),
'description': self._element_to_markdown(
product.select_one('.description, .product-description, [itemprop="description"]')
),
'image': None,
'rating': self._get_text(product, '.rating, .stars, [itemprop="ratingValue"]'),
}
img = product.select_one('.product-image img, .gallery img, [itemprop="image"]')
if img:
product_data['image'] = self._resolve_url(img.get('src', ''))
if product_data['name']:
extracted.append(product_data)
return extracted
def _element_to_markdown(self, el: Optional[Tag]) -> str:
"""Convert a single element to markdown."""
if not el:
return ''
lines = []
self._walk_element(el, lines)
return '\n'.join(lines).strip()
def _walk_element(self, el: Tag, lines: List[str], depth: int = 0):
"""Recursively walk element and build markdown lines."""
if isinstance(el, str):
text = el.strip()
if text:
lines.append(text)
return
if not isinstance(el, Tag):
return
tag = el.name.lower() if el.name else ''
# Block elements
if tag in ('p', 'div'):
content = self._inline_content(el)
if content:
lines.append(content)
lines.append('')
elif tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
level = int(tag[1])
content = self._inline_content(el)
if content:
lines.append(f"{'#' * level} {content}")
lines.append('')
elif tag == 'blockquote':
quote_lines = []
for child in el.children:
self._walk_element(child, quote_lines, depth + 1)
for line in quote_lines:
if line:
lines.append(f"> {line}")
else:
lines.append('>')
lines.append('')
elif tag in ('ul', 'ol'):
for i, li in enumerate(el.find_all('li', recursive=False)):
prefix = f"{i+1}." if tag == 'ol' else "-"
content = self._inline_content(li)
lines.append(f"{prefix} {content}")
lines.append('')
elif tag == 'pre':
code = el.get_text()
# Try to detect language from class
lang = ''
code_el = el.select_one('code')
if code_el:
classes = code_el.get('class', [])
for cls in classes:
if cls.startswith('language-') or cls.startswith('lang-'):
lang = cls.split('-', 1)[1]
break
lines.append(f"```{lang}")
lines.append(code.strip())
lines.append("```")
lines.append('')
elif tag == 'code' and el.parent and el.parent.name != 'pre':
# Inline code handled in _inline_content
pass
elif tag == 'hr':
lines.append('---')
lines.append('')
elif tag == 'br':
lines.append('')
elif tag == 'img':
src = self._resolve_url(el.get('src', ''))
alt = el.get('alt', '')
if src:
lines.append(f"![{alt}]({src})")
lines.append('')
elif tag == 'a':
# Links are handled inline
pass
elif tag == 'table':
lines.extend(self._table_to_markdown(el))
lines.append('')
else:
# Generic container - recurse
for child in el.children:
self._walk_element(child, lines, depth)
def _inline_content(self, el: Tag) -> str:
"""Extract inline content from an element, handling formatting."""
parts = []
for child in el.children:
if isinstance(child, str):
text = child.strip()
if text:
# Collapse whitespace
text = re.sub(r'\s+', ' ', text)
parts.append(text)
elif isinstance(child, Tag):
tag = child.name.lower() if child.name else ''
if tag in ('strong', 'b'):
inner = self._inline_content(child)
if inner:
parts.append(f"**{inner}**")
elif tag in ('em', 'i'):
inner = self._inline_content(child)
if inner:
parts.append(f"*{inner}*")
elif tag == 'code':
code = child.get_text()
if code:
parts.append(f"`{code}`")
elif tag == 'a':
href = self._resolve_url(child.get('href', ''))
text = self._inline_content(child) or href
if href:
parts.append(f"[{text}]({href})")
else:
parts.append(text)
elif tag == 'img':
src = self._resolve_url(child.get('src', ''))
alt = child.get('alt', '')
if src:
parts.append(f"![{alt}]({src})")
elif tag == 'br':
parts.append('\n')
elif tag in ('span', 'small', 'mark'):
# Pass through
inner = self._inline_content(child)
if inner:
parts.append(inner)
else:
# Unknown inline - just get text
inner = self._inline_content(child)
if inner:
parts.append(inner)
return ' '.join(parts).strip()
def _table_to_markdown(self, table: Tag) -> List[str]:
"""Convert HTML table to markdown."""
lines = []
rows = table.find_all('tr')
if not rows:
return lines
# Extract headers
header_row = rows[0]
headers = [self._inline_content(th) for th in header_row.find_all(['th', 'td'])]
if headers:
lines.append('| ' + ' | '.join(headers) + ' |')
lines.append('| ' + ' | '.join(['---'] * len(headers)) + ' |')
# Extract body rows
for row in rows[1:]:
cells = [self._inline_content(td) for td in row.find_all(['td', 'th'])]
if cells:
# Pad cells if needed
while len(cells) < len(headers):
cells.append('')
lines.append('| ' + ' | '.join(cells) + ' |')
return lines
def _get_text(self, el: Tag, selectors: str) -> str:
"""Get text from first matching selector."""
for selector in selectors.split(','):
found = el.select_one(selector.strip())
if found:
return found.get_text(strip=True)
return ''
def _resolve_url(self, url: str) -> str:
"""Resolve relative URL to absolute."""
if not url or url.startswith('data:'):
return url
if self.base_url and not url.startswith(('http://', 'https://', '//')):
return urljoin(self.base_url, url)
return url
# Renderers for different content types
def _render_forum(self, posts: List[Dict]) -> str:
"""Render forum posts to markdown."""
lines = []
for i, post in enumerate(posts):
if i > 0:
lines.append('')
lines.append('---')
lines.append('')
# Post header with avatar and username
# Always include avatar (placeholder if none) to maintain grid layout
if post['avatar']:
avatar_md = f"![avatar]({post['avatar']})"
else:
# Use # as placeholder - JS will detect and replace with initial
avatar_md = "![avatar](#)"
header_parts = [avatar_md]
if post['username']:
header_parts.append(f"**{post['username']}**")
if post['timestamp']:
header_parts.append(f"*{post['timestamp']}*")
lines.append(' '.join(header_parts))
lines.append('')
# Post content
if post['content']:
lines.append(post['content'])
return '\n'.join(lines)
def _render_qa(self, qa: Dict) -> str:
"""Render Q&A content to markdown."""
lines = []
if qa['question']:
q = qa['question']
if q['title']:
lines.append(f"# {q['title']}")
lines.append('')
if q['votes']:
lines.append(f"**{q['votes']} votes**")
if q['author']:
lines.append(f"*Asked by {q['author']}*")
lines.append('')
if q['content']:
lines.append(q['content'])
lines.append('')
if qa['answers']:
lines.append('---')
lines.append('')
lines.append(f"## {len(qa['answers'])} Answers")
lines.append('')
for answer in qa['answers']:
if answer['accepted']:
lines.append('### ✓ Accepted Answer')
else:
lines.append('### Answer')
if answer['votes']:
lines.append(f"**{answer['votes']} votes**")
if answer['author']:
lines.append(f"*By {answer['author']}*")
lines.append('')
if answer['content']:
lines.append(answer['content'])
lines.append('')
lines.append('---')
lines.append('')
return '\n'.join(lines)
def _render_blog(self, articles: List[Dict]) -> str:
"""Render blog articles to markdown."""
lines = []
for article in articles:
if article['title']:
lines.append(f"# {article['title']}")
lines.append('')
meta = []
if article['author']:
meta.append(f"By {article['author']}")
if article['date']:
meta.append(article['date'])
if meta:
lines.append(f"*{' | '.join(meta)}*")
lines.append('')
if article['content']:
lines.append(article['content'])
return '\n'.join(lines)
def _render_ecommerce(self, products: List[Dict]) -> str:
"""Render product info to markdown."""
lines = []
for product in products:
if product['name']:
lines.append(f"# {product['name']}")
lines.append('')
if product['image']:
lines.append(f"![{product['name']}]({product['image']})")
lines.append('')
if product['price']:
lines.append(f"**Price:** {product['price']}")
lines.append('')
if product['rating']:
lines.append(f"**Rating:** {product['rating']}")
lines.append('')
if product['description']:
lines.append(product['description'])
return '\n'.join(lines)
def _render_generic(self, soup: BeautifulSoup) -> str:
"""Fallback generic rendering."""
# Find main content area
main = (
soup.select_one('main') or
soup.select_one('article') or
soup.select_one('.content') or
soup.select_one('#content') or
soup.select_one('.main') or
soup.body or
soup
)
return self._element_to_markdown(main)
def html_to_markdown(html: str, base_url: str = '') -> str:
"""Convert HTML to markdown with smart structure detection.
Args:
html: Raw HTML content
base_url: Base URL for resolving relative links
Returns:
Clean markdown with preserved semantic structure
"""
converter = SmartMarkdownConverter(base_url)
return converter.convert(html)
if __name__ == '__main__':
import sys
if len(sys.argv) < 2:
print("Usage: python html2md.py <file.html> [base_url]")
sys.exit(1)
with open(sys.argv[1]) as f:
html = f.read()
base_url = sys.argv[2] if len(sys.argv) > 2 else ''
print(html_to_markdown(html, base_url))

75
make-executable.sh Executable file
View file

@ -0,0 +1,75 @@
#!/bin/bash
#
# make-executable.sh - Create self-extracting searchable archive
#
# Usage:
# ./make-executable.sh archive.tar.gz
# ./archive.run
#
# The resulting .run file is a single executable that:
# 1. Extracts the Python serve.py to /tmp
# 2. Runs it pointing at itself as the tarball source
# 3. Serves the archive on http://localhost:6543
#
# Requirements:
# - gcc with zlib (-lz)
# - python3 with pyramid installed (on target system)
set -e
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
BOOTSTRAP_C="$SCRIPT_DIR/bootstrap.c"
if [ $# -lt 1 ]; then
echo "Usage: $0 <archive.tar.gz> [output.run]"
echo ""
echo "Creates a self-extracting searchable archive."
echo "The output file can be run directly to start the search server."
exit 1
fi
TARBALL="$1"
OUTPUT="${2:-${TARBALL%.tar.gz}.run}"
if [ ! -f "$TARBALL" ]; then
echo "Error: Tarball not found: $TARBALL"
exit 1
fi
if [ ! -f "$BOOTSTRAP_C" ]; then
echo "Error: bootstrap.c not found: $BOOTSTRAP_C"
exit 1
fi
# Create temp directory
TMPDIR=$(mktemp -d)
trap "rm -rf $TMPDIR" EXIT
echo "Compiling bootstrap..."
gcc -O2 -o "$TMPDIR/bootstrap" "$BOOTSTRAP_C" -lz
BOOTSTRAP_SIZE=$(stat -c%s "$TMPDIR/bootstrap" 2>/dev/null || stat -f%z "$TMPDIR/bootstrap")
echo "Bootstrap size: $BOOTSTRAP_SIZE bytes"
echo "Creating self-extracting archive..."
# Concatenate: bootstrap + tarball + magic + offset
cat "$TMPDIR/bootstrap" "$TARBALL" > "$TMPDIR/combined"
# Append magic marker
echo -n "NEOPIG" >> "$TMPDIR/combined"
# Append offset as 16-char hex string
printf '%016x' "$BOOTSTRAP_SIZE" >> "$TMPDIR/combined"
# Make executable and move to output
chmod +x "$TMPDIR/combined"
mv "$TMPDIR/combined" "$OUTPUT"
FINAL_SIZE=$(stat -c%s "$OUTPUT" 2>/dev/null || stat -f%z "$OUTPUT")
echo ""
echo "Created: $OUTPUT ($FINAL_SIZE bytes)"
echo ""
echo "To run:"
echo " ./$OUTPUT"
echo " # Opens http://localhost:6543"

279
neopig.py
View file

@ -80,9 +80,11 @@ class NeoPig:
user_agent: str = "neopig/1.0 (ethical image crawler)",
screenshot_config: ScreenshotConfig = None,
fast_mode: bool = False,
trim_wrapper: bool = False,
):
self.db = Database(db_path)
self.vault = ImageVault(vault_path)
self.trim_wrapper = trim_wrapper
# Triple filevault system: html_vault/, media_vault/, and linkpeek_vault/
self.domain_vaults = VaultManager(
html_vault_base=f"{vault_path}/html_vault",
@ -107,6 +109,7 @@ class NeoPig:
'media_downloaded': 0,
'media_new': 0, # New media (for git commit)
'duplicates_skipped': 0,
'content_exists': 0, # Same content (MD5) already in vault
'screenshots_taken': 0,
'errors': 0,
'bytes_downloaded': 0, # Total bytes fetched from network
@ -164,7 +167,12 @@ class NeoPig:
logger.debug(f"Failed to save state: {e}")
def _load_state(self, target_url: str) -> bool:
"""Load saved crawl state. Returns True if state was loaded."""
"""Load saved crawl state. Returns True if state was loaded.
Note: seen_media and seen_screenshots are NOT loaded from state file -
they come from the database (source of truth for successful downloads).
Only seen_pages (for fast mode) and stats are loaded from state.
"""
state_file = self._get_state_file(target_url)
if not state_file.exists():
return False
@ -172,8 +180,11 @@ class NeoPig:
try:
with open(state_file, 'r') as f:
state = json.load(f)
self.seen_media = set(state.get('seen_media', []))
self.seen_screenshots = set(state.get('seen_screenshots', []))
# NOTE: Do NOT load seen_media or seen_screenshots from state file!
# Database is the source of truth - state file may have entries
# added before success (old buggy code). seen_media/seen_screenshots
# are loaded from DB in main() before crawl() is called.
# In fast mode, skip already-crawled pages for speed
# In normal mode, re-fetch pages to detect content changes (git handles versioning)
if self.fast_mode:
@ -184,9 +195,9 @@ class NeoPig:
if key in saved_stats:
self.stats[key] = saved_stats[key]
if self.fast_mode:
logger.info(f"Fast resume: skipping {len(self.seen_pages)} pages, {len(self.seen_media)} media, {len(self.seen_screenshots)} screenshots")
logger.info(f"Fast resume: skipping {len(self.seen_pages)} pages (media/screenshots from DB)")
else:
logger.info(f"Resuming: {len(self.seen_media)} media, {len(self.seen_screenshots)} screenshots (pages will be re-checked)")
logger.info(f"Resume: loading stats from state (media/screenshots from DB)")
return True
except Exception as e:
logger.warning(f"Could not load state: {e}")
@ -231,20 +242,24 @@ class NeoPig:
if title_tag:
title = title_tag.get_text(strip=True)
# Convert to markdown using html2text
# Convert to markdown using smart converter
markdown = ''
try:
import html2text
h = html2text.HTML2Text()
h.ignore_links = False
h.ignore_images = False
h.body_width = 0 # No wrapping
h.ignore_emphasis = False
if base_url:
h.baseurl = base_url # Resolve relative URLs to absolute
markdown = h.handle(html)[:200000]
except ImportError:
pass
from html2md import html_to_markdown
markdown = html_to_markdown(html, base_url=base_url)[:200000]
except Exception:
# Fallback to html2text
try:
import html2text
h = html2text.HTML2Text()
h.ignore_links = False
h.ignore_images = False
h.body_width = 0
if base_url:
h.baseurl = base_url
markdown = h.handle(html)[:200000]
except ImportError:
pass
# Remove script and style elements for plain text
for tag in soup(['script', 'style', 'nav', 'header', 'footer']):
@ -428,52 +443,64 @@ class NeoPig:
return f"{b/(1024*1024*1024):.1f}GB"
# Track total URIs discovered for progress bar
uris_total = [1] # Start with 1 for target page, use list for mutability in closure
uris_total = [0] # Use list for mutability in closure
def on_uris_total(total: int):
uris_total[0] = total
self.pbar.total = total
self.pbar.refresh()
if self.pbar and total > 0:
self.pbar.total = total
self.pbar.refresh()
# Create progress bar with actual bar display
# Start with previous progress if resuming
# Create progress bar with dynamic total
initial_pages = len(self.seen_pages)
self.pbar = tqdm(
total=max(1, initial_pages),
total=1, # Start with 1, will be updated as links are discovered
initial=initial_pages,
unit="pages",
dynamic_ncols=True,
bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}] {postfix}',
bar_format='{n_fmt}/{total_fmt} pages [{elapsed}] {postfix}',
mininterval=0.1,
)
self.pbar.set_postfix_str(
f"media: {self.stats['media_downloaded']}/{self.stats['media_found']}, "
f"new: {self.stats['media_downloaded']}, "
f"skip: {self.stats['duplicates_skipped']}, "
f"dup: {self.stats['content_exists']}, "
f"found: {self.stats['media_found']}, "
f"ss: {self.stats['screenshots_taken']}, "
f"err: {self.stats['errors']}, "
f"{format_size(self.stats['bytes_stored'])} stored ({format_size(self.stats['bytes_downloaded'])} fetched)"
f"{format_size(self.stats['bytes_stored'])} stored"
)
def update_pbar():
self.pbar.set_postfix_str(
f"media: {self.stats['media_downloaded']}/{self.stats['media_found']}, "
f"new: {self.stats['media_downloaded']}, "
f"skip: {self.stats['duplicates_skipped']}, "
f"dup: {self.stats['content_exists']}, "
f"found: {self.stats['media_found']}, "
f"ss: {self.stats['screenshots_taken']}, "
f"err: {self.stats['errors']}, "
f"{format_size(self.stats['bytes_stored'])} stored ({format_size(self.stats['bytes_downloaded'])} fetched)"
f"{format_size(self.stats['bytes_stored'])} stored"
)
self.pbar.refresh()
# Media callback - called for each discovered media item
async def on_media_discovered(item: Dict[str, Any]):
url = item.get('url')
if not url or url in self.seen_media:
if not url:
return
if url in self.seen_media:
# Debug: log when skipping
if self.stats['media_found'] < 5: # Only first few
logger.debug(f"SKIP (in seen_media): {url[:80]}")
return
self.seen_media.add(url)
self.stats['media_found'] += 1
update_pbar()
if download_media:
await self._process_media_item(item, job_id, keywords)
success = await self._process_media_item(item, job_id, keywords)
if success:
self.seen_media.add(url) # Only mark seen after success
update_pbar()
self._save_state(target_uri)
# Note: Screenshots are now captured per-page in on_page_fetched,
@ -551,8 +578,8 @@ class NeoPig:
item: Dict[str, Any],
job_id: int,
keywords: List[str]
):
"""Download and store a media item with page context.
) -> bool:
"""Download and store a media item with page context. Returns True on success.
Implements the skeleton key approach:
- If detail_page_url is set, resolves canonical image URL
@ -602,7 +629,7 @@ class NeoPig:
if existing_hash:
self.stats['duplicates_skipped'] += 1
logger.debug(f"Already crawled: {media_uri} from {page_uri}")
return
return True # Already have it, consider success
# Check if content already in vault (same MD5 = same content)
# We still need to add the new page context even if content exists
@ -615,7 +642,7 @@ class NeoPig:
result = await self.fetcher.fetch_media(media_uri)
if not result:
self.stats['errors'] += 1
return
return False
md5_hash = result['md5_hash']
self.stats['bytes_downloaded'] += result.get('size', 0)
@ -639,9 +666,9 @@ class NeoPig:
searchable_text=searchable_text,
crawl_job_id=job_id,
)
self.stats['duplicates_skipped'] += 1
self.stats['content_exists'] += 1
logger.debug(f"Content exists, added context: {md5_hash} from {page_uri}")
return
return True
# Store in vault (new content)
ext = self._get_extension(media_uri, result.get('mime_type', ''))
@ -674,10 +701,12 @@ class NeoPig:
self.stats['media_downloaded'] += 1
logger.debug(f"Stored: {md5_hash} ({media_uri})")
return True
except Exception as e:
logger.warning(f"Failed to process {media_uri}: {e}")
self.stats['errors'] += 1
return False
async def _capture_page_screenshot(
self,
@ -699,12 +728,14 @@ class NeoPig:
content_length: Raw HTML content length for dynamic delay calculation
"""
if not self.screenshot_config.enabled:
logger.debug(f"Screenshots disabled, skipping {page_uri}")
return
if page_uri in self.seen_screenshots:
logger.debug(f"Screenshot already exists for {page_uri}")
return
self.seen_screenshots.add(page_uri)
logger.info(f"Taking screenshot: {page_uri}")
try:
# Enforce crawl delay before screenshot (headless browser makes HTTP request)
@ -748,6 +779,7 @@ class NeoPig:
)
self.stats['screenshots_taken'] += 1
self.seen_screenshots.add(page_uri) # Only mark seen after success
logger.debug(f"Screenshot captured: {page_uri} -> {md5_hash}")
except Exception as e:
@ -781,22 +813,54 @@ class NeoPig:
return 'bin'
async def backfill_markdown(db_path: str, domain_filter: str = None):
"""Re-process stored HTML to regenerate markdown with absolute URLs.
def trim_html_wrapper(html: str) -> str:
"""Strip nav, header, footer, sidebar, and logo elements from HTML.
Useful for cleaning up Discourse and similar sites before markdown conversion.
"""
from bs4 import BeautifulSoup
soup = BeautifulSoup(html, 'html.parser')
# Remove common wrapper elements
selectors_to_remove = [
'nav', 'header', 'footer', 'aside',
'.sidebar', '.nav', '.navigation', '.menu',
'.header', '.footer', '.logo', '.site-logo',
'#header', '#footer', '#nav', '#sidebar',
'.d-header', '.d-footer', # Discourse specific
'.header-wrapper', '.footer-wrapper',
'[role="banner"]', '[role="navigation"]', '[role="contentinfo"]',
]
for selector in selectors_to_remove:
for tag in soup.select(selector):
tag.decompose()
# Remove site logo images (be specific to avoid removing content images)
for img in soup.find_all('img'):
src = img.get('src', '').lower()
alt = img.get('alt', '').lower()
cls = ' '.join(img.get('class', [])).lower()
# Only remove if it's clearly a site logo, not general icons
is_logo = 'logo' in cls or 'brand' in cls or 'site-logo' in src
is_logo = is_logo or (alt and ('logo' in alt or 'brand' in alt))
if is_logo:
img.decompose()
return str(soup)
async def backfill_markdown(db_path: str, domain_filter: str = None, trim_wrapper: bool = False):
"""Re-process stored HTML to regenerate markdown with smart structure detection.
Args:
db_path: Path to SQLite database
domain_filter: Only process pages matching this domain (e.g., 'example.com')
trim_wrapper: Strip nav/header/footer/logo before conversion (applied before smart conversion)
"""
import html2text
from html2md import html_to_markdown
import aiosqlite
h = html2text.HTML2Text()
h.ignore_links = False
h.ignore_images = False
h.body_width = 0
h.ignore_emphasis = False
async with aiosqlite.connect(db_path) as db:
db.row_factory = aiosqlite.Row
@ -814,6 +878,9 @@ async def backfill_markdown(db_path: str, domain_filter: str = None):
params = ()
logger.info("Backfilling markdown for ALL pages (no domain filter)")
if trim_wrapper:
logger.info("Trim wrapper enabled: stripping nav/header/footer/logo before conversion")
cursor = await db.execute(count_sql, params)
total = (await cursor.fetchone())[0]
logger.info(f"Found {total} pages to process...")
@ -826,14 +893,16 @@ async def backfill_markdown(db_path: str, domain_filter: str = None):
if not row['raw_html']:
continue
h.baseurl = row['uri']
try:
new_markdown = h.handle(row['raw_html'])[:200000]
raw = row['raw_html']
if trim_wrapper:
raw = trim_html_wrapper(raw)
new_markdown = html_to_markdown(raw, base_url=row['uri'])[:200000]
await db.execute("UPDATE pages SET markdown = ? WHERE id = ?", (new_markdown, row['id']))
updated += 1
if updated % 100 == 0:
await db.commit()
logger.info(f" Processed {updated}/{len(rows)} pages...")
logger.info(f" Processed {updated}/{total} pages...")
except Exception as e:
logger.warning(f"Error processing {row['uri']}: {e}")
@ -883,14 +952,14 @@ async def main():
parser.add_argument(
"--db",
default="neopig.db",
help="Database path (default: neopig.db)"
default="data/neopig.db",
help="Database path (default: data/neopig.db)"
)
parser.add_argument(
"--vault",
default="vault",
help="Vault storage path (default: vault)"
default="data/vault",
help="Vault storage path (default: data/vault)"
)
parser.add_argument(
@ -964,8 +1033,33 @@ async def main():
help="Re-process stored HTML for DOMAIN to regenerate markdown with absolute URLs (e.g., example.com)"
)
parser.add_argument(
"--trim-wrapper",
action="store_true",
help="With --backfill-markdown: strip nav/header/footer/logo before conversion"
)
parser.add_argument(
"--serve",
action="store_true",
help="Start SERP server to watch crawl live"
)
parser.add_argument(
"--port",
type=int,
default=8000,
help="Port for SERP server (default: 8000)"
)
args = parser.parse_args()
# Create data directory if needed
from pathlib import Path
db_dir = Path(args.db).parent
if db_dir and str(db_dir) != '.':
db_dir.mkdir(parents=True, exist_ok=True)
# Setup logging to work with tqdm progress bars
setup_logging(level=logging.DEBUG if args.verbose else logging.INFO)
@ -982,7 +1076,7 @@ async def main():
# Handle --backfill-markdown
if args.backfill_markdown:
await backfill_markdown(args.db, domain_filter=args.backfill_markdown)
await backfill_markdown(args.db, domain_filter=args.backfill_markdown, trim_wrapper=args.trim_wrapper)
return
# Require targets for crawling
@ -1018,6 +1112,7 @@ async def main():
vault_path=args.vault,
screenshot_config=screenshot_config,
fast_mode=args.fast,
trim_wrapper=args.trim_wrapper,
)
await pig.init()
@ -1030,25 +1125,69 @@ async def main():
pig._clear_state(target)
logger.info("Starting fresh (state files cleared)")
else:
# Load previously crawled media URIs from DB to enable resume
# Load previously crawled media/screenshots from DB (source of truth)
crawled_media = await pig.db.get_crawled_media_uris()
if crawled_media:
logger.info(f"Resuming: {len(crawled_media)} media already in database")
crawled_screenshots = await pig.db.get_crawled_screenshot_uris()
if crawled_media or crawled_screenshots:
logger.info(f"Resuming: {len(crawled_media)} media, {len(crawled_screenshots)} screenshots in database")
pig.seen_media = crawled_media
pig.seen_screenshots = crawled_screenshots
# Crawl all targets concurrently
async def crawl_target(target: str):
logger.info(f"=== Starting crawl: {target} ===")
return await pig.crawl(
target_uri=target,
keywords=args.keywords,
mode=mode,
depth=args.depth,
max_pages=args.max_pages,
download_media=not args.no_download,
)
# Start SERP server if requested
serp_process = None
if args.serve:
import subprocess
import sys
serp_script = Path(__file__).parent / 'serp.py'
if serp_script.exists():
serp_cmd = [
sys.executable, str(serp_script),
'--port', str(args.port),
'--db', args.db,
'--vault', args.vault,
]
serp_process = subprocess.Popen(
serp_cmd,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
logger.info(f"SERP server started at http://localhost:{args.port}")
logger.info("Watch the crawl live - pages appear as they're indexed!")
else:
logger.warning("serp.py not found - --serve disabled")
await asyncio.gather(*[crawl_target(t) for t in args.targets])
try:
# Crawl all targets concurrently
async def crawl_target(target: str):
logger.info(f"=== Starting crawl: {target} ===")
return await pig.crawl(
target_uri=target,
keywords=args.keywords,
mode=mode,
depth=args.depth,
max_pages=args.max_pages,
download_media=not args.no_download,
)
await asyncio.gather(*[crawl_target(t) for t in args.targets])
# Keep SERP server running after crawl
if serp_process:
logger.info(f"Crawl complete. SERP server still running at http://localhost:{args.port}")
logger.info("Press Ctrl+C to stop...")
try:
serp_process.wait()
except KeyboardInterrupt:
pass
finally:
# Cleanup SERP server
if serp_process and serp_process.poll() is None:
logger.info("Stopping SERP server...")
serp_process.terminate()
try:
serp_process.wait(timeout=5)
except subprocess.TimeoutExpired:
serp_process.kill()
if __name__ == "__main__":

1
requirements.txt
View file

@ -6,6 +6,7 @@ html5lib>=1.1
miniuri>=0.1.0
# Database
sqlalchemy[asyncio]>=2.0.0
aiosqlite>=0.19.0
# Storage

1606
serp.py
View file

File diff suppressed because it is too large Load diff