engineering/pig.py
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

modified: .gitignore

Browse source 
modified:   database.py
	modified:   html2md.py
	modified:   neopig.py
	modified:   screenshot.py
	modified:   serp.py
This commit is contained in:
Russell Ballestrini 2025-12-30 11:35:25 -05:00
parent 3952978ba4
commit 293ed64a3b
6 changed files with 158 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -8,7 +8,6 @@ venv/
# Database and vault (generated data)
*.db
vault/
test_vault/
# Vendored dependencies

15
database.py
View file

@ -669,6 +669,21 @@ class Database:
row = result.fetchone()
return row[0] if row else None
async def get_page_screenshots(self, page_uri: str) -> List[str]:
"""Get all screenshot hashes for a page (for chunked screenshots). Returns list of md5_hash ordered by file size desc."""
async with self.session() as session:
stmt = (
select(Media.md5_hash)
.join(MediaSource, Media.md5_hash == MediaSource.md5_hash)
.where(and_(
MediaSource.page_uri == page_uri,
Media.media_type == 'screenshot'
))
.order_by(Media.file_size.desc())
)
result = await session.execute(stmt)
return [row[0] for row in result.fetchall()]
async def get_recent_media(self, limit: int = 50, media_type: str = None) -> List[Dict[str, Any]]:
"""Get recently discovered media."""
async with self.session() as session:

19
html2md.py
View file

@ -297,10 +297,21 @@ class SmartMarkdownConverter:
# Block elements
if tag in ('p', 'div'):
content = self._inline_content(el)
if content:
lines.append(content)
lines.append('')
# Check if contains block-level elements (pre, ul, ol, etc.)
block_tags = {'pre', 'ul', 'ol', 'blockquote', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'table'}
has_blocks = any(
isinstance(c, Tag) and c.name and c.name.lower() in block_tags
for c in el.children
)
if has_blocks:
# Recursively process children to preserve block structure
for child in el.children:
self._walk_element(child, lines, depth)
else:
content = self._inline_content(el)
if content:
lines.append(content)
lines.append('')
elif tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
level = int(tag[1])

53
neopig.py
View file

@ -915,13 +915,14 @@ def trim_html_wrapper(html: str) -> str:
return str(soup)
async def backfill_markdown(db_path: str, domain_filter: str = None, trim_wrapper: bool = False):
async def backfill_markdown(db_path: str, domain_filter: str = None, trim_wrapper: bool = False, quiet: bool = False):
"""Re-process stored HTML to regenerate markdown with smart structure detection.
Args:
db_path: Path to SQLite database
domain_filter: Only process pages matching this domain (e.g., 'example.com')
trim_wrapper: Strip nav/header/footer/logo before conversion (applied before smart conversion)
quiet: Disable progress bar
"""
from html2md import html_to_markdown
import aiosqlite
@ -933,28 +934,25 @@ async def backfill_markdown(db_path: str, domain_filter: str = None, trim_wrappe
if domain_filter:
# Match domain in URI
pattern = f"%://{domain_filter}%" if '://' not in domain_filter else f"%{domain_filter}%"
count_sql = "SELECT COUNT(*) FROM pages WHERE raw_html IS NOT NULL AND uri LIKE ?"
select_sql = "SELECT id, uri, raw_html FROM pages WHERE raw_html IS NOT NULL AND uri LIKE ?"
params = (pattern,)
logger.info(f"Backfilling markdown for domain: {domain_filter}")
else:
count_sql = "SELECT COUNT(*) FROM pages WHERE raw_html IS NOT NULL"
select_sql = "SELECT id, uri, raw_html FROM pages WHERE raw_html IS NOT NULL"
params = ()
logger.info("Backfilling markdown for ALL pages (no domain filter)")
logger.info("Backfilling markdown for ALL pages")
if trim_wrapper:
logger.info("Trim wrapper enabled: stripping nav/header/footer/logo before conversion")
cursor = await db.execute(count_sql, params)
total = (await cursor.fetchone())[0]
logger.info(f"Found {total} pages to process...")
logger.info("Trim wrapper enabled")
cursor = await db.execute(select_sql, params)
rows = await cursor.fetchall()
total = len(rows)
logger.info(f"Found {total} pages to process")
updated = 0
for row in rows:
errors = 0
for row in tqdm(rows, desc="Markdown", unit="pages", disable=quiet):
if not row['raw_html']:
continue
@ -967,12 +965,12 @@ async def backfill_markdown(db_path: str, domain_filter: str = None, trim_wrappe
updated += 1
if updated % 100 == 0:
await db.commit()
logger.info(f" Processed {updated}/{total} pages...")
except Exception as e:
logger.warning(f"Error processing {row['uri']}: {e}")
errors += 1
logger.debug(f"Error processing {row['uri']}: {e}")
await db.commit()
logger.info(f"Backfill complete: {updated} pages updated")
logger.info(f"Backfill complete: {updated} updated, {errors} errors")
async def backfill_screenshots(
@ -981,6 +979,7 @@ async def backfill_screenshots(
domain_filter: str = None,
delete_old: bool = True,
fast_mode: bool = False,
quiet: bool = False,
):
"""Re-capture screenshots as JPEG to replace old PNGs.
@ -990,6 +989,7 @@ async def backfill_screenshots(
domain_filter: Only process pages matching this domain
delete_old: Delete old PNG files after successful JPEG capture
fast_mode: Skip delay between captures (for sites without robots.txt)
quiet: Disable progress bar
"""
from screenshot import ScreenshotCapture, ScreenshotConfig
from storage import ImageVault
@ -1054,8 +1054,22 @@ async def backfill_screenshots(
domain_last_fetched = {} # Track last fetch time per domain
crawl_delay = 2.0 # Default crawl delay in seconds
pbar = tqdm(total=total, initial=skipped_count, desc="Screenshots", unit="pages")
for row in rows:
# Concurrent workers in fast mode (based on CPU count)
import multiprocessing
max_workers = multiprocessing.cpu_count() if fast_mode else 1
semaphore = asyncio.Semaphore(max_workers)
if fast_mode and max_workers > 1:
logger.info(f"Fast mode: {max_workers} concurrent workers")
pbar = tqdm(total=total, initial=skipped_count, desc="Screenshots", unit="pages", disable=quiet)
# Lock for thread-safe updates
state_lock = asyncio.Lock()
stats = {'captured': 0, 'failed': 0, 'bytes_saved': 0}
async def process_row(row):
nonlocal completed_uris
async with semaphore:
page_uri = row.page_uri
old_hash = row.md5_hash
domain = Uri(page_uri).hostname.lower()
@ -1337,6 +1351,12 @@ async def main():
help="With --backfill-markdown: strip nav/header/footer/logo before conversion"
)
parser.add_argument(
"-q", "--quiet",
action="store_true",
help="Disable progress bars"
)
parser.add_argument(
"--backfill-screenshots",
metavar="DOMAIN",
@ -1386,7 +1406,7 @@ async def main():
# Handle --backfill-markdown
if args.backfill_markdown:
await backfill_markdown(args.db, domain_filter=args.backfill_markdown, trim_wrapper=args.trim_wrapper)
await backfill_markdown(args.db, domain_filter=args.backfill_markdown, trim_wrapper=args.trim_wrapper, quiet=args.quiet)
return
# Handle --backfill-screenshots
@ -1416,6 +1436,7 @@ async def main():
domain_filter=args.backfill_screenshots,
delete_old=not args.keep_old_screenshots,
fast_mode=args.fast,
quiet=args.quiet,
)
finally:
if serp_process:

10
screenshot.py
View file

@ -194,13 +194,13 @@ class ScreenshotCapture:
content_length: Length of raw HTML content in bytes
Returns:
Delay in milliseconds (min 2000ms, max 30000ms)
Delay in milliseconds (min 3000ms, max 60000ms)
"""
# Base delay of 2 seconds
base_delay = 2000
# Base delay of 3 seconds
base_delay = 3000
# Add 2 seconds per 50KB of content, up to a max of 28 seconds
additional_delay = min((content_length // 50000) * 2000, 28000)
# Add 3 seconds per 50KB of content, up to a max of 57 seconds
additional_delay = min((content_length // 50000) * 3000, 57000)
return base_delay + additional_delay

116
serp.py
View file

@ -1912,13 +1912,12 @@ async def view_page(
escaped = html_module.escape(page["content"][:50000])
content_html = f"<pre style='white-space:pre-wrap;'>{escaped}</pre>"
# Find screenshot
# Find all screenshots (may be chunked for tall pages)
screenshot_hashes = await db.get_page_screenshots(uri)
screenshot_html = ""
screenshot_hash = await db.get_page_screenshot(uri)
if screenshot_hash:
screenshot_html = f'''<a href="/media/{screenshot_hash}" target="_blank">
<img src="/media/{screenshot_hash}" alt="Page screenshot" style="max-width:100%;max-height:50vh;border-radius:8px;">
</a>'''
if screenshot_hashes:
imgs = ''.join(f'<img src="/media/{h}" alt="Page screenshot chunk" style="width:100%;display:block;">' for h in screenshot_hashes)
screenshot_html = f'<div class="screenshot-stack">{imgs}</div>'
# Find media from this page
media_items = await db.get_page_media(uri)
@ -1942,13 +1941,36 @@ async def view_page(
<div class="media-grid">{''.join(media_cards)}</div>
</div>'''
has_screenshot = bool(screenshot_html)
return f"""<!DOCTYPE html>
<html>
<head>
<title>{html_module.escape(page_title)} - neopig</title>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github-dark.min.css">
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
<style>{VIEW_CSS}</style>
<style>
{VIEW_CSS}
.page-layout {{ display: flex; gap: 20px; }}
.page-main {{ flex: 1; min-width: 0; }}
.page-sidebar {{ width: 400px; flex-shrink: 0; transition: width 0.3s, opacity 0.3s; }}
.page-sidebar.collapsed {{ width: 40px; overflow: hidden; }}
.page-sidebar.collapsed .screenshot-stack {{ opacity: 0; }}
.sidebar-toggle {{
cursor: pointer; padding: 8px 12px; background: #333; border-radius: 4px;
margin-bottom: 10px; display: inline-block; user-select: none;
}}
.sidebar-toggle:hover {{ background: #444; }}
.screenshot-stack {{ max-height: 80vh; overflow-y: auto; border-radius: 8px; }}
.screenshot-stack img {{ border-radius: 0; }}
.screenshot-stack img:first-child {{ border-radius: 8px 8px 0 0; }}
.screenshot-stack img:last-child {{ border-radius: 0 0 8px 8px; }}
.screenshot-stack img:only-child {{ border-radius: 8px; }}
@media (max-width: 900px) {{
.page-layout {{ flex-direction: column; }}
.page-sidebar {{ width: 100%; }}
.page-sidebar.collapsed {{ width: 100%; height: 40px; }}
}}
</style>
</head>
<body>
<div class="nav">
@ -1958,19 +1980,42 @@ async def view_page(
<a href="/random">Random</a>
</div>
<div class="container">
<div class="hero">
{screenshot_html or ''}
</div>
<h1>{html_module.escape(page_title)}</h1>
<div class="meta">
<div class="meta-row"><span class="meta-label">URL:</span><span class="meta-value"><a href="{uri}" target="_blank">{uri}</a></span></div>
<div class="meta-row"><span class="meta-label">Media:</span><span class="meta-value">{len(media_items)} items</span></div>
</div>
{media_grid}
<h3>Page Content</h3>
<div class="content-rendered">{content_html or '<p>No content available</p>'}</div>
<div class="page-layout">
<div class="page-main">
{media_grid}
<h3>Page Content</h3>
<div class="content-rendered">{content_html or '<p>No content available</p>'}</div>
</div>
{f'''<div class="page-sidebar" id="sidebar">
<span class="sidebar-toggle" onclick="toggleSidebar()">◀ Screenshot</span>
{screenshot_html}
</div>''' if has_screenshot else ''}
</div>
</div>
<script>hljs.highlightAll();</script>
<script>
hljs.highlightAll();
function toggleSidebar() {{
const sb = document.getElementById('sidebar');
const toggle = sb.querySelector('.sidebar-toggle');
sb.classList.toggle('collapsed');
const isCollapsed = sb.classList.contains('collapsed');
toggle.textContent = isCollapsed ? '▶ Screenshot' : '◀ Screenshot';
localStorage.setItem('neopig_sidebar_collapsed', isCollapsed);
}}
// Restore state from localStorage
if (localStorage.getItem('neopig_sidebar_collapsed') === 'true') {{
const sb = document.getElementById('sidebar');
if (sb) {{
sb.classList.add('collapsed');
sb.querySelector('.sidebar-toggle').textContent = '▶ Screenshot';
}}
}}
</script>
{'' if noai else f'''<script>
window.UNCLOSEAI_SYSTEM_PROMPT = "This is an archived copy of {source_domain} preserved by neopig. The original site may no longer exist. You are viewing: {page_title} ({uri}). Help users explore the preserved discussions, media, and pages.";
</script>
@ -2269,27 +2314,38 @@ async def get_stats_endpoint():
@app.get("/random")
async def random_media():
"""Redirect to a random media item."""
async def random_item(type: str = Query(None, description="Type: media or page (random if not specified)")):
"""Redirect to a random media item or page."""
from sqlalchemy import select, func
from database import Media
from database import Media, Page
import random
async with db.session() as session:
# Get a random media item (excluding screenshots)
stmt = (
select(Media.md5_hash)
.where(Media.media_type != 'screenshot')
.order_by(func.random())
.limit(1)
)
result = await session.execute(stmt)
row = result.fetchone()
# If no type specified, randomly pick between media and page
if type is None:
type = random.choice(["media", "page"])
if row:
return RedirectResponse(url=f"/view/{row[0]}", status_code=302)
async with db.session() as session:
if type == "page":
# Get a random page
stmt = select(Page.uri).order_by(func.random()).limit(1)
result = await session.execute(stmt)
row = result.fetchone()
if row:
return RedirectResponse(url=f"/page/{row[0]}", status_code=302)
else:
return RedirectResponse(url="/", status_code=302)
# Get a random media item (excluding screenshots)
stmt = (
select(Media.md5_hash)
.where(Media.media_type != 'screenshot')
.order_by(func.random())
.limit(1)
)
result = await session.execute(stmt)
row = result.fetchone()
if row:
return RedirectResponse(url=f"/view/{row[0]}", status_code=302)
return RedirectResponse(url="/", status_code=302)
@app.get("/api/search")