From 222c23bb14816fd760038f9bcbb378f2c3efcb2b Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Mon, 29 Dec 2025 12:13:33 -0500 Subject: [PATCH] modified: CLAUDE.md modified: archive.py modified: database.py modified: neopig.py modified: screenshot.py modified: serp.py --- CLAUDE.md | 4 + archive.py | 381 ++++++++++++++++++++++++++++++++++++++++++++++++-- database.py | 106 ++++++++++++++ neopig.py | 58 ++++++-- screenshot.py | 4 +- serp.py | 216 +++++++++++++++++++++++----- 6 files changed, 710 insertions(+), 59 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1b28ecf..80b9d55 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,6 +4,10 @@ **NEVER attribute Claude in commit messages.** No co-author tags, no "Generated with Claude" footers, no AI mentions. Keep commits clean and human-authored in appearance. +## Naming Conventions + +**Always use `uri`, never `url`.** This applies to variable names, function names, column names, and comments. URI is the correct term (Uniform Resource Identifier). + ## Quick Start ```bash diff --git a/archive.py b/archive.py index e734580..4bb5764 100644 --- a/archive.py +++ b/archive.py @@ -708,7 +708,7 @@ async function search() {{ }} container.innerHTML = data.map(r => `
-

${{r.title}}

+

${{r.title}}

${{r.snippet}}

${{r.path}}

@@ -779,27 +779,31 @@ MEDIA_HTML = """ margin: 0; padding: 20px; background: #0a0a0a; color: #e0e0e0; }} - .container {{ max-width: 1400px; margin: 0 auto; }} + .container {{ max-width: 1600px; margin: 0 auto; }} h1 {{ color: #ff6b6b; }} nav {{ margin-bottom: 20px; }} nav a {{ color: #ff6b6b; margin-right: 15px; text-decoration: none; }} nav a:hover {{ text-decoration: underline; }} .grid {{ display: grid; - grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); - gap: 15px; + grid-template-columns: repeat(auto-fill, minmax(220px, 1fr)); + grid-auto-rows: min-content; + gap: 12px; }} .card {{ background: #1a1a1a; border-radius: 8px; overflow: hidden; transition: transform 0.2s; + break-inside: avoid; }} .card:hover {{ transform: scale(1.02); }} .card a {{ display: block; }} .card img, .card video {{ - width: 100%; height: 150px; - object-fit: contain; background: #222; + width: 100%; + height: auto; + display: block; + background: #222; }} - .card-info {{ padding: 10px; }} + .card-info {{ padding: 8px 10px; }} .card-title {{ font-size: 11px; color: #888; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; @@ -823,6 +827,249 @@ MEDIA_HTML = """ """ +MEDIA_DETAIL_HTML = """ + + + + + {caption} - neopig Archive + + + +
+ + +
+ {media_element} +
{caption}
+
+
Image: {media_uri}
+
Source: {page_uri}
+
+
+ +
+ +
Source Page Content
+
+ {page_content} +
+
+ +""" + + +PAGE_VIEW_HTML = """ + + + + + {title} - neopig Archive + + + +
+ +

{title}

+
+ [View Original HTML] + {screenshot_link} +
+
+ {content} +
+ {screenshot_embed} +
+ +""" + + +def simple_markdown_to_html(md: str) -> str: + """Convert markdown to HTML (simple stdlib-only implementation).""" + import re + lines = md.split('\n') + html_lines = [] + in_code_block = False + in_list = False + + for line in lines: + # Code blocks + if line.startswith('```'): + if in_code_block: + html_lines.append('') + in_code_block = False + else: + html_lines.append('
')
+                in_code_block = True
+            continue
+
+        if in_code_block:
+            html_lines.append(html.escape(line))
+            continue
+
+        # Close list if needed
+        if in_list and not line.strip().startswith(('- ', '* ', '1. ')):
+            html_lines.append('')
+            in_list = False
+
+        # Headers
+        if line.startswith('### '):
+            html_lines.append(f'
{html.escape(line[4:])}
')
+        elif line.startswith('## '):
+            html_lines.append(f'
{html.escape(line[3:])}
')
+        elif line.startswith('# '):
+            html_lines.append(f'
{html.escape(line[2:])}
')
+        # Blockquotes
+        elif line.startswith('> '):
+            html_lines.append(f'
{html.escape(line[2:])}
')
+        # Horizontal rule
+        elif line.strip() in ('---', '***', '___'):
+            html_lines.append('
')
+        # Lists
+        elif line.strip().startswith(('- ', '* ')):
+            if not in_list:
+                html_lines.append('
    ')
+                in_list = True
+            content = line.strip()[2:]
+            html_lines.append(f'
  • {html.escape(content)}
    • ')
+        # Empty line
+        elif not line.strip():
+            html_lines.append('
    ')
+        # Regular paragraph
+        else:
+            escaped = html.escape(line)
+            # Inline code
+            escaped = re.sub(r'`([^`]+)`', r'\1', escaped)
+            # Bold
+            escaped = re.sub(r'[*][*]([^*]+)[*][*]', r'\1', escaped)
+            # Italic
+            escaped = re.sub(r'[*]([^*]+)[*]', r'\1', escaped)
+            # Links [text](url)
+            link_re = re.compile(r'\[([^]]+)\]\(([^)]+)\)')
+            escaped = link_re.sub(r'\1', escaped)
+            # Images ![alt](url)
+            img_re = re.compile(r'!\[([^]]*)\]\(([^)]+)\)')
+            escaped = img_re.sub(r'\1', escaped)
+            html_lines.append(f'
    {escaped}
    ')
+
+    if in_list:
+        html_lines.append('
')
+    if in_code_block:
+        html_lines.append('
') + + return '\n'.join(html_lines) + class ArchiveHandler(BaseHTTPRequestHandler): """HTTP request handler for the archive.""" @@ -882,7 +1129,7 @@ class ArchiveHandler(BaseHTTPRequestHandler): if path == '/browse': pages = list_files('html', '*.html') items = ''.join([ - f'
{html.escape(p)}
' + f'
{html.escape(p)}
' for p in pages[:500] ]) if not items: @@ -891,6 +1138,55 @@ class ArchiveHandler(BaseHTTPRequestHandler): self.send_html(content) return + # Page view - render markdown with neopig styling + if path.startswith('/view/'): + page_path = path[6:] # Remove '/view/' + # Try markdown first, fall back to HTML + md_path = 'markdown/' + page_path.replace('.html', '.md') + html_path = 'html/' + page_path + + md_content, _ = read_file(md_path) + html_content_raw, _ = read_file(html_path) + + if md_content: + # Render markdown + rendered = simple_markdown_to_html(md_content.decode('utf-8', errors='replace')) + elif html_content_raw: + # Extract body from HTML and show as-is + html_str = html_content_raw.decode('utf-8', errors='replace') + # Simple body extraction + import re as re_mod + body_match = re_mod.search(r']*>(.*?)', html_str, re_mod.DOTALL | re_mod.IGNORECASE) + rendered = body_match.group(1) if body_match else html_str + else: + self.send_404() + return + + # Extract title + title = page_path.replace('.html', '').replace('/', ' > ') + + # Check for screenshot + ss_path = page_path.replace('.html', '.png').replace('/', '_') + screenshot_exists = f'screenshots/{ss_path}' in [f'screenshots/{f}' for f in list_files('screenshots', '*.png')] + + screenshot_link = f' | [View Screenshot]' if screenshot_exists else '' + screenshot_embed = ( + '
' + '
Page Screenshot:
' + f'Screenshot' + '
' + ) if screenshot_exists else '' + + content = PAGE_VIEW_HTML.format( + title=html.escape(title), + path=html.escape(page_path), + content=rendered, + screenshot_link=screenshot_link, + screenshot_embed=screenshot_embed, + ) + self.send_html(content) + return + if path == '/media': files = list_files('media') items = [] @@ -900,9 +1196,10 @@ class ArchiveHandler(BaseHTTPRequestHandler): media_el = f'' else: media_el = f'' + # Link to detail view instead of raw file items.append( '
' - f'{media_el}' + f'{media_el}' f'
{html.escape(f)}
' '
' ) @@ -914,6 +1211,72 @@ class ArchiveHandler(BaseHTTPRequestHandler): self.send_html(content) return + # Media detail view - image at top with source page content below + if path.startswith('/media/detail/'): + media_path = path[14:] # Remove '/media/detail/' + media_file = 'media/' + media_path + + # Check media exists + media_content, mime = read_file(media_file) + if not media_content: + self.send_404() + return + + # Determine media element type + ext = Path(media_path).suffix.lower() + if ext in ('.mp4', '.webm', '.mov'): + media_el = f'' + else: + media_el = f'' + + # Try to find the source page - media path mirrors URL structure + # e.g., media/images/foo.jpg might come from t/topic-name/123.html + # For now, use filename as caption + caption = Path(media_path).stem.replace('-', ' ').replace('_', ' ') + media_uri = f'/media/{media_path}' + page_uri = METADATA.get('target_url', METADATA.get('domain', 'unknown')) + page_path = 'index.html' + + # Try to find associated page content from the database + page_content = '

Source page content not available in archive database.

' + if DB_PATH and DB_PATH.exists(): + conn = sqlite3.connect(DB_PATH) + c = conn.cursor() + try: + # Search for pages that might contain this media + media_name = Path(media_path).name + c.execute(""" + SELECT path, title, content FROM pages + WHERE content LIKE ? OR path LIKE ? + LIMIT 1 + """, (f'%{media_name}%', f'%{media_name}%')) + row = c.fetchone() + if row: + page_path = row[0].replace('html/', '') + caption = row[1] or caption + # Render the content + md_file = 'markdown/' + page_path.replace('.html', '.md') + md_content, _ = read_file(md_file) + if md_content: + page_content = simple_markdown_to_html(md_content.decode('utf-8', errors='replace')) + else: + page_content = f'

{html.escape(row[2][:2000] if row[2] else "")}...

' + except Exception: + pass + finally: + conn.close() + + content = MEDIA_DETAIL_HTML.format( + media_element=media_el, + caption=html.escape(caption), + media_uri=html.escape(media_uri), + page_uri=html.escape(page_uri), + page_path=html.escape(page_path), + page_content=page_content, + ) + self.send_html(content) + return + if path == '/screenshots': files = list_files('screenshots', '*.png') items = [] diff --git a/database.py b/database.py index c912391..6854b42 100644 --- a/database.py +++ b/database.py @@ -88,6 +88,53 @@ class Database: ) """) + # Pages table for full-text search + await db.execute(""" + CREATE TABLE IF NOT EXISTS pages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + uri TEXT NOT NULL UNIQUE, + path TEXT, + title TEXT, + content TEXT, + crawl_job_id INTEGER, + crawled_at TEXT NOT NULL, + FOREIGN KEY (crawl_job_id) REFERENCES crawl_jobs(id) + ) + """) + + # FTS5 virtual table for page search + await db.execute(""" + CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5( + title, content, uri, path, + content='pages', + content_rowid='id' + ) + """) + + # Triggers to keep FTS in sync + await db.execute(""" + CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN + INSERT INTO pages_fts(rowid, title, content, uri, path) + VALUES (new.id, new.title, new.content, new.uri, new.path); + END + """) + + await db.execute(""" + CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN + INSERT INTO pages_fts(pages_fts, rowid, title, content, uri, path) + VALUES ('delete', old.id, old.title, old.content, old.uri, old.path); + END + """) + + await db.execute(""" + CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN + INSERT INTO pages_fts(pages_fts, rowid, title, content, uri, path) + VALUES ('delete', old.id, old.title, old.content, old.uri, old.path); + INSERT INTO pages_fts(rowid, title, content, uri, path) + VALUES (new.id, new.title, new.content, new.uri, new.path); + END + """) + # Indexes await db.execute("CREATE INDEX IF NOT EXISTS idx_media_type ON media(media_type)") await db.execute("CREATE INDEX IF NOT EXISTS idx_media_analysis ON media(analysis_status)") @@ -95,6 +142,7 @@ class Database: await db.execute("CREATE INDEX IF NOT EXISTS idx_sources_job ON media_sources(crawl_job_id)") await db.execute("CREATE INDEX IF NOT EXISTS idx_sources_media_uri ON media_sources(media_uri)") await db.execute("CREATE INDEX IF NOT EXISTS idx_sources_page_uri ON media_sources(page_uri)") + await db.execute("CREATE INDEX IF NOT EXISTS idx_pages_uri ON pages(uri)") await db.commit() @@ -309,6 +357,64 @@ class Database: rows = await cursor.fetchall() return {row[0] for row in rows} + async def store_page( + self, + uri: str, + title: str, + content: str, + path: str = "", + crawl_job_id: int = None + ) -> None: + """Store a page for full-text search.""" + async with aiosqlite.connect(self.db_path) as db: + await db.execute( + """ + INSERT OR REPLACE INTO pages (uri, path, title, content, crawl_job_id, crawled_at) + VALUES (?, ?, ?, ?, ?, ?) + """, + (uri, path, title, content[:100000], crawl_job_id, datetime.now(timezone.utc).isoformat()) + ) + await db.commit() + + async def search_pages(self, query: str, limit: int = 50) -> List[Dict[str, Any]]: + """Search pages using FTS5 with LIKE fallback.""" + async with aiosqlite.connect(self.db_path) as db: + db.row_factory = aiosqlite.Row + results = [] + + # Try FTS5 with prefix matching + try: + fts_query = ' '.join(f'"{word}"*' for word in query.split()) + cursor = await db.execute(""" + SELECT p.uri, p.path, p.title, + snippet(pages_fts, 1, '', '', '...', 40) as snippet + FROM pages_fts + JOIN pages p ON pages_fts.rowid = p.id + WHERE pages_fts MATCH ? + ORDER BY rank + LIMIT ? + """, (fts_query, limit)) + results = [dict(row) for row in await cursor.fetchall()] + except Exception: + pass + + # Fallback to LIKE + if not results: + try: + like_q = f'%{query}%' + cursor = await db.execute(""" + SELECT uri, path, title, substr(content, 1, 200) as snippet + FROM pages + WHERE title LIKE ? COLLATE NOCASE + OR content LIKE ? COLLATE NOCASE + LIMIT ? + """, (like_q, like_q, limit)) + results = [dict(row) for row in await cursor.fetchall()] + except Exception: + pass + + return results + async def get_stats(self) -> Dict[str, Any]: """Get database statistics.""" async with aiosqlite.connect(self.db_path) as db: diff --git a/neopig.py b/neopig.py index d7acb8e..2e38b95 100644 --- a/neopig.py +++ b/neopig.py @@ -212,21 +212,60 @@ class NeoPig: self._domain_stats[domain] = {'pages_changed': 0, 'media_new': 0, 'screenshots_new': 0} self._domain_stats[domain][stat] += increment + def _extract_text_from_html(self, html: str) -> tuple: + """Extract title and text content from HTML for search indexing.""" + try: + from bs4 import BeautifulSoup + soup = BeautifulSoup(html, 'html.parser') + + # Extract title + title = '' + title_tag = soup.find('title') + if title_tag: + title = title_tag.get_text(strip=True) + + # Remove script and style elements + for tag in soup(['script', 'style', 'nav', 'header', 'footer']): + tag.decompose() + + # Get text content + text = soup.get_text(separator=' ', strip=True) + # Clean up whitespace + import re + text = re.sub(r'\s+', ' ', text) + + return title, text[:100000] + except Exception: + return '', '' + async def _archive_page_to_vault( self, - url: str, + uri: str, html: str, media_mappings: Dict[str, str] = None, + crawl_job_id: int = None, ): - """Archive a page to the HTML vault.""" - domain = self._get_domain(url) + """Archive a page to the HTML vault and store for search.""" + domain = self._get_domain(uri) html_vault = self.domain_vaults.get_html_vault(domain) - is_changed, _ = await html_vault.archive_page(url, html, media_mappings) + is_changed, _ = await html_vault.archive_page(uri, html, media_mappings) self.stats['bytes_downloaded'] += len(html.encode('utf-8')) if is_changed: self._track_domain_stat(domain, 'pages_changed') self.stats['pages_changed'] += 1 + # Store page content for full-text search + title, content = self._extract_text_from_html(html) + parsed = urlparse(uri) + path = parsed.path or '/' + await self.db.store_page( + uri=uri, + title=title, + content=content, + path=path, + crawl_job_id=crawl_job_id, + ) + async def _archive_media_to_vault( self, url: str, @@ -424,17 +463,16 @@ class NeoPig: # Page callback - archive raw HTML to vault and capture screenshot # Screenshot happens here (same crawl delay window as page fetch) - async def on_page_fetched(url: str, html: str): + async def on_page_fetched(uri: str, html: str): # Track this page as crawled for resume support - self.seen_pages.add(url) + self.seen_pages.add(uri) - # For now, archive without media URL rewriting (we'd need to download media first) - # TODO: Build media_mappings after media is downloaded - await self._archive_page_to_vault(url, html, media_mappings=None) + # Archive to vault and store for search (with job_id for tracking) + await self._archive_page_to_vault(uri, html, media_mappings=None, crawl_job_id=job_id) # Capture screenshot for every page (honors crawl delay as a UNIT with page fetch) if self.screenshot_config.enabled: - await self._capture_page_screenshot(url, job_id, page_title='') + await self._capture_page_screenshot(uri, job_id, page_title='') # Save state periodically for resume support self._save_state(target_uri) diff --git a/screenshot.py b/screenshot.py index 540d133..16be92c 100644 --- a/screenshot.py +++ b/screenshot.py @@ -30,8 +30,8 @@ logger = logging.getLogger(__name__) # Engine preference order - lightest/fastest first ENGINE_PREFERENCE = [ - 'wkhtmltoimage', # Native Qt WebKit - very fast, no browser download - 'cutycapt', # Native Qt WebKit - very fast, no browser download + 'cutycapt', # Native Qt WebKit - fast, supports full-page screenshots + 'wkhtmltoimage', # Native Qt WebKit - fast, viewport-only 'playwright-webkit', # WebKit via Playwright - lighter than Chromium 'playwright-firefox', 'playwright-chromium', diff --git a/serp.py b/serp.py index 9da5a2f..fba32ef 100644 --- a/serp.py +++ b/serp.py @@ -210,6 +210,13 @@ SEARCH_HTML = """ font-size: 14px; color: #888; } + .section-title { + color: #ff6b6b; + font-size: 18px; + margin: 25px 0 15px 0; + border-bottom: 1px solid #333; + padding-bottom: 8px; + } .results { display: grid; grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); @@ -276,14 +283,56 @@ SEARCH_HTML = """ .result-hash:hover { text-decoration: underline; } + /* Page results */ + .page-results { + display: flex; + flex-direction: column; + gap: 10px; + } + .page-result { + background: #1a1a1a; + border-radius: 8px; + padding: 15px; + transition: background 0.2s; + } + .page-result:hover { + background: #252525; + } + .page-result a { + color: #ff6b6b; + text-decoration: none; + font-size: 16px; + font-weight: 500; + } + .page-result a:hover { + text-decoration: underline; + } + .page-path { + font-size: 12px; + color: #4ade80; + margin-top: 4px; + font-family: monospace; + } + .page-snippet { + font-size: 13px; + color: #999; + margin-top: 8px; + line-height: 1.5; + } + .page-snippet mark { + background: #ff6b6b33; + color: #ff9999; + padding: 1px 3px; + border-radius: 2px; + } -

🐷 neopig

-

Search hydrated media

+

neopig

+

Search hydrated media and pages

- +