modified: .gitignore

modified:   database.py
	modified:   html2md.py
	modified:   neopig.py
	modified:   screenshot.py
	modified:   serp.py

.gitignore

@@ -8,7 +8,6 @@ venv/
 
 # Database and vault (generated data)
 *.db
-vault/
 test_vault/
 
 # Vendored dependencies

database.py

@@ -669,6 +669,21 @@ class Database:
             row = result.fetchone()
             return row[0] if row else None
 
+    async def get_page_screenshots(self, page_uri: str) -> List[str]:
+        """Get all screenshot hashes for a page (for chunked screenshots). Returns list of md5_hash ordered by file size desc."""
+        async with self.session() as session:
+            stmt = (
+                select(Media.md5_hash)
+                .join(MediaSource, Media.md5_hash == MediaSource.md5_hash)
+                .where(and_(
+                    MediaSource.page_uri == page_uri,
+                    Media.media_type == 'screenshot'
+                ))
+                .order_by(Media.file_size.desc())
+            )
+            result = await session.execute(stmt)
+            return [row[0] for row in result.fetchall()]
+
     async def get_recent_media(self, limit: int = 50, media_type: str = None) -> List[Dict[str, Any]]:
         """Get recently discovered media."""
         async with self.session() as session:

html2md.py

@@ -297,10 +297,21 @@ class SmartMarkdownConverter:
 
         # Block elements
         if tag in ('p', 'div'):
-            content = self._inline_content(el)
+            # Check if contains block-level elements (pre, ul, ol, etc.)
-            if content:
+            block_tags = {'pre', 'ul', 'ol', 'blockquote', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'table'}
-                lines.append(content)
+            has_blocks = any(
-                lines.append('')
+                isinstance(c, Tag) and c.name and c.name.lower() in block_tags
+                for c in el.children
+            )
+            if has_blocks:
+                # Recursively process children to preserve block structure
+                for child in el.children:
+                    self._walk_element(child, lines, depth)
+            else:
+                content = self._inline_content(el)
+                if content:
+                    lines.append(content)
+                    lines.append('')
 
         elif tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
             level = int(tag[1])

neopig.py

@@ -915,13 +915,14 @@ def trim_html_wrapper(html: str) -> str:
     return str(soup)
 
 
-async def backfill_markdown(db_path: str, domain_filter: str = None, trim_wrapper: bool = False):
+async def backfill_markdown(db_path: str, domain_filter: str = None, trim_wrapper: bool = False, quiet: bool = False):
     """Re-process stored HTML to regenerate markdown with smart structure detection.
 
     Args:
         db_path: Path to SQLite database
         domain_filter: Only process pages matching this domain (e.g., 'example.com')
         trim_wrapper: Strip nav/header/footer/logo before conversion (applied before smart conversion)
+        quiet: Disable progress bar
     """
     from html2md import html_to_markdown
     import aiosqlite
@@ -933,28 +934,25 @@ async def backfill_markdown(db_path: str, domain_filter: str = None, trim_wrappe
         if domain_filter:
             # Match domain in URI
             pattern = f"%://{domain_filter}%" if '://' not in domain_filter else f"%{domain_filter}%"
-            count_sql = "SELECT COUNT(*) FROM pages WHERE raw_html IS NOT NULL AND uri LIKE ?"
             select_sql = "SELECT id, uri, raw_html FROM pages WHERE raw_html IS NOT NULL AND uri LIKE ?"
             params = (pattern,)
             logger.info(f"Backfilling markdown for domain: {domain_filter}")
         else:
-            count_sql = "SELECT COUNT(*) FROM pages WHERE raw_html IS NOT NULL"
             select_sql = "SELECT id, uri, raw_html FROM pages WHERE raw_html IS NOT NULL"
             params = ()
-            logger.info("Backfilling markdown for ALL pages (no domain filter)")
+            logger.info("Backfilling markdown for ALL pages")
 
         if trim_wrapper:
-            logger.info("Trim wrapper enabled: stripping nav/header/footer/logo before conversion")
+            logger.info("Trim wrapper enabled")
-
-        cursor = await db.execute(count_sql, params)
-        total = (await cursor.fetchone())[0]
-        logger.info(f"Found {total} pages to process...")
 
         cursor = await db.execute(select_sql, params)
         rows = await cursor.fetchall()
+        total = len(rows)
+        logger.info(f"Found {total} pages to process")
 
         updated = 0
-        for row in rows:
+        errors = 0
+        for row in tqdm(rows, desc="Markdown", unit="pages", disable=quiet):
             if not row['raw_html']:
                 continue
 
@@ -967,12 +965,12 @@ async def backfill_markdown(db_path: str, domain_filter: str = None, trim_wrappe
                 updated += 1
                 if updated % 100 == 0:
                     await db.commit()
-                    logger.info(f"  Processed {updated}/{total} pages...")
             except Exception as e:
-                logger.warning(f"Error processing {row['uri']}: {e}")
+                errors += 1
+                logger.debug(f"Error processing {row['uri']}: {e}")
 
         await db.commit()
-        logger.info(f"Backfill complete: {updated} pages updated")
+        logger.info(f"Backfill complete: {updated} updated, {errors} errors")
 
 
 async def backfill_screenshots(
@@ -981,6 +979,7 @@ async def backfill_screenshots(
     domain_filter: str = None,
     delete_old: bool = True,
     fast_mode: bool = False,
+    quiet: bool = False,
 ):
     """Re-capture screenshots as JPEG to replace old PNGs.
 
@@ -990,6 +989,7 @@ async def backfill_screenshots(
         domain_filter: Only process pages matching this domain
         delete_old: Delete old PNG files after successful JPEG capture
         fast_mode: Skip delay between captures (for sites without robots.txt)
+        quiet: Disable progress bar
     """
     from screenshot import ScreenshotCapture, ScreenshotConfig
     from storage import ImageVault
@@ -1054,8 +1054,22 @@ async def backfill_screenshots(
         domain_last_fetched = {}  # Track last fetch time per domain
         crawl_delay = 2.0  # Default crawl delay in seconds
 
-        pbar = tqdm(total=total, initial=skipped_count, desc="Screenshots", unit="pages")
+        # Concurrent workers in fast mode (based on CPU count)
-        for row in rows:
+        import multiprocessing
+        max_workers = multiprocessing.cpu_count() if fast_mode else 1
+        semaphore = asyncio.Semaphore(max_workers)
+        if fast_mode and max_workers > 1:
+            logger.info(f"Fast mode: {max_workers} concurrent workers")
+
+        pbar = tqdm(total=total, initial=skipped_count, desc="Screenshots", unit="pages", disable=quiet)
+
+        # Lock for thread-safe updates
+        state_lock = asyncio.Lock()
+        stats = {'captured': 0, 'failed': 0, 'bytes_saved': 0}
+
+        async def process_row(row):
+            nonlocal completed_uris
+            async with semaphore:
             page_uri = row.page_uri
             old_hash = row.md5_hash
             domain = Uri(page_uri).hostname.lower()
@@ -1337,6 +1351,12 @@ async def main():
         help="With --backfill-markdown: strip nav/header/footer/logo before conversion"
     )
 
+    parser.add_argument(
+        "-q", "--quiet",
+        action="store_true",
+        help="Disable progress bars"
+    )
+
     parser.add_argument(
         "--backfill-screenshots",
         metavar="DOMAIN",
@@ -1386,7 +1406,7 @@ async def main():
 
     # Handle --backfill-markdown
     if args.backfill_markdown:
-        await backfill_markdown(args.db, domain_filter=args.backfill_markdown, trim_wrapper=args.trim_wrapper)
+        await backfill_markdown(args.db, domain_filter=args.backfill_markdown, trim_wrapper=args.trim_wrapper, quiet=args.quiet)
         return
 
     # Handle --backfill-screenshots
@@ -1416,6 +1436,7 @@ async def main():
                 domain_filter=args.backfill_screenshots,
                 delete_old=not args.keep_old_screenshots,
                 fast_mode=args.fast,
+                quiet=args.quiet,
             )
         finally:
             if serp_process:

screenshot.py

@@ -194,13 +194,13 @@ class ScreenshotCapture:
             content_length: Length of raw HTML content in bytes
 
         Returns:
-            Delay in milliseconds (min 2000ms, max 30000ms)
+            Delay in milliseconds (min 3000ms, max 60000ms)
         """
-        # Base delay of 2 seconds
+        # Base delay of 3 seconds
-        base_delay = 2000
+        base_delay = 3000
 
-        # Add 2 seconds per 50KB of content, up to a max of 28 seconds
+        # Add 3 seconds per 50KB of content, up to a max of 57 seconds
-        additional_delay = min((content_length // 50000) * 2000, 28000)
+        additional_delay = min((content_length // 50000) * 3000, 57000)
 
         return base_delay + additional_delay
 

serp.py

@@ -1912,13 +1912,12 @@ async def view_page(
         escaped = html_module.escape(page["content"][:50000])
         content_html = f"<pre style='white-space:pre-wrap;'>{escaped}</pre>"
 
-    # Find screenshot
+    # Find all screenshots (may be chunked for tall pages)
+    screenshot_hashes = await db.get_page_screenshots(uri)
     screenshot_html = ""
-    screenshot_hash = await db.get_page_screenshot(uri)
+    if screenshot_hashes:
-    if screenshot_hash:
+        imgs = ''.join(f'<img src="/media/{h}" alt="Page screenshot chunk" style="width:100%;display:block;">' for h in screenshot_hashes)
-        screenshot_html = f'''<a href="/media/{screenshot_hash}" target="_blank">
+        screenshot_html = f'<div class="screenshot-stack">{imgs}</div>'
-                    <img src="/media/{screenshot_hash}" alt="Page screenshot" style="max-width:100%;max-height:50vh;border-radius:8px;">
-                </a>'''
 
     # Find media from this page
     media_items = await db.get_page_media(uri)
@@ -1942,13 +1941,36 @@ async def view_page(
             <div class="media-grid">{''.join(media_cards)}</div>
         </div>'''
 
+    has_screenshot = bool(screenshot_html)
     return f"""<!DOCTYPE html>
 <html>
 <head>
     <title>{html_module.escape(page_title)} - neopig</title>
     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github-dark.min.css">
     <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
-    <style>{VIEW_CSS}</style>
+    <style>
+        {VIEW_CSS}
+        .page-layout {{ display: flex; gap: 20px; }}
+        .page-main {{ flex: 1; min-width: 0; }}
+        .page-sidebar {{ width: 400px; flex-shrink: 0; transition: width 0.3s, opacity 0.3s; }}
+        .page-sidebar.collapsed {{ width: 40px; overflow: hidden; }}
+        .page-sidebar.collapsed .screenshot-stack {{ opacity: 0; }}
+        .sidebar-toggle {{
+            cursor: pointer; padding: 8px 12px; background: #333; border-radius: 4px;
+            margin-bottom: 10px; display: inline-block; user-select: none;
+        }}
+        .sidebar-toggle:hover {{ background: #444; }}
+        .screenshot-stack {{ max-height: 80vh; overflow-y: auto; border-radius: 8px; }}
+        .screenshot-stack img {{ border-radius: 0; }}
+        .screenshot-stack img:first-child {{ border-radius: 8px 8px 0 0; }}
+        .screenshot-stack img:last-child {{ border-radius: 0 0 8px 8px; }}
+        .screenshot-stack img:only-child {{ border-radius: 8px; }}
+        @media (max-width: 900px) {{
+            .page-layout {{ flex-direction: column; }}
+            .page-sidebar {{ width: 100%; }}
+            .page-sidebar.collapsed {{ width: 100%; height: 40px; }}
+        }}
+    </style>
 </head>
 <body>
     <div class="nav">
@@ -1958,19 +1980,42 @@ async def view_page(
         <a href="/random">Random</a>
     </div>
     <div class="container">
-        <div class="hero">
-            {screenshot_html or ''}
-        </div>
         <h1>{html_module.escape(page_title)}</h1>
         <div class="meta">
             <div class="meta-row"><span class="meta-label">URL:</span><span class="meta-value"><a href="{uri}" target="_blank">{uri}</a></span></div>
             <div class="meta-row"><span class="meta-label">Media:</span><span class="meta-value">{len(media_items)} items</span></div>
         </div>
-        {media_grid}
+        <div class="page-layout">
-        <h3>Page Content</h3>
+            <div class="page-main">
-        <div class="content-rendered">{content_html or '<p>No content available</p>'}</div>
+                {media_grid}
+                <h3>Page Content</h3>
+                <div class="content-rendered">{content_html or '<p>No content available</p>'}</div>
+            </div>
+            {f'''<div class="page-sidebar" id="sidebar">
+                <span class="sidebar-toggle" onclick="toggleSidebar()">◀ Screenshot</span>
+                {screenshot_html}
+            </div>''' if has_screenshot else ''}
+        </div>
     </div>
-    <script>hljs.highlightAll();</script>
+    <script>
+        hljs.highlightAll();
+        function toggleSidebar() {{
+            const sb = document.getElementById('sidebar');
+            const toggle = sb.querySelector('.sidebar-toggle');
+            sb.classList.toggle('collapsed');
+            const isCollapsed = sb.classList.contains('collapsed');
+            toggle.textContent = isCollapsed ? '▶ Screenshot' : '◀ Screenshot';
+            localStorage.setItem('neopig_sidebar_collapsed', isCollapsed);
+        }}
+        // Restore state from localStorage
+        if (localStorage.getItem('neopig_sidebar_collapsed') === 'true') {{
+            const sb = document.getElementById('sidebar');
+            if (sb) {{
+                sb.classList.add('collapsed');
+                sb.querySelector('.sidebar-toggle').textContent = '▶ Screenshot';
+            }}
+        }}
+    </script>
     {'' if noai else f'''<script>
     window.UNCLOSEAI_SYSTEM_PROMPT = "This is an archived copy of {source_domain} preserved by neopig. The original site may no longer exist. You are viewing: {page_title} ({uri}). Help users explore the preserved discussions, media, and pages.";
     </script>
@@ -2269,27 +2314,38 @@ async def get_stats_endpoint():
 
 
 @app.get("/random")
-async def random_media():
+async def random_item(type: str = Query(None, description="Type: media or page (random if not specified)")):
-    """Redirect to a random media item."""
+    """Redirect to a random media item or page."""
     from sqlalchemy import select, func
-    from database import Media
+    from database import Media, Page
     import random
 
-    async with db.session() as session:
+    # If no type specified, randomly pick between media and page
-        # Get a random media item (excluding screenshots)
+    if type is None:
-        stmt = (
+        type = random.choice(["media", "page"])
-            select(Media.md5_hash)
-            .where(Media.media_type != 'screenshot')
-            .order_by(func.random())
-            .limit(1)
-        )
-        result = await session.execute(stmt)
-        row = result.fetchone()
 
-        if row:
+    async with db.session() as session:
-            return RedirectResponse(url=f"/view/{row[0]}", status_code=302)
+        if type == "page":
+            # Get a random page
+            stmt = select(Page.uri).order_by(func.random()).limit(1)
+            result = await session.execute(stmt)
+            row = result.fetchone()
+            if row:
+                return RedirectResponse(url=f"/page/{row[0]}", status_code=302)
         else:
-            return RedirectResponse(url="/", status_code=302)
+            # Get a random media item (excluding screenshots)
+            stmt = (
+                select(Media.md5_hash)
+                .where(Media.media_type != 'screenshot')
+                .order_by(func.random())
+                .limit(1)
+            )
+            result = await session.execute(stmt)
+            row = result.fetchone()
+            if row:
+                return RedirectResponse(url=f"/view/{row[0]}", status_code=302)
+
+        return RedirectResponse(url="/", status_code=302)
 
 
 @app.get("/api/search")