diff --git a/.gitignore b/.gitignore index af92ec7..5cf8252 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ venv/ # Database and vault (generated data) *.db -vault/ test_vault/ # Vendored dependencies diff --git a/database.py b/database.py index 56ca28a..86743f2 100644 --- a/database.py +++ b/database.py @@ -669,6 +669,21 @@ class Database: row = result.fetchone() return row[0] if row else None + async def get_page_screenshots(self, page_uri: str) -> List[str]: + """Get all screenshot hashes for a page (for chunked screenshots). Returns list of md5_hash ordered by file size desc.""" + async with self.session() as session: + stmt = ( + select(Media.md5_hash) + .join(MediaSource, Media.md5_hash == MediaSource.md5_hash) + .where(and_( + MediaSource.page_uri == page_uri, + Media.media_type == 'screenshot' + )) + .order_by(Media.file_size.desc()) + ) + result = await session.execute(stmt) + return [row[0] for row in result.fetchall()] + async def get_recent_media(self, limit: int = 50, media_type: str = None) -> List[Dict[str, Any]]: """Get recently discovered media.""" async with self.session() as session: diff --git a/html2md.py b/html2md.py index 816ae0f..d5734b9 100644 --- a/html2md.py +++ b/html2md.py @@ -297,10 +297,21 @@ class SmartMarkdownConverter: # Block elements if tag in ('p', 'div'): - content = self._inline_content(el) - if content: - lines.append(content) - lines.append('') + # Check if contains block-level elements (pre, ul, ol, etc.) + block_tags = {'pre', 'ul', 'ol', 'blockquote', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'table'} + has_blocks = any( + isinstance(c, Tag) and c.name and c.name.lower() in block_tags + for c in el.children + ) + if has_blocks: + # Recursively process children to preserve block structure + for child in el.children: + self._walk_element(child, lines, depth) + else: + content = self._inline_content(el) + if content: + lines.append(content) + lines.append('') elif tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'): level = int(tag[1]) diff --git a/neopig.py b/neopig.py index a91a744..c7af4a8 100644 --- a/neopig.py +++ b/neopig.py @@ -915,13 +915,14 @@ def trim_html_wrapper(html: str) -> str: return str(soup) -async def backfill_markdown(db_path: str, domain_filter: str = None, trim_wrapper: bool = False): +async def backfill_markdown(db_path: str, domain_filter: str = None, trim_wrapper: bool = False, quiet: bool = False): """Re-process stored HTML to regenerate markdown with smart structure detection. Args: db_path: Path to SQLite database domain_filter: Only process pages matching this domain (e.g., 'example.com') trim_wrapper: Strip nav/header/footer/logo before conversion (applied before smart conversion) + quiet: Disable progress bar """ from html2md import html_to_markdown import aiosqlite @@ -933,28 +934,25 @@ async def backfill_markdown(db_path: str, domain_filter: str = None, trim_wrappe if domain_filter: # Match domain in URI pattern = f"%://{domain_filter}%" if '://' not in domain_filter else f"%{domain_filter}%" - count_sql = "SELECT COUNT(*) FROM pages WHERE raw_html IS NOT NULL AND uri LIKE ?" select_sql = "SELECT id, uri, raw_html FROM pages WHERE raw_html IS NOT NULL AND uri LIKE ?" params = (pattern,) logger.info(f"Backfilling markdown for domain: {domain_filter}") else: - count_sql = "SELECT COUNT(*) FROM pages WHERE raw_html IS NOT NULL" select_sql = "SELECT id, uri, raw_html FROM pages WHERE raw_html IS NOT NULL" params = () - logger.info("Backfilling markdown for ALL pages (no domain filter)") + logger.info("Backfilling markdown for ALL pages") if trim_wrapper: - logger.info("Trim wrapper enabled: stripping nav/header/footer/logo before conversion") - - cursor = await db.execute(count_sql, params) - total = (await cursor.fetchone())[0] - logger.info(f"Found {total} pages to process...") + logger.info("Trim wrapper enabled") cursor = await db.execute(select_sql, params) rows = await cursor.fetchall() + total = len(rows) + logger.info(f"Found {total} pages to process") updated = 0 - for row in rows: + errors = 0 + for row in tqdm(rows, desc="Markdown", unit="pages", disable=quiet): if not row['raw_html']: continue @@ -967,12 +965,12 @@ async def backfill_markdown(db_path: str, domain_filter: str = None, trim_wrappe updated += 1 if updated % 100 == 0: await db.commit() - logger.info(f" Processed {updated}/{total} pages...") except Exception as e: - logger.warning(f"Error processing {row['uri']}: {e}") + errors += 1 + logger.debug(f"Error processing {row['uri']}: {e}") await db.commit() - logger.info(f"Backfill complete: {updated} pages updated") + logger.info(f"Backfill complete: {updated} updated, {errors} errors") async def backfill_screenshots( @@ -981,6 +979,7 @@ async def backfill_screenshots( domain_filter: str = None, delete_old: bool = True, fast_mode: bool = False, + quiet: bool = False, ): """Re-capture screenshots as JPEG to replace old PNGs. @@ -990,6 +989,7 @@ async def backfill_screenshots( domain_filter: Only process pages matching this domain delete_old: Delete old PNG files after successful JPEG capture fast_mode: Skip delay between captures (for sites without robots.txt) + quiet: Disable progress bar """ from screenshot import ScreenshotCapture, ScreenshotConfig from storage import ImageVault @@ -1054,8 +1054,22 @@ async def backfill_screenshots( domain_last_fetched = {} # Track last fetch time per domain crawl_delay = 2.0 # Default crawl delay in seconds - pbar = tqdm(total=total, initial=skipped_count, desc="Screenshots", unit="pages") - for row in rows: + # Concurrent workers in fast mode (based on CPU count) + import multiprocessing + max_workers = multiprocessing.cpu_count() if fast_mode else 1 + semaphore = asyncio.Semaphore(max_workers) + if fast_mode and max_workers > 1: + logger.info(f"Fast mode: {max_workers} concurrent workers") + + pbar = tqdm(total=total, initial=skipped_count, desc="Screenshots", unit="pages", disable=quiet) + + # Lock for thread-safe updates + state_lock = asyncio.Lock() + stats = {'captured': 0, 'failed': 0, 'bytes_saved': 0} + + async def process_row(row): + nonlocal completed_uris + async with semaphore: page_uri = row.page_uri old_hash = row.md5_hash domain = Uri(page_uri).hostname.lower() @@ -1337,6 +1351,12 @@ async def main(): help="With --backfill-markdown: strip nav/header/footer/logo before conversion" ) + parser.add_argument( + "-q", "--quiet", + action="store_true", + help="Disable progress bars" + ) + parser.add_argument( "--backfill-screenshots", metavar="DOMAIN", @@ -1386,7 +1406,7 @@ async def main(): # Handle --backfill-markdown if args.backfill_markdown: - await backfill_markdown(args.db, domain_filter=args.backfill_markdown, trim_wrapper=args.trim_wrapper) + await backfill_markdown(args.db, domain_filter=args.backfill_markdown, trim_wrapper=args.trim_wrapper, quiet=args.quiet) return # Handle --backfill-screenshots @@ -1416,6 +1436,7 @@ async def main(): domain_filter=args.backfill_screenshots, delete_old=not args.keep_old_screenshots, fast_mode=args.fast, + quiet=args.quiet, ) finally: if serp_process: diff --git a/screenshot.py b/screenshot.py index ce47ec2..0ff41a2 100644 --- a/screenshot.py +++ b/screenshot.py @@ -194,13 +194,13 @@ class ScreenshotCapture: content_length: Length of raw HTML content in bytes Returns: - Delay in milliseconds (min 2000ms, max 30000ms) + Delay in milliseconds (min 3000ms, max 60000ms) """ - # Base delay of 2 seconds - base_delay = 2000 + # Base delay of 3 seconds + base_delay = 3000 - # Add 2 seconds per 50KB of content, up to a max of 28 seconds - additional_delay = min((content_length // 50000) * 2000, 28000) + # Add 3 seconds per 50KB of content, up to a max of 57 seconds + additional_delay = min((content_length // 50000) * 3000, 57000) return base_delay + additional_delay diff --git a/serp.py b/serp.py index 667a0b3..a323c4c 100644 --- a/serp.py +++ b/serp.py @@ -1912,13 +1912,12 @@ async def view_page( escaped = html_module.escape(page["content"][:50000]) content_html = f"
{escaped}
" - # Find screenshot + # Find all screenshots (may be chunked for tall pages) + screenshot_hashes = await db.get_page_screenshots(uri) screenshot_html = "" - screenshot_hash = await db.get_page_screenshot(uri) - if screenshot_hash: - screenshot_html = f''' - Page screenshot - ''' + if screenshot_hashes: + imgs = ''.join(f'Page screenshot chunk' for h in screenshot_hashes) + screenshot_html = f'
{imgs}
' # Find media from this page media_items = await db.get_page_media(uri) @@ -1942,13 +1941,36 @@ async def view_page(
{''.join(media_cards)}
''' + has_screenshot = bool(screenshot_html) return f""" {html_module.escape(page_title)} - neopig - +
@@ -1958,19 +1980,42 @@ async def view_page( Random
-
- {screenshot_html or ''} -

{html_module.escape(page_title)}

URL:{uri}
Media:{len(media_items)} items
- {media_grid} -

Page Content

-
{content_html or '

No content available

'}
+
+
+ {media_grid} +

Page Content

+
{content_html or '

No content available

'}
+
+ {f'''
+ ◀ Screenshot + {screenshot_html} +
''' if has_screenshot else ''} +
- + {'' if noai else f''' @@ -2269,27 +2314,38 @@ async def get_stats_endpoint(): @app.get("/random") -async def random_media(): - """Redirect to a random media item.""" +async def random_item(type: str = Query(None, description="Type: media or page (random if not specified)")): + """Redirect to a random media item or page.""" from sqlalchemy import select, func - from database import Media + from database import Media, Page import random - async with db.session() as session: - # Get a random media item (excluding screenshots) - stmt = ( - select(Media.md5_hash) - .where(Media.media_type != 'screenshot') - .order_by(func.random()) - .limit(1) - ) - result = await session.execute(stmt) - row = result.fetchone() + # If no type specified, randomly pick between media and page + if type is None: + type = random.choice(["media", "page"]) - if row: - return RedirectResponse(url=f"/view/{row[0]}", status_code=302) + async with db.session() as session: + if type == "page": + # Get a random page + stmt = select(Page.uri).order_by(func.random()).limit(1) + result = await session.execute(stmt) + row = result.fetchone() + if row: + return RedirectResponse(url=f"/page/{row[0]}", status_code=302) else: - return RedirectResponse(url="/", status_code=302) + # Get a random media item (excluding screenshots) + stmt = ( + select(Media.md5_hash) + .where(Media.media_type != 'screenshot') + .order_by(func.random()) + .limit(1) + ) + result = await session.execute(stmt) + row = result.fetchone() + if row: + return RedirectResponse(url=f"/view/{row[0]}", status_code=302) + + return RedirectResponse(url="/", status_code=302) @app.get("/api/search")