diff --git a/neopig.py b/neopig.py index 28fd87b..23dcbc4 100644 --- a/neopig.py +++ b/neopig.py @@ -932,7 +932,18 @@ class NeoPig: async def capture_one(page_uri: str): async with semaphore: try: - await self._capture_page_screenshot(page_uri, crawl_job_id or 0, content_length=0) + # Get content length for dynamic delay + content_length = 0 + async with aiosqlite.connect(db_path) as conn: + cursor = await conn.execute( + "SELECT LENGTH(raw_html) FROM pages WHERE uri = ?", + (page_uri,) + ) + row = await cursor.fetchone() + if row and row[0]: + content_length = row[0] + + await self._capture_page_screenshot(page_uri, crawl_job_id or 0, content_length=content_length) # Check if screenshot was actually captured (added to seen_screenshots) success = page_uri in self.seen_screenshots except Exception as e: