engineering/pig.py
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Screenshot backfill: query content_length for dynamic delay

Browse source
This commit is contained in:
Russell Ballestrini 2025-12-30 16:17:44 -05:00
parent 5228f21675
commit 3fa3c9722b
1 changed files with 12 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

13
neopig.py
View file

@ -932,7 +932,18 @@ class NeoPig:
async def capture_one(page_uri: str):
async with semaphore:
try:
await self._capture_page_screenshot(page_uri, crawl_job_id or 0, content_length=0)
# Get content length for dynamic delay
content_length = 0
async with aiosqlite.connect(db_path) as conn:
cursor = await conn.execute(
"SELECT LENGTH(raw_html) FROM pages WHERE uri = ?",
(page_uri,)
)
row = await cursor.fetchone()
if row and row[0]:
content_length = row[0]
await self._capture_page_screenshot(page_uri, crawl_job_id or 0, content_length=content_length)
# Check if screenshot was actually captured (added to seen_screenshots)
success = page_uri in self.seen_screenshots
except Exception as e: