engineering/pig.py
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Screenshot backfill: 1x CPU workers in normal mode (not just 1)

Browse source
This commit is contained in:
Russell Ballestrini 2025-12-30 15:31:17 -05:00
parent cd56e23a75
commit ad05d1a308
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
neopig.py
View file

@ -874,9 +874,9 @@ class NeoPig:
logger.info(f"Backfilling {len(pages)} missing screenshots for {domain}")
# Concurrent workers: 3x CPU in fast mode, 1 otherwise (for crawl delay)
# Concurrent workers: 3x CPU in fast mode, 1x CPU otherwise (crawl delay handled per-capture)
import multiprocessing
max_workers = multiprocessing.cpu_count() * 3 if fast_mode else 1
max_workers = multiprocessing.cpu_count() * 3 if fast_mode else multiprocessing.cpu_count()
semaphore = asyncio.Semaphore(max_workers)
if fast_mode and max_workers > 1:
logger.info(f"Fast mode: {max_workers} concurrent workers")