From feb6eed596a2c1edea5371c6b29bc40dddcd3ccd Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Tue, 30 Dec 2025 15:14:49 -0500 Subject: [PATCH] Fix crawl UI: async task, real-time progress, better defaults - Fix asyncio error: use create_task() directly instead of BackgroundTasks - Default depth=9, max_pages=-1 for full site crawls - Add update_crawl_job_stats() for live progress updates - Poll jobs every 2s while running, show live stats - Animated progress bar for running jobs - Fix get_media_type_from_extension() None path crash --- async_web_fetcher.py | 2 ++ database.py | 11 ++++++ serp.py | 83 ++++++++++++++++++++++++++++++++++++-------- 3 files changed, 81 insertions(+), 15 deletions(-) diff --git a/async_web_fetcher.py b/async_web_fetcher.py index 2ab7edd..711e852 100644 --- a/async_web_fetcher.py +++ b/async_web_fetcher.py @@ -379,6 +379,8 @@ def get_media_type_from_extension(url: str) -> Optional[str]: 'image', 'video', 'audio', or None """ parsed = Uri(url) + if not parsed.path: + return None path = parsed.path.lower() for ext in IMAGE_EXTENSIONS: diff --git a/database.py b/database.py index fe10efb..c0e39ab 100644 --- a/database.py +++ b/database.py @@ -282,6 +282,17 @@ class Database: await session.commit() return job.id + async def update_crawl_job_stats(self, job_id: int, stats: Dict[str, Any]) -> None: + """Update stats for a running crawl job (for progress tracking).""" + async with self.session() as session: + stmt = ( + update(CrawlJob) + .where(CrawlJob.id == job_id) + .values(stats=json.dumps(stats)) + ) + await session.execute(stmt) + await session.commit() + async def complete_crawl_job(self, job_id: int, stats: Dict[str, Any]) -> None: """Mark a crawl job as complete.""" async with self.session() as session: diff --git a/serp.py b/serp.py index eacc6a5..a1d3b3e 100644 --- a/serp.py +++ b/serp.py @@ -1010,6 +1010,22 @@ CRAWL_HTML = """ background: #ff6b6b; transition: width 0.3s; } + .progress-bar-fill.running { + width: 100%; + background: linear-gradient(90deg, #ff6b6b 0%, #4ecdc4 50%, #ff6b6b 100%); + background-size: 200% 100%; + animation: progress-wave 1.5s ease-in-out infinite; + } + @keyframes progress-wave { + 0% { background-position: 200% 0; } + 100% { background-position: -200% 0; } + } + .job.running { + border-color: #4ecdc4; + } + .job-stats.live { + color: #4ecdc4; + } .no-jobs { color: #666; @@ -1066,12 +1082,12 @@ CRAWL_HTML = """
- - + +
- +
@@ -1161,12 +1177,24 @@ CRAWL_HTML = """ return; } + const hasRunning = jobs.some(j => j.status === 'running'); + container.innerHTML = jobs.map(job => { const stats = job.stats ? JSON.parse(job.stats) : {}; const keywords = job.keywords ? JSON.parse(job.keywords) : []; + // Show live stats for running jobs + const liveStats = job.status === 'running' ? ` +
+ 📄 ${stats.pages_crawled || 0} pages + 🖼️ ${stats.media_found || 0} found + 💾 ${stats.media_downloaded || 0} saved + 📸 ${stats.screenshots_taken || 0} screenshots +
+ ` : ''; + return ` -
+
Job #${job.id} ${job.status} @@ -1185,14 +1213,20 @@ CRAWL_HTML = """ ♻️ ${stats.duplicates_skipped || 0} dupes
` : ''} + ${liveStats} ${job.status === 'running' ? `
-
+
` : ''}
`; }).join(''); + + // Poll faster while jobs are running + if (hasRunning) { + setTimeout(loadJobs, 2000); + } } catch (err) { document.getElementById('jobs').innerHTML = '
Failed to load jobs
'; } @@ -2274,14 +2308,33 @@ async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks): # Cap depth at 15 (convert -1 or values > 15 to 15) depth = request.depth if 1 <= request.depth <= 15 else 15 - stats = await pig.crawl( - target_uri=uri, - keywords=request.keywords, - mode=mode, - depth=depth, - max_pages=request.max_pages, - download_media=request.download_media, - ) + # Progress updater - runs alongside crawl + stop_progress = asyncio.Event() + + async def update_progress(): + while not stop_progress.is_set(): + await asyncio.sleep(2) # Update every 2 seconds + if not stop_progress.is_set(): + await db.update_crawl_job_stats(jid, pig.stats) + + progress_task = asyncio.create_task(update_progress()) + + try: + stats = await pig.crawl( + target_uri=uri, + keywords=request.keywords, + mode=mode, + depth=depth, + max_pages=request.max_pages, + download_media=request.download_media, + ) + finally: + stop_progress.set() + progress_task.cancel() + try: + await progress_task + except asyncio.CancelledError: + pass # Update job as completed await db.complete_crawl_job(jid, stats) @@ -2290,8 +2343,8 @@ async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks): logger.error(f"Crawl job {jid} failed: {e}") await db.complete_crawl_job(jid, {"error": str(e), "status": "failed"}) - # Schedule background task - background_tasks.add_task(asyncio.create_task, run_crawl()) + # Schedule async task on the event loop (not BackgroundTasks which runs in threadpool) + asyncio.create_task(run_crawl()) return {"job_ids": job_ids, "status": "running", "count": len(job_ids)}