diff --git a/async_web_fetcher.py b/async_web_fetcher.py
index 2ab7edd..711e852 100644
--- a/async_web_fetcher.py
+++ b/async_web_fetcher.py
@@ -379,6 +379,8 @@ def get_media_type_from_extension(url: str) -> Optional[str]:
'image', 'video', 'audio', or None
"""
parsed = Uri(url)
+ if not parsed.path:
+ return None
path = parsed.path.lower()
for ext in IMAGE_EXTENSIONS:
diff --git a/database.py b/database.py
index fe10efb..c0e39ab 100644
--- a/database.py
+++ b/database.py
@@ -282,6 +282,17 @@ class Database:
await session.commit()
return job.id
+ async def update_crawl_job_stats(self, job_id: int, stats: Dict[str, Any]) -> None:
+ """Update stats for a running crawl job (for progress tracking)."""
+ async with self.session() as session:
+ stmt = (
+ update(CrawlJob)
+ .where(CrawlJob.id == job_id)
+ .values(stats=json.dumps(stats))
+ )
+ await session.execute(stmt)
+ await session.commit()
+
async def complete_crawl_job(self, job_id: int, stats: Dict[str, Any]) -> None:
"""Mark a crawl job as complete."""
async with self.session() as session:
diff --git a/serp.py b/serp.py
index eacc6a5..a1d3b3e 100644
--- a/serp.py
+++ b/serp.py
@@ -1010,6 +1010,22 @@ CRAWL_HTML = """
background: #ff6b6b;
transition: width 0.3s;
}
+ .progress-bar-fill.running {
+ width: 100%;
+ background: linear-gradient(90deg, #ff6b6b 0%, #4ecdc4 50%, #ff6b6b 100%);
+ background-size: 200% 100%;
+ animation: progress-wave 1.5s ease-in-out infinite;
+ }
+ @keyframes progress-wave {
+ 0% { background-position: 200% 0; }
+ 100% { background-position: -200% 0; }
+ }
+ .job.running {
+ border-color: #4ecdc4;
+ }
+ .job-stats.live {
+ color: #4ecdc4;
+ }
.no-jobs {
color: #666;
@@ -1066,12 +1082,12 @@ CRAWL_HTML = """
-
-
+
+
-
+
@@ -1161,12 +1177,24 @@ CRAWL_HTML = """
return;
}
+ const hasRunning = jobs.some(j => j.status === 'running');
+
container.innerHTML = jobs.map(job => {
const stats = job.stats ? JSON.parse(job.stats) : {};
const keywords = job.keywords ? JSON.parse(job.keywords) : [];
+ // Show live stats for running jobs
+ const liveStats = job.status === 'running' ? `
+
+ 📄 ${stats.pages_crawled || 0} pages
+ 🖼️ ${stats.media_found || 0} found
+ 💾 ${stats.media_downloaded || 0} saved
+ 📸 ${stats.screenshots_taken || 0} screenshots
+
+ ` : '';
+
return `
-
+
` : ''}
+ ${liveStats}
${job.status === 'running' ? `
` : ''}
`;
}).join('');
+
+ // Poll faster while jobs are running
+ if (hasRunning) {
+ setTimeout(loadJobs, 2000);
+ }
} catch (err) {
document.getElementById('jobs').innerHTML = '
Failed to load jobs
';
}
@@ -2274,14 +2308,33 @@ async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks):
# Cap depth at 15 (convert -1 or values > 15 to 15)
depth = request.depth if 1 <= request.depth <= 15 else 15
- stats = await pig.crawl(
- target_uri=uri,
- keywords=request.keywords,
- mode=mode,
- depth=depth,
- max_pages=request.max_pages,
- download_media=request.download_media,
- )
+ # Progress updater - runs alongside crawl
+ stop_progress = asyncio.Event()
+
+ async def update_progress():
+ while not stop_progress.is_set():
+ await asyncio.sleep(2) # Update every 2 seconds
+ if not stop_progress.is_set():
+ await db.update_crawl_job_stats(jid, pig.stats)
+
+ progress_task = asyncio.create_task(update_progress())
+
+ try:
+ stats = await pig.crawl(
+ target_uri=uri,
+ keywords=request.keywords,
+ mode=mode,
+ depth=depth,
+ max_pages=request.max_pages,
+ download_media=request.download_media,
+ )
+ finally:
+ stop_progress.set()
+ progress_task.cancel()
+ try:
+ await progress_task
+ except asyncio.CancelledError:
+ pass
# Update job as completed
await db.complete_crawl_job(jid, stats)
@@ -2290,8 +2343,8 @@ async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks):
logger.error(f"Crawl job {jid} failed: {e}")
await db.complete_crawl_job(jid, {"error": str(e), "status": "failed"})
- # Schedule background task
- background_tasks.add_task(asyncio.create_task, run_crawl())
+ # Schedule async task on the event loop (not BackgroundTasks which runs in threadpool)
+ asyncio.create_task(run_crawl())
return {"job_ids": job_ids, "status": "running", "count": len(job_ids)}