engineering/pig.py
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix crawl UI: async task, real-time progress, better defaults

Browse source 
- Fix asyncio error: use create_task() directly instead of BackgroundTasks
- Default depth=9, max_pages=-1 for full site crawls
- Add update_crawl_job_stats() for live progress updates
- Poll jobs every 2s while running, show live stats
- Animated progress bar for running jobs
- Fix get_media_type_from_extension() None path crash
This commit is contained in:
Russell Ballestrini 2025-12-30 15:14:49 -05:00
parent cbed6d9d9a
commit feb6eed596
3 changed files with 81 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

2
async_web_fetcher.py
View file

@ -379,6 +379,8 @@ def get_media_type_from_extension(url: str) -> Optional[str]:
'image', 'video', 'audio', or None
"""
parsed = Uri(url)
if not parsed.path:
return None
path = parsed.path.lower()
for ext in IMAGE_EXTENSIONS:

11
database.py
View file

@ -282,6 +282,17 @@ class Database:
await session.commit()
return job.id
async def update_crawl_job_stats(self, job_id: int, stats: Dict[str, Any]) -> None:
"""Update stats for a running crawl job (for progress tracking)."""
async with self.session() as session:
stmt = (
update(CrawlJob)
.where(CrawlJob.id == job_id)
.values(stats=json.dumps(stats))
)
await session.execute(stmt)
await session.commit()
async def complete_crawl_job(self, job_id: int, stats: Dict[str, Any]) -> None:
"""Mark a crawl job as complete."""
async with self.session() as session:

83
serp.py
View file

@ -1010,6 +1010,22 @@ CRAWL_HTML = """
background: #ff6b6b;
transition: width 0.3s;
}
.progress-bar-fill.running {
width: 100%;
background: linear-gradient(90deg, #ff6b6b 0%, #4ecdc4 50%, #ff6b6b 100%);
background-size: 200% 100%;
animation: progress-wave 1.5s ease-in-out infinite;
}
@keyframes progress-wave {
0% { background-position: 200% 0; }
100% { background-position: -200% 0; }
}
.job.running {
border-color: #4ecdc4;
}
.job-stats.live {
color: #4ecdc4;
}
.no-jobs {
color: #666;
@ -1066,12 +1082,12 @@ CRAWL_HTML = """
</select>
</div>
<div class="form-group">
<label>Depth (5 recommended, max 15)</label>
<input type="number" id="depth" value="5" min="1" max="15">
<label>Depth (9 = full site, max 15)</label>
<input type="number" id="depth" value="9" min="1" max="15">
</div>
<div class="form-group">
<label>Max Pages (-1 = unlimited)</label>
<input type="number" id="max_pages" value="500" min="-1">
<input type="number" id="max_pages" value="-1" min="-1">
</div>
</div>
@ -1161,12 +1177,24 @@ CRAWL_HTML = """
return;
}
const hasRunning = jobs.some(j => j.status === 'running');
container.innerHTML = jobs.map(job => {
const stats = job.stats ? JSON.parse(job.stats) : {};
const keywords = job.keywords ? JSON.parse(job.keywords) : [];
// Show live stats for running jobs
const liveStats = job.status === 'running' ? `
<div class="job-stats live">
<span>📄 ${stats.pages_crawled || 0} pages</span>
<span>🖼️ ${stats.media_found || 0} found</span>
<span>💾 ${stats.media_downloaded || 0} saved</span>
<span>📸 ${stats.screenshots_taken || 0} screenshots</span>
</div>
` : '';
return `
<div class="job">
<div class="job ${job.status}">
<div class="job-header">
<span class="job-id">Job #${job.id}</span>
<span class="job-status ${job.status}">${job.status}</span>
@ -1185,14 +1213,20 @@ CRAWL_HTML = """
<span>♻️ ${stats.duplicates_skipped || 0} dupes</span>
</div>
` : ''}
${liveStats}
${job.status === 'running' ? `
<div class="progress-bar">
<div class="progress-bar-fill" style="width: 50%"></div>
<div class="progress-bar-fill running"></div>
</div>
` : ''}
</div>
`;
}).join('');
// Poll faster while jobs are running
if (hasRunning) {
setTimeout(loadJobs, 2000);
}
} catch (err) {
document.getElementById('jobs').innerHTML = '<div class="no-jobs">Failed to load jobs</div>';
}
@ -2274,14 +2308,33 @@ async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks):
# Cap depth at 15 (convert -1 or values > 15 to 15)
depth = request.depth if 1 <= request.depth <= 15 else 15
stats = await pig.crawl(
target_uri=uri,
keywords=request.keywords,
mode=mode,
depth=depth,
max_pages=request.max_pages,
download_media=request.download_media,
)
# Progress updater - runs alongside crawl
stop_progress = asyncio.Event()
async def update_progress():
while not stop_progress.is_set():
await asyncio.sleep(2) # Update every 2 seconds
if not stop_progress.is_set():
await db.update_crawl_job_stats(jid, pig.stats)
progress_task = asyncio.create_task(update_progress())
try:
stats = await pig.crawl(
target_uri=uri,
keywords=request.keywords,
mode=mode,
depth=depth,
max_pages=request.max_pages,
download_media=request.download_media,
)
finally:
stop_progress.set()
progress_task.cancel()
try:
await progress_task
except asyncio.CancelledError:
pass
# Update job as completed
await db.complete_crawl_job(jid, stats)
@ -2290,8 +2343,8 @@ async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks):
logger.error(f"Crawl job {jid} failed: {e}")
await db.complete_crawl_job(jid, {"error": str(e), "status": "failed"})
# Schedule background task
background_tasks.add_task(asyncio.create_task, run_crawl())
# Schedule async task on the event loop (not BackgroundTasks which runs in threadpool)
asyncio.create_task(run_crawl())
return {"job_ids": job_ids, "status": "running", "count": len(job_ids)}