diff --git a/neopig.py b/neopig.py index 4866541..a0a5963 100644 --- a/neopig.py +++ b/neopig.py @@ -154,6 +154,7 @@ class NeoPig: # Track stats self.stats = { 'pages_crawled': 0, + 'pages_pending': 0, # Queue size for ETA calculation 'pages_changed': 0, # Pages with content changes (for git commit) 'media_found': 0, 'media_downloaded': 0, @@ -164,6 +165,7 @@ class NeoPig: 'errors': 0, 'bytes_downloaded': 0, # Total bytes fetched from network 'bytes_stored': 0, # Unique bytes stored in vault + 'crawl_started': 0, # Unix timestamp for rate calculation } # Track seen media URLs -> md5_hash (to add page context without re-download) @@ -494,6 +496,7 @@ class NeoPig: # Track timing for stats start_time = datetime.now(timezone.utc) + self.stats['crawl_started'] = int(start_time.timestamp()) def format_size(b: int) -> str: if b < 1024: @@ -510,6 +513,7 @@ class NeoPig: def on_uris_total(total: int): uris_total[0] = total + self.stats['pages_pending'] = max(0, total - self.stats['pages_crawled']) if self.pbar and total > 0: self.pbar.total = total self.pbar.refresh() diff --git a/serp.py b/serp.py index 9eee22b..b66c6de 100644 --- a/serp.py +++ b/serp.py @@ -1394,10 +1394,24 @@ CRAWL_HTML = """ `; } - // Crawl job display + // Crawl job display with ETA + const calcEta = () => { + const crawled = stats.pages_crawled || 0; + const pending = stats.pages_pending || 0; + const started = stats.crawl_started || 0; + if (crawled === 0 || started === 0 || pending === 0) return ''; + const elapsed = Date.now() / 1000 - started; + if (elapsed < 5) return ''; // Wait for meaningful data + const rate = crawled / elapsed; + const etaSec = pending / rate; + if (etaSec < 60) return `~${Math.round(etaSec)}s`; + if (etaSec < 3600) return `~${Math.round(etaSec / 60)}m`; + return `~${(etaSec / 3600).toFixed(1)}h`; + }; + const eta = calcEta(); const liveStats = job.status === 'running' ? `
- 📄 ${stats.pages_crawled || 0} pages + 📄 ${stats.pages_crawled || 0}${stats.pages_pending ? '/' + ((stats.pages_crawled || 0) + stats.pages_pending) : ''} pages${eta ? ' ⏱️' + eta : ''} 🖼️ ${stats.media_found || 0} found 💾 ${stats.media_downloaded || 0} saved 📸 ${stats.screenshots_taken || 0} screenshots