Add crawl ETA estimates to UI based on pages/sec rate

neopig.py

@@ -154,6 +154,7 @@ class NeoPig:
         # Track stats
         self.stats = {
             'pages_crawled': 0,
+            'pages_pending': 0,  # Queue size for ETA calculation
             'pages_changed': 0,  # Pages with content changes (for git commit)
             'media_found': 0,
             'media_downloaded': 0,
@@ -164,6 +165,7 @@ class NeoPig:
             'errors': 0,
             'bytes_downloaded': 0,  # Total bytes fetched from network
             'bytes_stored': 0,  # Unique bytes stored in vault
+            'crawl_started': 0,  # Unix timestamp for rate calculation
         }
 
         # Track seen media URLs -> md5_hash (to add page context without re-download)
@@ -494,6 +496,7 @@ class NeoPig:
 
         # Track timing for stats
         start_time = datetime.now(timezone.utc)
+        self.stats['crawl_started'] = int(start_time.timestamp())
 
         def format_size(b: int) -> str:
             if b < 1024:
@@ -510,6 +513,7 @@ class NeoPig:
 
         def on_uris_total(total: int):
             uris_total[0] = total
+            self.stats['pages_pending'] = max(0, total - self.stats['pages_crawled'])
             if self.pbar and total > 0:
                 self.pbar.total = total
                 self.pbar.refresh()

serp.py

@@ -1394,10 +1394,24 @@ CRAWL_HTML = """
                         `;
                     }
 
-                    // Crawl job display
+                    // Crawl job display with ETA
+                    const calcEta = () => {
+                        const crawled = stats.pages_crawled || 0;
+                        const pending = stats.pages_pending || 0;
+                        const started = stats.crawl_started || 0;
+                        if (crawled === 0 || started === 0 || pending === 0) return '';
+                        const elapsed = Date.now() / 1000 - started;
+                        if (elapsed < 5) return '';  // Wait for meaningful data
+                        const rate = crawled / elapsed;
+                        const etaSec = pending / rate;
+                        if (etaSec < 60) return `~${Math.round(etaSec)}s`;
+                        if (etaSec < 3600) return `~${Math.round(etaSec / 60)}m`;
+                        return `~${(etaSec / 3600).toFixed(1)}h`;
+                    };
+                    const eta = calcEta();
                     const liveStats = job.status === 'running' ? `
                         <div class="job-stats live">
-                            <span>📄 ${stats.pages_crawled || 0} pages</span>
+                            <span>📄 ${stats.pages_crawled || 0}${stats.pages_pending ? '/' + ((stats.pages_crawled || 0) + stats.pages_pending) : ''} pages${eta ? ' ⏱️' + eta : ''}</span>
                             <span>🖼️ ${stats.media_found || 0} found</span>
                             <span>💾 ${stats.media_downloaded || 0} saved</span>
                             <span>📸 ${stats.screenshots_taken || 0} screenshots</span>