Add instant replay for crawl jobs and NEOPIG_NO_VCS flag
- Add /api/crawl/jobs/{job_id}/replay SSE endpoint for replaying crawls
- Replay streams media at original discovery pace with speed multiplier
- Add Replay button to crawl jobs UI (works even when crawling disabled)
- Track active replays to prevent double-clicking same job
- Live feed page handles ?replay=JOB_ID parameter for replay mode
- Add NEOPIG_NO_VCS=1 env var to skip VCS repository detection
- Fix hljs.highlightAll() call when highlight.js not loaded
This commit is contained in:
parent
2dc46b1bce
commit
e5972a3171
6 changed files with 195 additions and 13 deletions
19
database.py
19
database.py
|
|
@ -1171,3 +1171,22 @@ class Database:
|
|||
|
||||
result = await session.execute(stmt)
|
||||
return [row[0] for row in result.fetchall()]
|
||||
|
||||
async def get_job_media_timeline(self, job_id: int) -> List[Dict[str, Any]]:
|
||||
"""Get all media for a job ordered by discovery time for replay.
|
||||
|
||||
Returns list of {md5_hash, media_type, discovered_at, page_uri, page_title}
|
||||
sorted by discovered_at ascending.
|
||||
"""
|
||||
async with self.session() as session:
|
||||
stmt = (
|
||||
select(
|
||||
Media.md5_hash, Media.media_type, Media.mime_type,
|
||||
MediaSource.discovered_at, MediaSource.page_uri, MediaSource.page_title
|
||||
)
|
||||
.join(MediaSource, Media.md5_hash == MediaSource.md5_hash)
|
||||
.where(MediaSource.crawl_job_id == job_id)
|
||||
.order_by(MediaSource.discovered_at.asc())
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
return [dict(row._mapping) for row in result.fetchall()]
|
||||
|
|
|
|||
27
neopig.py
27
neopig.py
|
|
@ -51,6 +51,9 @@ from tqdm import tqdm
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Environment configuration
|
||||
NO_VCS = os.environ.get("NEOPIG_NO_VCS", "").lower() in ("1", "true", "yes")
|
||||
|
||||
# Track child processes for cleanup on exit
|
||||
_CHILD_PROCESSES: List = []
|
||||
|
||||
|
|
@ -913,17 +916,19 @@ class NeoPig:
|
|||
keywords = keywords or []
|
||||
|
||||
# Check if target is a VCS repository (git, hg, svn, etc.)
|
||||
vcs_type, clone_url = detect_vcs(target_uri)
|
||||
if vcs_type:
|
||||
logger.info(f"Detected {vcs_type} repository: {clone_url}")
|
||||
return await self.clone_and_index(
|
||||
target_uri=target_uri,
|
||||
clone_url=clone_url,
|
||||
vcs_type=vcs_type,
|
||||
keywords=keywords,
|
||||
job_id=job_id,
|
||||
quiet=quiet,
|
||||
)
|
||||
# Skip if NEOPIG_NO_VCS=1
|
||||
if not NO_VCS:
|
||||
vcs_type, clone_url = detect_vcs(target_uri)
|
||||
if vcs_type:
|
||||
logger.info(f"Detected {vcs_type} repository: {clone_url}")
|
||||
return await self.clone_and_index(
|
||||
target_uri=target_uri,
|
||||
clone_url=clone_url,
|
||||
vcs_type=vcs_type,
|
||||
keywords=keywords,
|
||||
job_id=job_id,
|
||||
quiet=quiet,
|
||||
)
|
||||
|
||||
# Create crawl job (unless one was provided)
|
||||
if job_id is None:
|
||||
|
|
|
|||
72
serp.py
72
serp.py
|
|
@ -2714,6 +2714,78 @@ async def get_crawl_job_logs(job_id: int, tail: int = Query(0, description="Retu
|
|||
return Response(content=logs, media_type="text/plain")
|
||||
|
||||
|
||||
@app.get("/api/crawl/jobs/{job_id}/replay")
|
||||
async def replay_crawl_job(job_id: int, speed: float = Query(1.0, ge=0.1, le=100.0)):
|
||||
"""
|
||||
Replay a completed crawl job as SSE stream.
|
||||
|
||||
Streams media items at the original discovery pace (adjusted by speed multiplier).
|
||||
This is a read-only operation - no network requests or DB modifications.
|
||||
Connect with EventSource to watch the replay in the /live feed.
|
||||
|
||||
Args:
|
||||
job_id: The crawl job to replay
|
||||
speed: Playback speed multiplier (1.0 = realtime, 2.0 = 2x faster, 0.5 = half speed)
|
||||
"""
|
||||
from datetime import datetime
|
||||
|
||||
# Verify job exists
|
||||
job = await db.get_crawl_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
# Get media timeline for this job
|
||||
timeline = await db.get_job_media_timeline(job_id)
|
||||
if not timeline:
|
||||
raise HTTPException(status_code=404, detail="No media found for this job")
|
||||
|
||||
async def replay_generator():
|
||||
prev_time = None
|
||||
for item in timeline:
|
||||
# Parse discovery time
|
||||
discovered_at = item.get('discovered_at')
|
||||
if discovered_at:
|
||||
try:
|
||||
curr_time = datetime.fromisoformat(discovered_at.replace('Z', '+00:00'))
|
||||
|
||||
# Calculate delay from previous item
|
||||
if prev_time:
|
||||
delta = (curr_time - prev_time).total_seconds()
|
||||
# Apply speed multiplier and cap delay at 5 seconds max
|
||||
delay = min(delta / speed, 5.0)
|
||||
if delay > 0:
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
prev_time = curr_time
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Format media item for SSE (same format as live stream)
|
||||
media_event = {
|
||||
'md5_hash': item['md5_hash'],
|
||||
'media_type': item.get('media_type', 'image'),
|
||||
'mime_type': item.get('mime_type', ''),
|
||||
'page_uri': item.get('page_uri', ''),
|
||||
'page_title': item.get('page_title', ''),
|
||||
'replay': True, # Flag to indicate this is a replay
|
||||
'job_id': job_id,
|
||||
}
|
||||
yield f"data: {json.dumps(media_event)}\n\n"
|
||||
|
||||
# Send end-of-replay marker
|
||||
yield f"data: {json.dumps({'type': 'replay_end', 'job_id': job_id, 'total': len(timeline)})}\n\n"
|
||||
|
||||
return StreamingResponse(
|
||||
replay_generator(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/crawl")
|
||||
async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -52,6 +52,8 @@ h2 { color: #ff6b6b; font-size: 18px; margin-bottom: 15px; }
|
|||
.recrawl-btn:hover { background: #2980b9; }
|
||||
.delete-btn { background: #7f8c8d; }
|
||||
.delete-btn:hover { background: #c0392b; }
|
||||
.replay-btn { background: #9b59b6; }
|
||||
.replay-btn:hover { background: #8e44ad; }
|
||||
.job-links { margin-left: 10px; font-size: 12px; }
|
||||
.job-links a { color: #4ecdc4; margin-right: 8px; }
|
||||
.job-links a:hover { color: #7bed72; }
|
||||
|
|
@ -284,6 +286,51 @@ async function deleteJob(jobId) {
|
|||
} catch (err) { alert(T.error + ': ' + err.message); }
|
||||
}
|
||||
|
||||
// Track active replays to prevent double-clicking
|
||||
const activeReplays = new Set();
|
||||
|
||||
function replayJob(jobId) {
|
||||
// Prevent duplicate replays of same job
|
||||
if (activeReplays.has(jobId)) {
|
||||
alert('Replay for job #' + jobId + ' is already running');
|
||||
return;
|
||||
}
|
||||
|
||||
activeReplays.add(jobId);
|
||||
|
||||
// Update button to show active state
|
||||
const btn = document.querySelector(`button[onclick="replayJob(${jobId})"]`);
|
||||
if (btn) {
|
||||
btn.textContent = '⏳ Replaying...';
|
||||
btn.disabled = true;
|
||||
}
|
||||
|
||||
// Open live feed page with replay parameter
|
||||
const replayWindow = window.open('/live?replay=' + jobId, '_blank');
|
||||
|
||||
// Listen for window close to reset state (works for same-origin)
|
||||
const checkClosed = setInterval(() => {
|
||||
if (replayWindow && replayWindow.closed) {
|
||||
clearInterval(checkClosed);
|
||||
activeReplays.delete(jobId);
|
||||
if (btn) {
|
||||
btn.textContent = '▶ Replay';
|
||||
btn.disabled = false;
|
||||
}
|
||||
}
|
||||
}, 1000);
|
||||
|
||||
// Auto-reset after 5 minutes as fallback
|
||||
setTimeout(() => {
|
||||
clearInterval(checkClosed);
|
||||
activeReplays.delete(jobId);
|
||||
if (btn) {
|
||||
btn.textContent = '▶ Replay';
|
||||
btn.disabled = false;
|
||||
}
|
||||
}, 300000);
|
||||
}
|
||||
|
||||
function formatBytes(bytes) {
|
||||
if (!bytes) return '';
|
||||
if (bytes < 1024) return bytes + ' B';
|
||||
|
|
@ -379,6 +426,7 @@ function renderJob(job) {
|
|||
${job.status === 'running' && job.job_kind === 'crawl' ? '<button class="pause-btn" onclick="pauseJob(' + job.id + ')">' + T.pause + '</button>' : ''}
|
||||
${(job.status === 'paused' || job.status === 'cancelled') && job.job_kind === 'crawl' ? '<button class="start-btn" onclick="resumeJob(' + job.id + ')">' + T.resume + '</button>' : ''}
|
||||
${(job.status === 'completed' || job.status === 'failed' || job.status === 'cancelled') && job.job_kind === 'crawl' ? "<button class=\"recrawl-btn\" onclick='loadJobSettings(" + JSON.stringify(job) + ")'>↻</button>" : ''}
|
||||
${job.status === 'completed' && job.job_kind === 'crawl' && (stats.media_found > 0 || stats.media_downloaded > 0) ? '<button class="replay-btn" onclick="replayJob(' + job.id + ')">▶ Replay</button>' : ''}
|
||||
<button class="console-btn" onclick="openConsole(${job.id})">${T.console}</button>
|
||||
${job.status !== 'running' ? '<button class="delete-btn" onclick="deleteJob(' + job.id + ')">' + T.delete + '</button>' : ''}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -77,20 +77,53 @@ h1 { color: #ff6b6b; margin-bottom: 5px; }
|
|||
const T = {{ t_json|safe }};
|
||||
const DOMAIN_FILTER = {{ domain_json|safe }};
|
||||
|
||||
// Check for replay mode from URL
|
||||
const urlParams = new URLSearchParams(window.location.search);
|
||||
const REPLAY_JOB_ID = urlParams.get('replay');
|
||||
|
||||
let running = true;
|
||||
let seenHashes = new Map();
|
||||
let newCount = 0;
|
||||
let startTime = Date.now();
|
||||
let eventSource = null;
|
||||
let replayMode = !!REPLAY_JOB_ID;
|
||||
|
||||
// Update UI for replay mode
|
||||
if (replayMode) {
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
const h1 = document.querySelector('h1');
|
||||
h1.textContent = '▶ Replay Job #' + REPLAY_JOB_ID;
|
||||
h1.style.color = '#9b59b6';
|
||||
const subtitle = document.querySelector('.subtitle');
|
||||
subtitle.textContent = 'Replaying crawl at original discovery pace';
|
||||
subtitle.style.color = '#9b59b6';
|
||||
});
|
||||
}
|
||||
|
||||
function connectSSE() {
|
||||
if (eventSource) eventSource.close();
|
||||
eventSource = new EventSource('/api/live/stream');
|
||||
|
||||
// Use replay endpoint if in replay mode
|
||||
const sseUrl = replayMode
|
||||
? '/api/crawl/jobs/' + REPLAY_JOB_ID + '/replay'
|
||||
: '/api/live/stream';
|
||||
|
||||
eventSource = new EventSource(sseUrl);
|
||||
|
||||
eventSource.onmessage = function(event) {
|
||||
if (!running) return;
|
||||
try {
|
||||
const item = JSON.parse(event.data);
|
||||
|
||||
// Handle replay end marker
|
||||
if (item.type === 'replay_end') {
|
||||
const status = document.getElementById('status');
|
||||
status.textContent = '✓ Replay complete (' + item.total + ' items)';
|
||||
status.style.color = '#10b981';
|
||||
eventSource.close();
|
||||
return;
|
||||
}
|
||||
|
||||
addMediaCard(item, true);
|
||||
} catch (e) {
|
||||
console.error('SSE parse error:', e);
|
||||
|
|
@ -98,6 +131,11 @@ function connectSSE() {
|
|||
};
|
||||
|
||||
eventSource.onerror = function(e) {
|
||||
if (replayMode) {
|
||||
// Replay might have ended normally
|
||||
console.log('Replay stream ended');
|
||||
return;
|
||||
}
|
||||
console.log('SSE reconnecting...');
|
||||
setTimeout(connectSSE, 3000);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -198,7 +198,7 @@ h3 { color: #ff6b6b; font-size: 16px; margin: 30px 0 15px 0; border-bottom: 1px
|
|||
{% block scripts %}
|
||||
<script>
|
||||
const T = {{ t_json|safe }};
|
||||
hljs.highlightAll();
|
||||
if (typeof hljs !== 'undefined') hljs.highlightAll();
|
||||
|
||||
function toggleScreenshots() {
|
||||
const container = document.getElementById('screenshot-container');
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue