diff --git a/web/zebra-spaces.html b/web/zebra-spaces.html index d455c36..efc8bbe 100644 --- a/web/zebra-spaces.html +++ b/web/zebra-spaces.html @@ -2235,6 +2235,14 @@ const _lastTranscriptByUuid = new Map(); let _whisperBusy = false; let _whisperDroppedChunks = 0; let _whisperLastDropLogAt = 0; +/* Per-tick counters surfaced in the telemetry line. Reset each tick + * so we see RATE, not cumulative. Lets us SEE from server logs which + * clients have transcribe on, who's saturating their CPU, and how + * each device's inference latency compares (Snapdragon-8 vs mid- + * range MediaTek, x86 desktop vs phone). Fox 2026-06-05: "no wasted + * computations we can also know if some devices perform worse or + * better across mesh." */ +const _whisperTick = { sent: 0, dropped: 0, emitted: 0, totalLatencyMs: 0 }; function appendTranscriptLine(uuid, text){ const log = document.getElementById('transcript-log'); @@ -2260,6 +2268,75 @@ function appendTranscriptLine(uuid, text){ if (nearBottom) log.scrollTop = log.scrollHeight; } +/* Shared handler — used by both remote-speaker capture (one per uuid + * in listenerAudioNodes) and self capture (one for the local mic). + * Filters identical for both; uses the uuid to tag the transcript + * line so "you" vs "alice" appears correctly in the log. */ +async function handleWhisperChunk(uuid, chunk){ + if (_whisperBusy){ + _whisperDroppedChunks++; + _whisperTick.dropped++; + const now = Date.now(); + if (now - _whisperLastDropLogAt > 30000){ + _whisperLastDropLogAt = now; + logLine('', 'whisper: dropped '+_whisperDroppedChunks+' chunks (worker saturated)'); + } + return; + } + _whisperBusy = true; + _whisperTick.sent++; + const t0 = performance.now(); + try { + const txt = (await transcribeViaWorker(chunk)).trim(); + _whisperTick.totalLatencyMs += (performance.now() - t0); + if (txt.length < 2) return; + if (/^\[/.test(txt)) return; + const norm = txt.toLowerCase().replace(/[.!?,;:\s]+$/,'').trim(); + if (WHISPER_HALLUCINATIONS.has(norm)) return; + if (_lastTranscriptByUuid.get(uuid) === txt) return; + _lastTranscriptByUuid.set(uuid, txt); + appendTranscriptLine(uuid, txt); + _whisperTick.emitted++; + } catch (err){ + logLine('err', 'whisper transcribe '+uuid.slice(0,4)+': '+err.message); + } finally { + _whisperBusy = false; + } +} + +/* Local mic self-transcription. The host's own voice was not being + * transcribed because the previous code only attached capture nodes + * to the listenerAudioNodes map (which holds REMOTE speakers). + * micStream is the local mic — the same one we publish — so we wire + * a capture node to it too. Tagged with myUUID so the line shows + * the user's handle. */ +let _selfCapture = null; +async function startSelfCapture(){ + if (_selfCapture || !audioCtx || !micStream) return; + const ok = await loadWhisperCaptureWorklet(audioCtx); + if (!ok) return; + try { + const src = audioCtx.createMediaStreamSource(micStream); + const cap = new AudioWorkletNode(audioCtx, 'whisper-capture'); + cap.port.onmessage = (e) => { + if (e.data && e.data.chunk) handleWhisperChunk(myUUID, e.data.chunk); + }; + src.connect(cap); + cap.port.postMessage({ cmd: 'start' }); + _selfCapture = { src, cap }; + logLine('', 'whisper: self capture started'); + } catch (e){ + logLine('err', 'whisper self capture: '+e.message); + } +} +function stopSelfCapture(){ + if (!_selfCapture) return; + try { _selfCapture.cap.port.postMessage({ cmd: 'stop' }); } catch(_){} + try { _selfCapture.cap.disconnect(); } catch(_){} + try { _selfCapture.src.disconnect(); } catch(_){} + _selfCapture = null; +} + async function startCaptureForUuid(uuid){ const node = listenerAudioNodes.get(uuid); if (!node || !audioCtx || node.capture) return; @@ -2267,39 +2344,8 @@ async function startCaptureForUuid(uuid){ if (!ok) return; try { const capture = new AudioWorkletNode(audioCtx, 'whisper-capture'); - capture.port.onmessage = async (e) => { - if (!e.data || !e.data.chunk) return; - /* Inflight gate: only one transcription in flight at a time, - * globally. If a new chunk arrives while busy, DROP it — the - * worker is still chewing on something else and queueing - * would just stack stale audio. */ - if (_whisperBusy){ - _whisperDroppedChunks++; - const now = Date.now(); - if (now - _whisperLastDropLogAt > 30000){ - _whisperLastDropLogAt = now; - logLine('', 'whisper: dropped '+_whisperDroppedChunks+' chunks (worker saturated)'); - } - return; - } - _whisperBusy = true; - try { - const txt = (await transcribeViaWorker(e.data.chunk)).trim(); - if (txt.length < 2) return; - if (/^\[/.test(txt)) return; /* "[BLANK_AUDIO]" etc. */ - /* normalize for hallucination match: lowercase, strip trailing - * punctuation. "You." / "you" / " YOU!" all collapse to "you". */ - const norm = txt.toLowerCase().replace(/[.!?,;:\s]+$/,'').trim(); - if (WHISPER_HALLUCINATIONS.has(norm)) return; - /* consecutive-duplicate suppression per speaker */ - if (_lastTranscriptByUuid.get(uuid) === txt) return; - _lastTranscriptByUuid.set(uuid, txt); - appendTranscriptLine(uuid, txt); - } catch (err){ - logLine('err', 'whisper transcribe '+uuid.slice(0,4)+': '+err.message); - } finally { - _whisperBusy = false; - } + capture.port.onmessage = (e) => { + if (e.data && e.data.chunk) handleWhisperChunk(uuid, e.data.chunk); }; /* tap the source — capture runs in parallel with the worklet * chain, doesn't need to connect to destination (we just want @@ -2332,11 +2378,18 @@ async function toggleTranscribe(){ if (transcribeEnabled){ /* pre-warm the worker so the first chunk doesn't wait on model load. */ ensureWhisperWorker(); + /* Remote speakers */ for (const [uuid] of listenerAudioNodes){ startCaptureForUuid(uuid).catch(e => logLine('err', 'transcribe start '+uuid+': '+e.message)); } + /* Self (local mic). If the user is a listener with no mic yet, + * this is a no-op — startSelfCapture early-returns. When the + * user gets promoted and gets a mic, that path can also start + * self capture (see getMic / applyMicMode). */ + startSelfCapture().catch(e => logLine('err', 'self transcribe start: '+e.message)); } else { for (const [uuid] of listenerAudioNodes) stopCaptureForUuid(uuid); + stopSelfCapture(); } } function installJitterBuffer(uuid, node){ @@ -4526,6 +4579,11 @@ async function getMic(){ const t = micStream.getAudioTracks()[0]; tagTrack(t); await enforceMicConstraints(t); watchMicTrack(t); + /* If transcribe is already on (user enabled it before granting + * mic), wire self-capture now that micStream exists. */ + if (transcribeEnabled){ + startSelfCapture().catch(e => logLine('err','self transcribe start: '+e.message)); + } return micStream; } /* If the underlying mic device disappears (BT disconnect, USB unplug, OS @@ -4561,6 +4619,12 @@ async function reacquireMic(){ if (micStream) micStream.getTracks().forEach(t=>t.stop()); micStream = ns; if (myUUID){ stopMeter(myUUID); startMeter(myUUID, micStream); } + /* rebuild self-capture worklet against the fresh stream — the old + * MediaStreamAudioSourceNode was bound to the prior micStream. */ + if (transcribeEnabled){ + stopSelfCapture(); + startSelfCapture().catch(e => logLine('err','self transcribe re-start: '+e.message)); + } logLine('', 'mic re-acquired'); } finally { micReacquireInFlight = false; @@ -4741,6 +4805,12 @@ async function applyMicMode(){ } /* old analyser is now dead — rewire local meter against the fresh stream */ if (myUUID){ stopMeter(myUUID); startMeter(myUUID, micStream); } + /* rebuild self-capture against the new micStream so we don't keep + * transcribing the (now-stopped) old one. */ + if (transcribeEnabled){ + stopSelfCapture(); + startSelfCapture().catch(e => logLine('err','self transcribe re-start: '+e.message)); + } } /* per-peer meter: one analyser node + one rAF loop, keyed by uuid. The tick * reads members.get(uuid)._meterEl fresh each frame so renderRoom can replace @@ -5101,6 +5171,30 @@ async function dumpTelemetry(){ parts.push('sListen=' + (selfListenerMode ? '1' : '0')); parts.push('streamMode=' + streamMode.size); parts.push('muted=' + (muted ? '1' : '0')); + /* Whisper transcribe stats — per-tick rates (reset below), so we + * can see across devices: who has it on, how many chunks they're + * processing per tick, how many they're dropping (CPU saturated), + * and average per-chunk inference latency (proxy for device perf). + * Snapdragon-8 should show ~150-300ms; older phones 1500-3000ms; + * desktop x86 50-150ms. */ + if (transcribeEnabled){ + const avg = _whisperTick.sent > 0 ? Math.round(_whisperTick.totalLatencyMs / _whisperTick.sent) : 0; + const selfOn = _selfCapture ? 1 : 0; + const remoteOn = 0; let _r = 0; + for (const [, n] of listenerAudioNodes) if (n.capture) _r++; + parts.push('xcr=on sntCnt=' + remoteOn + ' selfCap=' + selfOn + + ' caps=' + _r + + ' sent=' + _whisperTick.sent + + ' drp=' + _whisperTick.dropped + + ' emt=' + _whisperTick.emitted + + ' avgMs=' + avg + + ' wrkr=' + (whisperWorkerReady ? '1' : '0')); + } else { + parts.push('xcr=off'); + } + /* reset per-tick whisper counters */ + _whisperTick.sent = 0; _whisperTick.dropped = 0; + _whisperTick.emitted = 0; _whisperTick.totalLatencyMs = 0; /* receiver stats from sfuSubPC (the main listener path) */ if (sfuSubPC && typeof sfuSubPC.getStats === 'function'){ try { @@ -7270,8 +7364,8 @@ logLine('', 'ready — pick a handle, type a rendezvous code, enter the space');