zebra-spaces: self-transcribe + Whisper telemetry stats

Two related additions:

1. Self-transcription. The host's own voice wasn't being transcribed
   because capture nodes only attached to listenerAudioNodes (remote
   speakers' streams from SFU/mesh worklets). micStream — the local
   capture for publishing — was never tapped. Added startSelfCapture
   / stopSelfCapture that wraps micStream in a parallel
   whisper-capture worklet. Lines are tagged with myUUID so the local
   user's handle shows in the transcript log.

   Wired into:
   - toggleTranscribe ON/OFF → starts/stops self capture alongside
     remote captures
   - getMic → starts self capture if transcribe is already on (covers
     "user enabled transcribe before granting mic permission")
   - applyMicMode / reacquireMic → stops + restarts self capture
     against the new micStream so we don't keep transcribing a
     stopped MediaStreamTrack

2. Whisper telemetry stats. Each 5s tick now appends:
   xcr=on  sent=N drp=N emt=N avgMs=N wrkr=1 caps=N selfCap=1
     - xcr: on/off
     - sent: chunks shipped to the worker this tick
     - drp:  chunks dropped by the inflight gate (worker saturated)
     - emt:  text lines emitted after silence/hallucination/dup filters
     - avgMs: average inference latency per chunk (device-perf proxy)
     - wrkr: 1 once the worker has loaded the model
     - caps: # of active remote-capture worklets
     - selfCap: 1 if local mic is being captured
   Reset each tick so we see RATE, not cumulative.

   Lets us compare devices across mesh from server-side signal log:
   Snapdragon-8 should show avgMs ~150-300; mid-range phones 1500-
   3000; desktop x86 50-150. drp > 0 means worker can't keep up with
   the audio chunks arriving (multiple speakers talking at once).

Factored the chunk-handling pipeline into handleWhisperChunk(uuid,
chunk) so both remote and self captures share the same filter +
inflight-gate + telemetry path.
This commit is contained in:
Russell Ballestrini 2026-06-05 13:21:49 -04:00
parent 9b91c726e5
commit e23808d410
No known key found for this signature in database

View file

@ -2235,6 +2235,14 @@ const _lastTranscriptByUuid = new Map();
let _whisperBusy = false; let _whisperBusy = false;
let _whisperDroppedChunks = 0; let _whisperDroppedChunks = 0;
let _whisperLastDropLogAt = 0; let _whisperLastDropLogAt = 0;
/* Per-tick counters surfaced in the telemetry line. Reset each tick
* so we see RATE, not cumulative. Lets us SEE from server logs which
* clients have transcribe on, who's saturating their CPU, and how
* each device's inference latency compares (Snapdragon-8 vs mid-
* range MediaTek, x86 desktop vs phone). Fox 2026-06-05: "no wasted
* computations we can also know if some devices perform worse or
* better across mesh." */
const _whisperTick = { sent: 0, dropped: 0, emitted: 0, totalLatencyMs: 0 };
function appendTranscriptLine(uuid, text){ function appendTranscriptLine(uuid, text){
const log = document.getElementById('transcript-log'); const log = document.getElementById('transcript-log');
@ -2260,21 +2268,14 @@ function appendTranscriptLine(uuid, text){
if (nearBottom) log.scrollTop = log.scrollHeight; if (nearBottom) log.scrollTop = log.scrollHeight;
} }
async function startCaptureForUuid(uuid){ /* Shared handler — used by both remote-speaker capture (one per uuid
const node = listenerAudioNodes.get(uuid); * in listenerAudioNodes) and self capture (one for the local mic).
if (!node || !audioCtx || node.capture) return; * Filters identical for both; uses the uuid to tag the transcript
const ok = await loadWhisperCaptureWorklet(audioCtx); * line so "you" vs "alice" appears correctly in the log. */
if (!ok) return; async function handleWhisperChunk(uuid, chunk){
try {
const capture = new AudioWorkletNode(audioCtx, 'whisper-capture');
capture.port.onmessage = async (e) => {
if (!e.data || !e.data.chunk) return;
/* Inflight gate: only one transcription in flight at a time,
* globally. If a new chunk arrives while busy, DROP it — the
* worker is still chewing on something else and queueing
* would just stack stale audio. */
if (_whisperBusy){ if (_whisperBusy){
_whisperDroppedChunks++; _whisperDroppedChunks++;
_whisperTick.dropped++;
const now = Date.now(); const now = Date.now();
if (now - _whisperLastDropLogAt > 30000){ if (now - _whisperLastDropLogAt > 30000){
_whisperLastDropLogAt = now; _whisperLastDropLogAt = now;
@ -2283,23 +2284,68 @@ async function startCaptureForUuid(uuid){
return; return;
} }
_whisperBusy = true; _whisperBusy = true;
_whisperTick.sent++;
const t0 = performance.now();
try { try {
const txt = (await transcribeViaWorker(e.data.chunk)).trim(); const txt = (await transcribeViaWorker(chunk)).trim();
_whisperTick.totalLatencyMs += (performance.now() - t0);
if (txt.length < 2) return; if (txt.length < 2) return;
if (/^\[/.test(txt)) return; /* "[BLANK_AUDIO]" etc. */ if (/^\[/.test(txt)) return;
/* normalize for hallucination match: lowercase, strip trailing
* punctuation. "You." / "you" / " YOU!" all collapse to "you". */
const norm = txt.toLowerCase().replace(/[.!?,;:\s]+$/,'').trim(); const norm = txt.toLowerCase().replace(/[.!?,;:\s]+$/,'').trim();
if (WHISPER_HALLUCINATIONS.has(norm)) return; if (WHISPER_HALLUCINATIONS.has(norm)) return;
/* consecutive-duplicate suppression per speaker */
if (_lastTranscriptByUuid.get(uuid) === txt) return; if (_lastTranscriptByUuid.get(uuid) === txt) return;
_lastTranscriptByUuid.set(uuid, txt); _lastTranscriptByUuid.set(uuid, txt);
appendTranscriptLine(uuid, txt); appendTranscriptLine(uuid, txt);
_whisperTick.emitted++;
} catch (err){ } catch (err){
logLine('err', 'whisper transcribe '+uuid.slice(0,4)+': '+err.message); logLine('err', 'whisper transcribe '+uuid.slice(0,4)+': '+err.message);
} finally { } finally {
_whisperBusy = false; _whisperBusy = false;
} }
}
/* Local mic self-transcription. The host's own voice was not being
* transcribed because the previous code only attached capture nodes
* to the listenerAudioNodes map (which holds REMOTE speakers).
* micStream is the local mic — the same one we publish — so we wire
* a capture node to it too. Tagged with myUUID so the line shows
* the user's handle. */
let _selfCapture = null;
async function startSelfCapture(){
if (_selfCapture || !audioCtx || !micStream) return;
const ok = await loadWhisperCaptureWorklet(audioCtx);
if (!ok) return;
try {
const src = audioCtx.createMediaStreamSource(micStream);
const cap = new AudioWorkletNode(audioCtx, 'whisper-capture');
cap.port.onmessage = (e) => {
if (e.data && e.data.chunk) handleWhisperChunk(myUUID, e.data.chunk);
};
src.connect(cap);
cap.port.postMessage({ cmd: 'start' });
_selfCapture = { src, cap };
logLine('', 'whisper: self capture started');
} catch (e){
logLine('err', 'whisper self capture: '+e.message);
}
}
function stopSelfCapture(){
if (!_selfCapture) return;
try { _selfCapture.cap.port.postMessage({ cmd: 'stop' }); } catch(_){}
try { _selfCapture.cap.disconnect(); } catch(_){}
try { _selfCapture.src.disconnect(); } catch(_){}
_selfCapture = null;
}
async function startCaptureForUuid(uuid){
const node = listenerAudioNodes.get(uuid);
if (!node || !audioCtx || node.capture) return;
const ok = await loadWhisperCaptureWorklet(audioCtx);
if (!ok) return;
try {
const capture = new AudioWorkletNode(audioCtx, 'whisper-capture');
capture.port.onmessage = (e) => {
if (e.data && e.data.chunk) handleWhisperChunk(uuid, e.data.chunk);
}; };
/* tap the source — capture runs in parallel with the worklet /* tap the source — capture runs in parallel with the worklet
* chain, doesn't need to connect to destination (we just want * chain, doesn't need to connect to destination (we just want
@ -2332,11 +2378,18 @@ async function toggleTranscribe(){
if (transcribeEnabled){ if (transcribeEnabled){
/* pre-warm the worker so the first chunk doesn't wait on model load. */ /* pre-warm the worker so the first chunk doesn't wait on model load. */
ensureWhisperWorker(); ensureWhisperWorker();
/* Remote speakers */
for (const [uuid] of listenerAudioNodes){ for (const [uuid] of listenerAudioNodes){
startCaptureForUuid(uuid).catch(e => logLine('err', 'transcribe start '+uuid+': '+e.message)); startCaptureForUuid(uuid).catch(e => logLine('err', 'transcribe start '+uuid+': '+e.message));
} }
/* Self (local mic). If the user is a listener with no mic yet,
* this is a no-op — startSelfCapture early-returns. When the
* user gets promoted and gets a mic, that path can also start
* self capture (see getMic / applyMicMode). */
startSelfCapture().catch(e => logLine('err', 'self transcribe start: '+e.message));
} else { } else {
for (const [uuid] of listenerAudioNodes) stopCaptureForUuid(uuid); for (const [uuid] of listenerAudioNodes) stopCaptureForUuid(uuid);
stopSelfCapture();
} }
} }
function installJitterBuffer(uuid, node){ function installJitterBuffer(uuid, node){
@ -4526,6 +4579,11 @@ async function getMic(){
const t = micStream.getAudioTracks()[0]; const t = micStream.getAudioTracks()[0];
tagTrack(t); await enforceMicConstraints(t); tagTrack(t); await enforceMicConstraints(t);
watchMicTrack(t); watchMicTrack(t);
/* If transcribe is already on (user enabled it before granting
* mic), wire self-capture now that micStream exists. */
if (transcribeEnabled){
startSelfCapture().catch(e => logLine('err','self transcribe start: '+e.message));
}
return micStream; return micStream;
} }
/* If the underlying mic device disappears (BT disconnect, USB unplug, OS /* If the underlying mic device disappears (BT disconnect, USB unplug, OS
@ -4561,6 +4619,12 @@ async function reacquireMic(){
if (micStream) micStream.getTracks().forEach(t=>t.stop()); if (micStream) micStream.getTracks().forEach(t=>t.stop());
micStream = ns; micStream = ns;
if (myUUID){ stopMeter(myUUID); startMeter(myUUID, micStream); } if (myUUID){ stopMeter(myUUID); startMeter(myUUID, micStream); }
/* rebuild self-capture worklet against the fresh stream — the old
* MediaStreamAudioSourceNode was bound to the prior micStream. */
if (transcribeEnabled){
stopSelfCapture();
startSelfCapture().catch(e => logLine('err','self transcribe re-start: '+e.message));
}
logLine('', 'mic re-acquired'); logLine('', 'mic re-acquired');
} finally { } finally {
micReacquireInFlight = false; micReacquireInFlight = false;
@ -4741,6 +4805,12 @@ async function applyMicMode(){
} }
/* old analyser is now dead — rewire local meter against the fresh stream */ /* old analyser is now dead — rewire local meter against the fresh stream */
if (myUUID){ stopMeter(myUUID); startMeter(myUUID, micStream); } if (myUUID){ stopMeter(myUUID); startMeter(myUUID, micStream); }
/* rebuild self-capture against the new micStream so we don't keep
* transcribing the (now-stopped) old one. */
if (transcribeEnabled){
stopSelfCapture();
startSelfCapture().catch(e => logLine('err','self transcribe re-start: '+e.message));
}
} }
/* per-peer meter: one analyser node + one rAF loop, keyed by uuid. The tick /* per-peer meter: one analyser node + one rAF loop, keyed by uuid. The tick
* reads members.get(uuid)._meterEl fresh each frame so renderRoom can replace * reads members.get(uuid)._meterEl fresh each frame so renderRoom can replace
@ -5101,6 +5171,30 @@ async function dumpTelemetry(){
parts.push('sListen=' + (selfListenerMode ? '1' : '0')); parts.push('sListen=' + (selfListenerMode ? '1' : '0'));
parts.push('streamMode=' + streamMode.size); parts.push('streamMode=' + streamMode.size);
parts.push('muted=' + (muted ? '1' : '0')); parts.push('muted=' + (muted ? '1' : '0'));
/* Whisper transcribe stats — per-tick rates (reset below), so we
* can see across devices: who has it on, how many chunks they're
* processing per tick, how many they're dropping (CPU saturated),
* and average per-chunk inference latency (proxy for device perf).
* Snapdragon-8 should show ~150-300ms; older phones 1500-3000ms;
* desktop x86 50-150ms. */
if (transcribeEnabled){
const avg = _whisperTick.sent > 0 ? Math.round(_whisperTick.totalLatencyMs / _whisperTick.sent) : 0;
const selfOn = _selfCapture ? 1 : 0;
const remoteOn = 0; let _r = 0;
for (const [, n] of listenerAudioNodes) if (n.capture) _r++;
parts.push('xcr=on sntCnt=' + remoteOn + ' selfCap=' + selfOn +
' caps=' + _r +
' sent=' + _whisperTick.sent +
' drp=' + _whisperTick.dropped +
' emt=' + _whisperTick.emitted +
' avgMs=' + avg +
' wrkr=' + (whisperWorkerReady ? '1' : '0'));
} else {
parts.push('xcr=off');
}
/* reset per-tick whisper counters */
_whisperTick.sent = 0; _whisperTick.dropped = 0;
_whisperTick.emitted = 0; _whisperTick.totalLatencyMs = 0;
/* receiver stats from sfuSubPC (the main listener path) */ /* receiver stats from sfuSubPC (the main listener path) */
if (sfuSubPC && typeof sfuSubPC.getStats === 'function'){ if (sfuSubPC && typeof sfuSubPC.getStats === 'function'){
try { try {
@ -7270,8 +7364,8 @@ logLine('', 'ready — pick a handle, type a rendezvous code, enter the space');
<footer style="margin:2.2rem auto 0;font-size:0.65rem;color:#999;line-height:1.7;word-break:break-all;font-family:monospace"> <footer style="margin:2.2rem auto 0;font-size:0.65rem;color:#999;line-height:1.7;word-break:break-all;font-family:monospace">
<span id="pi-seal" style="color:#777;cursor:default;user-select:none" title="">page integrity</span> &nbsp;·&nbsp; built <span class="stamp-date">2026-06-05</span><br> <span id="pi-seal" style="color:#777;cursor:default;user-select:none" title="">page integrity</span> &nbsp;·&nbsp; built <span class="stamp-date">2026-06-05</span><br>
md5 <span class="stamp-md5">3fc72ebbc854628de3e74c7e8b0d594a</span><br> md5 <span class="stamp-md5">6f6e30026fe83b275be830b2e135d1aa</span><br>
sha256 <span class="stamp-sha">8c71febc8071fa7ceebd037dc6ea6839c9da98736c1a62932d946e5ee040fe26</span><br> sha256 <span class="stamp-sha">ab24f0b7071a56ed9b14fa3b096f1f4d2345847ffeb5d8ac5e2d64dd45d38e9e</span><br>
<span style="color:#bbb">hashes are of this page with these two fields zeroed — to verify, blank them and re-hash</span><br> <span style="color:#bbb">hashes are of this page with these two fields zeroed — to verify, blank them and re-hash</span><br>
<span style="color:#bbb">one self-contained file — <strong>save a copy</strong> and verify against these hashes; point at your own servers with ?signal= and ?turncred=, or <a href="host-your-own.html" style="color:#999">host your own community</a></span> <span style="color:#bbb">one self-contained file — <strong>save a copy</strong> and verify against these hashes; point at your own servers with ?signal= and ?turncred=, or <a href="host-your-own.html" style="color:#999">host your own community</a></span>
</footer> </footer>