zebra-spaces: self-transcribe + Whisper telemetry stats
Two related additions:
1. Self-transcription. The host's own voice wasn't being transcribed
because capture nodes only attached to listenerAudioNodes (remote
speakers' streams from SFU/mesh worklets). micStream — the local
capture for publishing — was never tapped. Added startSelfCapture
/ stopSelfCapture that wraps micStream in a parallel
whisper-capture worklet. Lines are tagged with myUUID so the local
user's handle shows in the transcript log.
Wired into:
- toggleTranscribe ON/OFF → starts/stops self capture alongside
remote captures
- getMic → starts self capture if transcribe is already on (covers
"user enabled transcribe before granting mic permission")
- applyMicMode / reacquireMic → stops + restarts self capture
against the new micStream so we don't keep transcribing a
stopped MediaStreamTrack
2. Whisper telemetry stats. Each 5s tick now appends:
xcr=on sent=N drp=N emt=N avgMs=N wrkr=1 caps=N selfCap=1
- xcr: on/off
- sent: chunks shipped to the worker this tick
- drp: chunks dropped by the inflight gate (worker saturated)
- emt: text lines emitted after silence/hallucination/dup filters
- avgMs: average inference latency per chunk (device-perf proxy)
- wrkr: 1 once the worker has loaded the model
- caps: # of active remote-capture worklets
- selfCap: 1 if local mic is being captured
Reset each tick so we see RATE, not cumulative.
Lets us compare devices across mesh from server-side signal log:
Snapdragon-8 should show avgMs ~150-300; mid-range phones 1500-
3000; desktop x86 50-150. drp > 0 means worker can't keep up with
the audio chunks arriving (multiple speakers talking at once).
Factored the chunk-handling pipeline into handleWhisperChunk(uuid,
chunk) so both remote and self captures share the same filter +
inflight-gate + telemetry path.
This commit is contained in:
parent
9b91c726e5
commit
e23808d410
1 changed files with 129 additions and 35 deletions
|
|
@ -2235,6 +2235,14 @@ const _lastTranscriptByUuid = new Map();
|
||||||
let _whisperBusy = false;
|
let _whisperBusy = false;
|
||||||
let _whisperDroppedChunks = 0;
|
let _whisperDroppedChunks = 0;
|
||||||
let _whisperLastDropLogAt = 0;
|
let _whisperLastDropLogAt = 0;
|
||||||
|
/* Per-tick counters surfaced in the telemetry line. Reset each tick
|
||||||
|
* so we see RATE, not cumulative. Lets us SEE from server logs which
|
||||||
|
* clients have transcribe on, who's saturating their CPU, and how
|
||||||
|
* each device's inference latency compares (Snapdragon-8 vs mid-
|
||||||
|
* range MediaTek, x86 desktop vs phone). Fox 2026-06-05: "no wasted
|
||||||
|
* computations we can also know if some devices perform worse or
|
||||||
|
* better across mesh." */
|
||||||
|
const _whisperTick = { sent: 0, dropped: 0, emitted: 0, totalLatencyMs: 0 };
|
||||||
|
|
||||||
function appendTranscriptLine(uuid, text){
|
function appendTranscriptLine(uuid, text){
|
||||||
const log = document.getElementById('transcript-log');
|
const log = document.getElementById('transcript-log');
|
||||||
|
|
@ -2260,21 +2268,14 @@ function appendTranscriptLine(uuid, text){
|
||||||
if (nearBottom) log.scrollTop = log.scrollHeight;
|
if (nearBottom) log.scrollTop = log.scrollHeight;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function startCaptureForUuid(uuid){
|
/* Shared handler — used by both remote-speaker capture (one per uuid
|
||||||
const node = listenerAudioNodes.get(uuid);
|
* in listenerAudioNodes) and self capture (one for the local mic).
|
||||||
if (!node || !audioCtx || node.capture) return;
|
* Filters identical for both; uses the uuid to tag the transcript
|
||||||
const ok = await loadWhisperCaptureWorklet(audioCtx);
|
* line so "you" vs "alice" appears correctly in the log. */
|
||||||
if (!ok) return;
|
async function handleWhisperChunk(uuid, chunk){
|
||||||
try {
|
|
||||||
const capture = new AudioWorkletNode(audioCtx, 'whisper-capture');
|
|
||||||
capture.port.onmessage = async (e) => {
|
|
||||||
if (!e.data || !e.data.chunk) return;
|
|
||||||
/* Inflight gate: only one transcription in flight at a time,
|
|
||||||
* globally. If a new chunk arrives while busy, DROP it — the
|
|
||||||
* worker is still chewing on something else and queueing
|
|
||||||
* would just stack stale audio. */
|
|
||||||
if (_whisperBusy){
|
if (_whisperBusy){
|
||||||
_whisperDroppedChunks++;
|
_whisperDroppedChunks++;
|
||||||
|
_whisperTick.dropped++;
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
if (now - _whisperLastDropLogAt > 30000){
|
if (now - _whisperLastDropLogAt > 30000){
|
||||||
_whisperLastDropLogAt = now;
|
_whisperLastDropLogAt = now;
|
||||||
|
|
@ -2283,23 +2284,68 @@ async function startCaptureForUuid(uuid){
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
_whisperBusy = true;
|
_whisperBusy = true;
|
||||||
|
_whisperTick.sent++;
|
||||||
|
const t0 = performance.now();
|
||||||
try {
|
try {
|
||||||
const txt = (await transcribeViaWorker(e.data.chunk)).trim();
|
const txt = (await transcribeViaWorker(chunk)).trim();
|
||||||
|
_whisperTick.totalLatencyMs += (performance.now() - t0);
|
||||||
if (txt.length < 2) return;
|
if (txt.length < 2) return;
|
||||||
if (/^\[/.test(txt)) return; /* "[BLANK_AUDIO]" etc. */
|
if (/^\[/.test(txt)) return;
|
||||||
/* normalize for hallucination match: lowercase, strip trailing
|
|
||||||
* punctuation. "You." / "you" / " YOU!" all collapse to "you". */
|
|
||||||
const norm = txt.toLowerCase().replace(/[.!?,;:\s]+$/,'').trim();
|
const norm = txt.toLowerCase().replace(/[.!?,;:\s]+$/,'').trim();
|
||||||
if (WHISPER_HALLUCINATIONS.has(norm)) return;
|
if (WHISPER_HALLUCINATIONS.has(norm)) return;
|
||||||
/* consecutive-duplicate suppression per speaker */
|
|
||||||
if (_lastTranscriptByUuid.get(uuid) === txt) return;
|
if (_lastTranscriptByUuid.get(uuid) === txt) return;
|
||||||
_lastTranscriptByUuid.set(uuid, txt);
|
_lastTranscriptByUuid.set(uuid, txt);
|
||||||
appendTranscriptLine(uuid, txt);
|
appendTranscriptLine(uuid, txt);
|
||||||
|
_whisperTick.emitted++;
|
||||||
} catch (err){
|
} catch (err){
|
||||||
logLine('err', 'whisper transcribe '+uuid.slice(0,4)+': '+err.message);
|
logLine('err', 'whisper transcribe '+uuid.slice(0,4)+': '+err.message);
|
||||||
} finally {
|
} finally {
|
||||||
_whisperBusy = false;
|
_whisperBusy = false;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Local mic self-transcription. The host's own voice was not being
|
||||||
|
* transcribed because the previous code only attached capture nodes
|
||||||
|
* to the listenerAudioNodes map (which holds REMOTE speakers).
|
||||||
|
* micStream is the local mic — the same one we publish — so we wire
|
||||||
|
* a capture node to it too. Tagged with myUUID so the line shows
|
||||||
|
* the user's handle. */
|
||||||
|
let _selfCapture = null;
|
||||||
|
async function startSelfCapture(){
|
||||||
|
if (_selfCapture || !audioCtx || !micStream) return;
|
||||||
|
const ok = await loadWhisperCaptureWorklet(audioCtx);
|
||||||
|
if (!ok) return;
|
||||||
|
try {
|
||||||
|
const src = audioCtx.createMediaStreamSource(micStream);
|
||||||
|
const cap = new AudioWorkletNode(audioCtx, 'whisper-capture');
|
||||||
|
cap.port.onmessage = (e) => {
|
||||||
|
if (e.data && e.data.chunk) handleWhisperChunk(myUUID, e.data.chunk);
|
||||||
|
};
|
||||||
|
src.connect(cap);
|
||||||
|
cap.port.postMessage({ cmd: 'start' });
|
||||||
|
_selfCapture = { src, cap };
|
||||||
|
logLine('', 'whisper: self capture started');
|
||||||
|
} catch (e){
|
||||||
|
logLine('err', 'whisper self capture: '+e.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
function stopSelfCapture(){
|
||||||
|
if (!_selfCapture) return;
|
||||||
|
try { _selfCapture.cap.port.postMessage({ cmd: 'stop' }); } catch(_){}
|
||||||
|
try { _selfCapture.cap.disconnect(); } catch(_){}
|
||||||
|
try { _selfCapture.src.disconnect(); } catch(_){}
|
||||||
|
_selfCapture = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function startCaptureForUuid(uuid){
|
||||||
|
const node = listenerAudioNodes.get(uuid);
|
||||||
|
if (!node || !audioCtx || node.capture) return;
|
||||||
|
const ok = await loadWhisperCaptureWorklet(audioCtx);
|
||||||
|
if (!ok) return;
|
||||||
|
try {
|
||||||
|
const capture = new AudioWorkletNode(audioCtx, 'whisper-capture');
|
||||||
|
capture.port.onmessage = (e) => {
|
||||||
|
if (e.data && e.data.chunk) handleWhisperChunk(uuid, e.data.chunk);
|
||||||
};
|
};
|
||||||
/* tap the source — capture runs in parallel with the worklet
|
/* tap the source — capture runs in parallel with the worklet
|
||||||
* chain, doesn't need to connect to destination (we just want
|
* chain, doesn't need to connect to destination (we just want
|
||||||
|
|
@ -2332,11 +2378,18 @@ async function toggleTranscribe(){
|
||||||
if (transcribeEnabled){
|
if (transcribeEnabled){
|
||||||
/* pre-warm the worker so the first chunk doesn't wait on model load. */
|
/* pre-warm the worker so the first chunk doesn't wait on model load. */
|
||||||
ensureWhisperWorker();
|
ensureWhisperWorker();
|
||||||
|
/* Remote speakers */
|
||||||
for (const [uuid] of listenerAudioNodes){
|
for (const [uuid] of listenerAudioNodes){
|
||||||
startCaptureForUuid(uuid).catch(e => logLine('err', 'transcribe start '+uuid+': '+e.message));
|
startCaptureForUuid(uuid).catch(e => logLine('err', 'transcribe start '+uuid+': '+e.message));
|
||||||
}
|
}
|
||||||
|
/* Self (local mic). If the user is a listener with no mic yet,
|
||||||
|
* this is a no-op — startSelfCapture early-returns. When the
|
||||||
|
* user gets promoted and gets a mic, that path can also start
|
||||||
|
* self capture (see getMic / applyMicMode). */
|
||||||
|
startSelfCapture().catch(e => logLine('err', 'self transcribe start: '+e.message));
|
||||||
} else {
|
} else {
|
||||||
for (const [uuid] of listenerAudioNodes) stopCaptureForUuid(uuid);
|
for (const [uuid] of listenerAudioNodes) stopCaptureForUuid(uuid);
|
||||||
|
stopSelfCapture();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
function installJitterBuffer(uuid, node){
|
function installJitterBuffer(uuid, node){
|
||||||
|
|
@ -4526,6 +4579,11 @@ async function getMic(){
|
||||||
const t = micStream.getAudioTracks()[0];
|
const t = micStream.getAudioTracks()[0];
|
||||||
tagTrack(t); await enforceMicConstraints(t);
|
tagTrack(t); await enforceMicConstraints(t);
|
||||||
watchMicTrack(t);
|
watchMicTrack(t);
|
||||||
|
/* If transcribe is already on (user enabled it before granting
|
||||||
|
* mic), wire self-capture now that micStream exists. */
|
||||||
|
if (transcribeEnabled){
|
||||||
|
startSelfCapture().catch(e => logLine('err','self transcribe start: '+e.message));
|
||||||
|
}
|
||||||
return micStream;
|
return micStream;
|
||||||
}
|
}
|
||||||
/* If the underlying mic device disappears (BT disconnect, USB unplug, OS
|
/* If the underlying mic device disappears (BT disconnect, USB unplug, OS
|
||||||
|
|
@ -4561,6 +4619,12 @@ async function reacquireMic(){
|
||||||
if (micStream) micStream.getTracks().forEach(t=>t.stop());
|
if (micStream) micStream.getTracks().forEach(t=>t.stop());
|
||||||
micStream = ns;
|
micStream = ns;
|
||||||
if (myUUID){ stopMeter(myUUID); startMeter(myUUID, micStream); }
|
if (myUUID){ stopMeter(myUUID); startMeter(myUUID, micStream); }
|
||||||
|
/* rebuild self-capture worklet against the fresh stream — the old
|
||||||
|
* MediaStreamAudioSourceNode was bound to the prior micStream. */
|
||||||
|
if (transcribeEnabled){
|
||||||
|
stopSelfCapture();
|
||||||
|
startSelfCapture().catch(e => logLine('err','self transcribe re-start: '+e.message));
|
||||||
|
}
|
||||||
logLine('', 'mic re-acquired');
|
logLine('', 'mic re-acquired');
|
||||||
} finally {
|
} finally {
|
||||||
micReacquireInFlight = false;
|
micReacquireInFlight = false;
|
||||||
|
|
@ -4741,6 +4805,12 @@ async function applyMicMode(){
|
||||||
}
|
}
|
||||||
/* old analyser is now dead — rewire local meter against the fresh stream */
|
/* old analyser is now dead — rewire local meter against the fresh stream */
|
||||||
if (myUUID){ stopMeter(myUUID); startMeter(myUUID, micStream); }
|
if (myUUID){ stopMeter(myUUID); startMeter(myUUID, micStream); }
|
||||||
|
/* rebuild self-capture against the new micStream so we don't keep
|
||||||
|
* transcribing the (now-stopped) old one. */
|
||||||
|
if (transcribeEnabled){
|
||||||
|
stopSelfCapture();
|
||||||
|
startSelfCapture().catch(e => logLine('err','self transcribe re-start: '+e.message));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
/* per-peer meter: one analyser node + one rAF loop, keyed by uuid. The tick
|
/* per-peer meter: one analyser node + one rAF loop, keyed by uuid. The tick
|
||||||
* reads members.get(uuid)._meterEl fresh each frame so renderRoom can replace
|
* reads members.get(uuid)._meterEl fresh each frame so renderRoom can replace
|
||||||
|
|
@ -5101,6 +5171,30 @@ async function dumpTelemetry(){
|
||||||
parts.push('sListen=' + (selfListenerMode ? '1' : '0'));
|
parts.push('sListen=' + (selfListenerMode ? '1' : '0'));
|
||||||
parts.push('streamMode=' + streamMode.size);
|
parts.push('streamMode=' + streamMode.size);
|
||||||
parts.push('muted=' + (muted ? '1' : '0'));
|
parts.push('muted=' + (muted ? '1' : '0'));
|
||||||
|
/* Whisper transcribe stats — per-tick rates (reset below), so we
|
||||||
|
* can see across devices: who has it on, how many chunks they're
|
||||||
|
* processing per tick, how many they're dropping (CPU saturated),
|
||||||
|
* and average per-chunk inference latency (proxy for device perf).
|
||||||
|
* Snapdragon-8 should show ~150-300ms; older phones 1500-3000ms;
|
||||||
|
* desktop x86 50-150ms. */
|
||||||
|
if (transcribeEnabled){
|
||||||
|
const avg = _whisperTick.sent > 0 ? Math.round(_whisperTick.totalLatencyMs / _whisperTick.sent) : 0;
|
||||||
|
const selfOn = _selfCapture ? 1 : 0;
|
||||||
|
const remoteOn = 0; let _r = 0;
|
||||||
|
for (const [, n] of listenerAudioNodes) if (n.capture) _r++;
|
||||||
|
parts.push('xcr=on sntCnt=' + remoteOn + ' selfCap=' + selfOn +
|
||||||
|
' caps=' + _r +
|
||||||
|
' sent=' + _whisperTick.sent +
|
||||||
|
' drp=' + _whisperTick.dropped +
|
||||||
|
' emt=' + _whisperTick.emitted +
|
||||||
|
' avgMs=' + avg +
|
||||||
|
' wrkr=' + (whisperWorkerReady ? '1' : '0'));
|
||||||
|
} else {
|
||||||
|
parts.push('xcr=off');
|
||||||
|
}
|
||||||
|
/* reset per-tick whisper counters */
|
||||||
|
_whisperTick.sent = 0; _whisperTick.dropped = 0;
|
||||||
|
_whisperTick.emitted = 0; _whisperTick.totalLatencyMs = 0;
|
||||||
/* receiver stats from sfuSubPC (the main listener path) */
|
/* receiver stats from sfuSubPC (the main listener path) */
|
||||||
if (sfuSubPC && typeof sfuSubPC.getStats === 'function'){
|
if (sfuSubPC && typeof sfuSubPC.getStats === 'function'){
|
||||||
try {
|
try {
|
||||||
|
|
@ -7270,8 +7364,8 @@ logLine('', 'ready — pick a handle, type a rendezvous code, enter the space');
|
||||||
|
|
||||||
<footer style="margin:2.2rem auto 0;font-size:0.65rem;color:#999;line-height:1.7;word-break:break-all;font-family:monospace">
|
<footer style="margin:2.2rem auto 0;font-size:0.65rem;color:#999;line-height:1.7;word-break:break-all;font-family:monospace">
|
||||||
<span id="pi-seal" style="color:#777;cursor:default;user-select:none" title="">page integrity</span> · built <span class="stamp-date">2026-06-05</span><br>
|
<span id="pi-seal" style="color:#777;cursor:default;user-select:none" title="">page integrity</span> · built <span class="stamp-date">2026-06-05</span><br>
|
||||||
md5 <span class="stamp-md5">3fc72ebbc854628de3e74c7e8b0d594a</span><br>
|
md5 <span class="stamp-md5">6f6e30026fe83b275be830b2e135d1aa</span><br>
|
||||||
sha256 <span class="stamp-sha">8c71febc8071fa7ceebd037dc6ea6839c9da98736c1a62932d946e5ee040fe26</span><br>
|
sha256 <span class="stamp-sha">ab24f0b7071a56ed9b14fa3b096f1f4d2345847ffeb5d8ac5e2d64dd45d38e9e</span><br>
|
||||||
<span style="color:#bbb">hashes are of this page with these two fields zeroed — to verify, blank them and re-hash</span><br>
|
<span style="color:#bbb">hashes are of this page with these two fields zeroed — to verify, blank them and re-hash</span><br>
|
||||||
<span style="color:#bbb">one self-contained file — <strong>save a copy</strong> and verify against these hashes; point at your own servers with ?signal= and ?turncred=, or <a href="host-your-own.html" style="color:#999">host your own community</a></span>
|
<span style="color:#bbb">one self-contained file — <strong>save a copy</strong> and verify against these hashes; point at your own servers with ?signal= and ?turncred=, or <a href="host-your-own.html" style="color:#999">host your own community</a></span>
|
||||||
</footer>
|
</footer>
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue