zebra-spaces: client-side Whisper STT — per-speaker live captions, off by default
Fox 2026-06-05: "do we access to a whisper speech to text?" → no, then
"yes implement this and please this is perfect, make it an off be
default toggle that is part of the client. individual speakers should
ues their names and show up like a log under the video area in middle."
Implementation:
1. whisper-capture AudioWorklet — separate from the jitter-buffer
worklet. Decimates the speaker's 48 kHz mono stream to 16 kHz
(Whisper's expected sample rate), batches 5-second chunks, ships
them to the main thread via transferable Float32Array on the
port. Disabled by default; { cmd: 'start' } / { cmd: 'stop' }
from JS gate the capture.
2. transformers.js + Xenova/whisper-tiny.en lazy-loaded from
jsdelivr CDN on first transcribe-toggle ON. ~40 MB one-time
download (cached by the browser); subsequent toggles are
instant. ONNX Runtime runs entirely client-side — no audio
leaves the listener's browser.
3. Per-uuid capture lifecycle. When transcribe is on, every speaker
in listenerAudioNodes gets a parallel AudioWorkletNode that taps
their source. Recognized text appends to a transcript log with
"HH:MM:SS name: text" lines. Names resolved via
members.get(uuid).handle. Empty / placeholder transcriptions
("." "[BLANK_AUDIO]") filtered out.
4. UI:
- Toggle button "transcribe (off/on)" in the controls column,
with explainer note about model size + privacy.
- #sec-transcript section in the timeline column directly under
the spotlight. Monospace font, 14rem max-height, scrollable,
auto-scrolls to bottom unless user is reading older lines.
- .hidden until first toggle ON; subsequent toggles show/hide.
5. New-speaker hook: attachAudioStreamViaWorklet checks
transcribeEnabled and auto-installs capture for late joiners.
6. detachListenerStream tears down node.capture along with the rest.
Bandwidth: zero (model and audio never leave the device).
CPU on listener: ~5-15% per speaker during the brief inference
window every 5s, idle otherwise. Tested mentally; needs real-world
verification.
This commit is contained in:
parent
54f69a3edb
commit
86df9622d1
1 changed files with 235 additions and 2 deletions
|
|
@ -183,6 +183,31 @@
|
|||
position: relative;
|
||||
}
|
||||
.controls { min-width: 0; }
|
||||
/* Transcript log — lives in the timeline column under the
|
||||
* spotlight. Off by default (the section is .hidden); toggled
|
||||
* visible via the controls-column transcribe button. Auto-scrolls
|
||||
* to bottom unless the user scrolls up to read older lines. */
|
||||
#sec-transcript {
|
||||
margin-top: 0.6rem; border: 1px solid #ddd; background: #fafafa;
|
||||
padding: 0.4rem 0.6rem; font-size: 0.85rem; min-width: 0;
|
||||
}
|
||||
html.theme-dark #sec-transcript { background: #0d0d0d; border-color: #333; }
|
||||
#transcript-log {
|
||||
max-height: 14rem; overflow-y: auto; line-height: 1.45;
|
||||
font-family: ui-monospace, monospace; white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
.transcript-line { padding: 0.1rem 0; border-bottom: 1px dotted #e5e5e5; }
|
||||
html.theme-dark .transcript-line { border-color: #222; }
|
||||
.transcript-line:last-child { border-bottom: none; }
|
||||
.transcript-line .ts { color: #888; font-size: 0.75rem; }
|
||||
.transcript-line .name { color: #050; font-weight: bold; }
|
||||
html.theme-dark .transcript-line .name { color: #6cc06c; }
|
||||
.transcript-line .txt { color: inherit; }
|
||||
/* Toggle button — same shape as other small buttons, with an
|
||||
* 'on' state so the user can see at a glance. */
|
||||
#btn-transcribe.on { background: #050; color: #fff; border-color: #050; }
|
||||
html.theme-dark #btn-transcribe.on { background: #6cc06c; color: #000; border-color: #6cc06c; }
|
||||
/* games live in the LEFT column under cameras + screens-thumbs as
|
||||
* proper tile-thumbs. Click one to spotlight it in the middle (an
|
||||
* iframe loads at full size); the click also broadcasts the spotlight
|
||||
|
|
@ -651,6 +676,12 @@ try {
|
|||
<section id="sec-spotlight" class="hidden">
|
||||
<div id="spotlight"></div>
|
||||
</section>
|
||||
<!-- transcript log: per-speaker live captions, off by default,
|
||||
toggled from the controls column. Hidden until the user opts in
|
||||
— no model download or audio capture until they ask for it. -->
|
||||
<section id="sec-transcript" class="hidden">
|
||||
<div id="transcript-log" aria-live="polite"></div>
|
||||
</section>
|
||||
</main>
|
||||
|
||||
<aside class="controls">
|
||||
|
|
@ -715,6 +746,10 @@ try {
|
|||
<div class="row" id="row-music-mode">
|
||||
<label class="note"><input type="checkbox" id="music-mode"> music mode — raw mic, no echo/noise cancellation (for playing audio through it)</label>
|
||||
</div>
|
||||
<div class="row">
|
||||
<button id="btn-transcribe" class="small">transcribe (off)</button>
|
||||
<span class="note">live captions per speaker, on-device whisper-tiny.en (~40 MB once, cached). nothing leaves your browser.</span>
|
||||
</div>
|
||||
<p class="note">join as listener; host promotes to mic. end-to-end encrypted.</p>
|
||||
</section>
|
||||
|
||||
|
|
@ -1581,6 +1616,7 @@ function renderIdentity(){
|
|||
}
|
||||
|
||||
$('btn-vault').addEventListener('click', () => $('vault-panel').classList.toggle('hidden'));
|
||||
$('btn-transcribe').addEventListener('click', () => toggleTranscribe().catch(e => logLine('err','transcribe toggle: '+e.message)));
|
||||
|
||||
/* log out — destructive: wipes Ed25519 + handle from localStorage and
|
||||
* generates a fresh identity. The booted/blocked window keys off the
|
||||
|
|
@ -1950,6 +1986,196 @@ function loadJitterWorklet(ctx){
|
|||
logLine('err', 'jitter-buffer worklet load: '+e.message+' — listener audio direct');
|
||||
});
|
||||
}
|
||||
|
||||
/* ==================================================================
|
||||
* Client-side speech-to-text — whisper-tiny.en via transformers.js
|
||||
*
|
||||
* Off by default. Toggleable in the UI. When on, every audible speaker
|
||||
* gets a parallel AudioWorklet capture node that decimates 48kHz mic
|
||||
* audio down to 16kHz mono, batches 5-second chunks, and ships them
|
||||
* to a Whisper pipeline running in the main thread. Recognized text
|
||||
* is appended to a transcript log under the spotlight as
|
||||
* "HH:MM:SS name: text".
|
||||
*
|
||||
* Privacy: model runs entirely in the listener's browser — no audio
|
||||
* leaves the device for transcription. First ON downloads ~40MB
|
||||
* model (cached afterward). Subsequent ON is instant.
|
||||
*
|
||||
* Fox 2026-06-05: "implement this and please this is perfect, make it
|
||||
* an off be default toggle that is part of the client. individual
|
||||
* speakers should use their names and show up like a log under the
|
||||
* video area in middle." */
|
||||
const WHISPER_CAPTURE_WORKLET_CODE = `
|
||||
class WhisperCaptureProcessor extends AudioWorkletProcessor {
|
||||
constructor(){
|
||||
super();
|
||||
this.enabled = false;
|
||||
/* 5 seconds at 16kHz target — sweet spot for whisper-tiny.en
|
||||
* accuracy without too much latency. */
|
||||
this.chunkTargetSamples = 16000 * 5;
|
||||
this.outRate = 16000;
|
||||
this.decimationFactor = Math.round(sampleRate / this.outRate);
|
||||
if (this.decimationFactor < 1) this.decimationFactor = 1;
|
||||
this.decimateCounter = 0;
|
||||
this.buf = new Float32Array(this.chunkTargetSamples + 4096);
|
||||
this.bufPos = 0;
|
||||
this.port.onmessage = (e) => {
|
||||
if (!e.data) return;
|
||||
if (e.data.cmd === 'start') this.enabled = true;
|
||||
else if (e.data.cmd === 'stop'){
|
||||
this.enabled = false;
|
||||
this.bufPos = 0;
|
||||
this.decimateCounter = 0;
|
||||
}
|
||||
};
|
||||
}
|
||||
process(inputs){
|
||||
if (!this.enabled) return true;
|
||||
const ch = inputs[0] && inputs[0][0];
|
||||
if (!ch || ch.length === 0) return true;
|
||||
for (let i = this.decimateCounter; i < ch.length; i += this.decimationFactor){
|
||||
if (this.bufPos < this.buf.length) this.buf[this.bufPos++] = ch[i];
|
||||
}
|
||||
this.decimateCounter = (this.decimateCounter + ch.length) % this.decimationFactor;
|
||||
if (this.bufPos >= this.chunkTargetSamples){
|
||||
const chunk = this.buf.slice(0, this.bufPos);
|
||||
this.port.postMessage({ chunk }, [chunk.buffer]);
|
||||
this.bufPos = 0;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('whisper-capture', WhisperCaptureProcessor);
|
||||
`;
|
||||
|
||||
let whisperCaptureWorkletReady = false;
|
||||
let whisperCaptureWorkletLoading = false;
|
||||
function loadWhisperCaptureWorklet(ctx){
|
||||
if (whisperCaptureWorkletReady || whisperCaptureWorkletLoading) return Promise.resolve(whisperCaptureWorkletReady);
|
||||
whisperCaptureWorkletLoading = true;
|
||||
const blob = new Blob([WHISPER_CAPTURE_WORKLET_CODE], { type: 'application/javascript' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
return ctx.audioWorklet.addModule(url).then(() => {
|
||||
URL.revokeObjectURL(url);
|
||||
whisperCaptureWorkletReady = true;
|
||||
whisperCaptureWorkletLoading = false;
|
||||
return true;
|
||||
}).catch(e => {
|
||||
URL.revokeObjectURL(url);
|
||||
whisperCaptureWorkletLoading = false;
|
||||
logLine('err', 'whisper-capture worklet load: '+e.message);
|
||||
return false;
|
||||
});
|
||||
}
|
||||
|
||||
let whisperPipeline = null;
|
||||
let whisperLoading = false;
|
||||
async function ensureWhisperPipeline(){
|
||||
if (whisperPipeline) return whisperPipeline;
|
||||
if (whisperLoading){
|
||||
while (whisperLoading && !whisperPipeline) await new Promise(r => setTimeout(r, 200));
|
||||
return whisperPipeline;
|
||||
}
|
||||
whisperLoading = true;
|
||||
logLine('', 'whisper: loading model (~40MB, first time only)…');
|
||||
try {
|
||||
/* transformers.js from jsDelivr — runs ONNX Runtime + whisper-tiny.en
|
||||
* fully in-browser. Model is downloaded once and cached by the
|
||||
* service worker / browser cache; subsequent toggles are instant. */
|
||||
const tx = await import('https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.0');
|
||||
whisperPipeline = await tx.pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');
|
||||
logLine('', 'whisper: model ready');
|
||||
} catch (e){
|
||||
logLine('err', 'whisper load failed: '+e.message);
|
||||
} finally {
|
||||
whisperLoading = false;
|
||||
}
|
||||
return whisperPipeline;
|
||||
}
|
||||
|
||||
let transcribeEnabled = false;
|
||||
function appendTranscriptLine(uuid, text){
|
||||
const log = document.getElementById('transcript-log');
|
||||
if (!log) return;
|
||||
const member = members.get(uuid);
|
||||
const name = (member && member.handle) ? member.handle : uuid.slice(0,4);
|
||||
const now = new Date();
|
||||
const ts = String(now.getHours()).padStart(2,'0')+':'+
|
||||
String(now.getMinutes()).padStart(2,'0')+':'+
|
||||
String(now.getSeconds()).padStart(2,'0');
|
||||
const line = document.createElement('div');
|
||||
line.className = 'transcript-line';
|
||||
const tsEl = document.createElement('span'); tsEl.className = 'ts'; tsEl.textContent = ts;
|
||||
const nameEl = document.createElement('span'); nameEl.className = 'name'; nameEl.textContent = name + ':';
|
||||
const txtEl = document.createElement('span'); txtEl.className = 'txt'; txtEl.textContent = text;
|
||||
line.appendChild(tsEl); line.append(' ');
|
||||
line.appendChild(nameEl); line.append(' ');
|
||||
line.appendChild(txtEl);
|
||||
log.appendChild(line);
|
||||
/* auto-scroll only if user is already at the bottom (let them scroll
|
||||
* up to read older transcript without being yanked back). */
|
||||
const nearBottom = (log.scrollTop + log.clientHeight) >= (log.scrollHeight - 40);
|
||||
if (nearBottom) log.scrollTop = log.scrollHeight;
|
||||
}
|
||||
|
||||
async function startCaptureForUuid(uuid){
|
||||
const node = listenerAudioNodes.get(uuid);
|
||||
if (!node || !audioCtx || node.capture) return;
|
||||
const ok = await loadWhisperCaptureWorklet(audioCtx);
|
||||
if (!ok) return;
|
||||
try {
|
||||
const capture = new AudioWorkletNode(audioCtx, 'whisper-capture');
|
||||
capture.port.onmessage = async (e) => {
|
||||
if (!e.data || !e.data.chunk) return;
|
||||
const pipeline = await ensureWhisperPipeline();
|
||||
if (!pipeline) return;
|
||||
try {
|
||||
const result = await pipeline(e.data.chunk, { sampling_rate: 16000 });
|
||||
const txt = (result && result.text ? result.text : '').trim();
|
||||
/* skip empty / placeholder transcriptions ("." " " "[BLANK_AUDIO]" etc.) */
|
||||
if (txt.length >= 2 && !/^\[/.test(txt)) appendTranscriptLine(uuid, txt);
|
||||
} catch (err){
|
||||
logLine('err', 'whisper transcribe '+uuid.slice(0,4)+': '+err.message);
|
||||
}
|
||||
};
|
||||
/* tap the source — capture runs in parallel with the worklet
|
||||
* chain, doesn't need to connect to destination (we just want
|
||||
* the chunks via port messages). */
|
||||
node.src.connect(capture);
|
||||
capture.port.postMessage({ cmd: 'start' });
|
||||
node.capture = capture;
|
||||
logLine('', 'whisper: capture started for '+uuid.slice(0,4));
|
||||
} catch (e){
|
||||
logLine('err', 'whisper capture install '+uuid.slice(0,4)+': '+e.message);
|
||||
}
|
||||
}
|
||||
|
||||
function stopCaptureForUuid(uuid){
|
||||
const node = listenerAudioNodes.get(uuid);
|
||||
if (!node || !node.capture) return;
|
||||
try { node.capture.port.postMessage({ cmd: 'stop' }); } catch(_){}
|
||||
try { node.capture.disconnect(); } catch(_){}
|
||||
try { node.src.disconnect(node.capture); } catch(_){}
|
||||
node.capture = null;
|
||||
}
|
||||
|
||||
async function toggleTranscribe(){
|
||||
transcribeEnabled = !transcribeEnabled;
|
||||
const btn = document.getElementById('btn-transcribe');
|
||||
const sec = document.getElementById('sec-transcript');
|
||||
if (btn) btn.classList.toggle('on', transcribeEnabled);
|
||||
if (btn) btn.textContent = transcribeEnabled ? 'transcribe (on)' : 'transcribe (off)';
|
||||
if (sec) sec.classList.toggle('hidden', !transcribeEnabled);
|
||||
if (transcribeEnabled){
|
||||
/* pre-warm the pipeline so the first chunk doesn't wait. */
|
||||
ensureWhisperPipeline().catch(()=>{});
|
||||
for (const [uuid] of listenerAudioNodes){
|
||||
startCaptureForUuid(uuid).catch(e => logLine('err', 'transcribe start '+uuid+': '+e.message));
|
||||
}
|
||||
} else {
|
||||
for (const [uuid] of listenerAudioNodes) stopCaptureForUuid(uuid);
|
||||
}
|
||||
}
|
||||
function installJitterBuffer(uuid, node){
|
||||
if (!node || node.jbuf || !workletReady) return;
|
||||
const target = node.targetSeconds || RECV_PLAYOUT_DELAY_SEC;
|
||||
|
|
@ -2426,6 +2652,12 @@ function attachAudioStreamViaWorklet(uuid, stream, targetSeconds){
|
|||
* existing stream is swapped through the buffer (see loadJitterWorklet) */
|
||||
loadJitterWorklet(audioCtx);
|
||||
if (workletReady) installJitterBuffer(uuid, node);
|
||||
/* If transcribe is on, install a parallel capture node for this
|
||||
* new speaker so their voice immediately starts flowing into the
|
||||
* transcript log. */
|
||||
if (transcribeEnabled){
|
||||
startCaptureForUuid(uuid).catch(e => logLine('err', 'transcribe '+uuid.slice(0,4)+': '+e.message));
|
||||
}
|
||||
logLine('', 'audio via AudioContext '+uuid.slice(0,4)+' target='+targetSeconds+'s ctxState='+audioCtx.state);
|
||||
return true;
|
||||
}
|
||||
|
|
@ -2467,6 +2699,7 @@ function detachListenerStream(uuid){
|
|||
if (!node) return;
|
||||
try { node.src.disconnect(); } catch(_){}
|
||||
try { if (node.jbuf) node.jbuf.disconnect(); } catch(_){}
|
||||
try { if (node.capture) node.capture.disconnect(); } catch(_){}
|
||||
try { node.gain.disconnect(); } catch(_){}
|
||||
listenerAudioNodes.delete(uuid);
|
||||
}
|
||||
|
|
@ -6848,8 +7081,8 @@ logLine('', 'ready — pick a handle, type a rendezvous code, enter the space');
|
|||
|
||||
<footer style="margin:2.2rem auto 0;font-size:0.65rem;color:#999;line-height:1.7;word-break:break-all;font-family:monospace">
|
||||
<span id="pi-seal" style="color:#777;cursor:default;user-select:none" title="">page integrity</span> · built <span class="stamp-date">2026-06-05</span><br>
|
||||
md5 <span class="stamp-md5">9728607e6d8187fed0bfa947f93cccaa</span><br>
|
||||
sha256 <span class="stamp-sha">26680e413131920d5c0dc981b20460b1c087d08193f1f5d89925d22e3013ec11</span><br>
|
||||
md5 <span class="stamp-md5">4591b92519ec28db69de15bb20eb2300</span><br>
|
||||
sha256 <span class="stamp-sha">0b792e6dd0d9bc69f888c239468c57506bc1d0feb9ded8b78bb650b886501c9b</span><br>
|
||||
<span style="color:#bbb">hashes are of this page with these two fields zeroed — to verify, blank them and re-hash</span><br>
|
||||
<span style="color:#bbb">one self-contained file — <strong>save a copy</strong> and verify against these hashes; point at your own servers with ?signal= and ?turncred=, or <a href="host-your-own.html" style="color:#999">host your own community</a></span>
|
||||
</footer>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue