zebra-spaces: deploy client-side Whisper STT
This commit is contained in:
parent
f23b35af2f
commit
ab3b48e293
1 changed files with 235 additions and 2 deletions
|
|
@ -183,6 +183,31 @@
|
|||
position: relative;
|
||||
}
|
||||
.controls { min-width: 0; }
|
||||
/* Transcript log — lives in the timeline column under the
|
||||
* spotlight. Off by default (the section is .hidden); toggled
|
||||
* visible via the controls-column transcribe button. Auto-scrolls
|
||||
* to bottom unless the user scrolls up to read older lines. */
|
||||
#sec-transcript {
|
||||
margin-top: 0.6rem; border: 1px solid #ddd; background: #fafafa;
|
||||
padding: 0.4rem 0.6rem; font-size: 0.85rem; min-width: 0;
|
||||
}
|
||||
html.theme-dark #sec-transcript { background: #0d0d0d; border-color: #333; }
|
||||
#transcript-log {
|
||||
max-height: 14rem; overflow-y: auto; line-height: 1.45;
|
||||
font-family: ui-monospace, monospace; white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
.transcript-line { padding: 0.1rem 0; border-bottom: 1px dotted #e5e5e5; }
|
||||
html.theme-dark .transcript-line { border-color: #222; }
|
||||
.transcript-line:last-child { border-bottom: none; }
|
||||
.transcript-line .ts { color: #888; font-size: 0.75rem; }
|
||||
.transcript-line .name { color: #050; font-weight: bold; }
|
||||
html.theme-dark .transcript-line .name { color: #6cc06c; }
|
||||
.transcript-line .txt { color: inherit; }
|
||||
/* Toggle button — same shape as other small buttons, with an
|
||||
* 'on' state so the user can see at a glance. */
|
||||
#btn-transcribe.on { background: #050; color: #fff; border-color: #050; }
|
||||
html.theme-dark #btn-transcribe.on { background: #6cc06c; color: #000; border-color: #6cc06c; }
|
||||
/* games live in the LEFT column under cameras + screens-thumbs as
|
||||
* proper tile-thumbs. Click one to spotlight it in the middle (an
|
||||
* iframe loads at full size); the click also broadcasts the spotlight
|
||||
|
|
@ -651,6 +676,12 @@ try {
|
|||
<section id="sec-spotlight" class="hidden">
|
||||
<div id="spotlight"></div>
|
||||
</section>
|
||||
<!-- transcript log: per-speaker live captions, off by default,
|
||||
toggled from the controls column. Hidden until the user opts in
|
||||
— no model download or audio capture until they ask for it. -->
|
||||
<section id="sec-transcript" class="hidden">
|
||||
<div id="transcript-log" aria-live="polite"></div>
|
||||
</section>
|
||||
</main>
|
||||
|
||||
<aside class="controls">
|
||||
|
|
@ -715,6 +746,10 @@ try {
|
|||
<div class="row" id="row-music-mode">
|
||||
<label class="note"><input type="checkbox" id="music-mode"> music mode — raw mic, no echo/noise cancellation (for playing audio through it)</label>
|
||||
</div>
|
||||
<div class="row">
|
||||
<button id="btn-transcribe" class="small">transcribe (off)</button>
|
||||
<span class="note">live captions per speaker, on-device whisper-tiny.en (~40 MB once, cached). nothing leaves your browser.</span>
|
||||
</div>
|
||||
<p class="note">join as listener; host promotes to mic. end-to-end encrypted.</p>
|
||||
</section>
|
||||
|
||||
|
|
@ -1581,6 +1616,7 @@ function renderIdentity(){
|
|||
}
|
||||
|
||||
$('btn-vault').addEventListener('click', () => $('vault-panel').classList.toggle('hidden'));
|
||||
$('btn-transcribe').addEventListener('click', () => toggleTranscribe().catch(e => logLine('err','transcribe toggle: '+e.message)));
|
||||
|
||||
/* log out — destructive: wipes Ed25519 + handle from localStorage and
|
||||
* generates a fresh identity. The booted/blocked window keys off the
|
||||
|
|
@ -1950,6 +1986,196 @@ function loadJitterWorklet(ctx){
|
|||
logLine('err', 'jitter-buffer worklet load: '+e.message+' — listener audio direct');
|
||||
});
|
||||
}
|
||||
|
||||
/* ==================================================================
|
||||
* Client-side speech-to-text — whisper-tiny.en via transformers.js
|
||||
*
|
||||
* Off by default. Toggleable in the UI. When on, every audible speaker
|
||||
* gets a parallel AudioWorklet capture node that decimates 48kHz mic
|
||||
* audio down to 16kHz mono, batches 5-second chunks, and ships them
|
||||
* to a Whisper pipeline running in the main thread. Recognized text
|
||||
* is appended to a transcript log under the spotlight as
|
||||
* "HH:MM:SS name: text".
|
||||
*
|
||||
* Privacy: model runs entirely in the listener's browser — no audio
|
||||
* leaves the device for transcription. First ON downloads ~40MB
|
||||
* model (cached afterward). Subsequent ON is instant.
|
||||
*
|
||||
* Fox 2026-06-05: "implement this and please this is perfect, make it
|
||||
* an off be default toggle that is part of the client. individual
|
||||
* speakers should use their names and show up like a log under the
|
||||
* video area in middle." */
|
||||
const WHISPER_CAPTURE_WORKLET_CODE = `
|
||||
class WhisperCaptureProcessor extends AudioWorkletProcessor {
|
||||
constructor(){
|
||||
super();
|
||||
this.enabled = false;
|
||||
/* 5 seconds at 16kHz target — sweet spot for whisper-tiny.en
|
||||
* accuracy without too much latency. */
|
||||
this.chunkTargetSamples = 16000 * 5;
|
||||
this.outRate = 16000;
|
||||
this.decimationFactor = Math.round(sampleRate / this.outRate);
|
||||
if (this.decimationFactor < 1) this.decimationFactor = 1;
|
||||
this.decimateCounter = 0;
|
||||
this.buf = new Float32Array(this.chunkTargetSamples + 4096);
|
||||
this.bufPos = 0;
|
||||
this.port.onmessage = (e) => {
|
||||
if (!e.data) return;
|
||||
if (e.data.cmd === 'start') this.enabled = true;
|
||||
else if (e.data.cmd === 'stop'){
|
||||
this.enabled = false;
|
||||
this.bufPos = 0;
|
||||
this.decimateCounter = 0;
|
||||
}
|
||||
};
|
||||
}
|
||||
process(inputs){
|
||||
if (!this.enabled) return true;
|
||||
const ch = inputs[0] && inputs[0][0];
|
||||
if (!ch || ch.length === 0) return true;
|
||||
for (let i = this.decimateCounter; i < ch.length; i += this.decimationFactor){
|
||||
if (this.bufPos < this.buf.length) this.buf[this.bufPos++] = ch[i];
|
||||
}
|
||||
this.decimateCounter = (this.decimateCounter + ch.length) % this.decimationFactor;
|
||||
if (this.bufPos >= this.chunkTargetSamples){
|
||||
const chunk = this.buf.slice(0, this.bufPos);
|
||||
this.port.postMessage({ chunk }, [chunk.buffer]);
|
||||
this.bufPos = 0;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('whisper-capture', WhisperCaptureProcessor);
|
||||
`;
|
||||
|
||||
let whisperCaptureWorkletReady = false;
|
||||
let whisperCaptureWorkletLoading = false;
|
||||
function loadWhisperCaptureWorklet(ctx){
|
||||
if (whisperCaptureWorkletReady || whisperCaptureWorkletLoading) return Promise.resolve(whisperCaptureWorkletReady);
|
||||
whisperCaptureWorkletLoading = true;
|
||||
const blob = new Blob([WHISPER_CAPTURE_WORKLET_CODE], { type: 'application/javascript' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
return ctx.audioWorklet.addModule(url).then(() => {
|
||||
URL.revokeObjectURL(url);
|
||||
whisperCaptureWorkletReady = true;
|
||||
whisperCaptureWorkletLoading = false;
|
||||
return true;
|
||||
}).catch(e => {
|
||||
URL.revokeObjectURL(url);
|
||||
whisperCaptureWorkletLoading = false;
|
||||
logLine('err', 'whisper-capture worklet load: '+e.message);
|
||||
return false;
|
||||
});
|
||||
}
|
||||
|
||||
let whisperPipeline = null;
|
||||
let whisperLoading = false;
|
||||
async function ensureWhisperPipeline(){
|
||||
if (whisperPipeline) return whisperPipeline;
|
||||
if (whisperLoading){
|
||||
while (whisperLoading && !whisperPipeline) await new Promise(r => setTimeout(r, 200));
|
||||
return whisperPipeline;
|
||||
}
|
||||
whisperLoading = true;
|
||||
logLine('', 'whisper: loading model (~40MB, first time only)…');
|
||||
try {
|
||||
/* transformers.js from jsDelivr — runs ONNX Runtime + whisper-tiny.en
|
||||
* fully in-browser. Model is downloaded once and cached by the
|
||||
* service worker / browser cache; subsequent toggles are instant. */
|
||||
const tx = await import('https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.0');
|
||||
whisperPipeline = await tx.pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');
|
||||
logLine('', 'whisper: model ready');
|
||||
} catch (e){
|
||||
logLine('err', 'whisper load failed: '+e.message);
|
||||
} finally {
|
||||
whisperLoading = false;
|
||||
}
|
||||
return whisperPipeline;
|
||||
}
|
||||
|
||||
let transcribeEnabled = false;
|
||||
function appendTranscriptLine(uuid, text){
|
||||
const log = document.getElementById('transcript-log');
|
||||
if (!log) return;
|
||||
const member = members.get(uuid);
|
||||
const name = (member && member.handle) ? member.handle : uuid.slice(0,4);
|
||||
const now = new Date();
|
||||
const ts = String(now.getHours()).padStart(2,'0')+':'+
|
||||
String(now.getMinutes()).padStart(2,'0')+':'+
|
||||
String(now.getSeconds()).padStart(2,'0');
|
||||
const line = document.createElement('div');
|
||||
line.className = 'transcript-line';
|
||||
const tsEl = document.createElement('span'); tsEl.className = 'ts'; tsEl.textContent = ts;
|
||||
const nameEl = document.createElement('span'); nameEl.className = 'name'; nameEl.textContent = name + ':';
|
||||
const txtEl = document.createElement('span'); txtEl.className = 'txt'; txtEl.textContent = text;
|
||||
line.appendChild(tsEl); line.append(' ');
|
||||
line.appendChild(nameEl); line.append(' ');
|
||||
line.appendChild(txtEl);
|
||||
log.appendChild(line);
|
||||
/* auto-scroll only if user is already at the bottom (let them scroll
|
||||
* up to read older transcript without being yanked back). */
|
||||
const nearBottom = (log.scrollTop + log.clientHeight) >= (log.scrollHeight - 40);
|
||||
if (nearBottom) log.scrollTop = log.scrollHeight;
|
||||
}
|
||||
|
||||
async function startCaptureForUuid(uuid){
|
||||
const node = listenerAudioNodes.get(uuid);
|
||||
if (!node || !audioCtx || node.capture) return;
|
||||
const ok = await loadWhisperCaptureWorklet(audioCtx);
|
||||
if (!ok) return;
|
||||
try {
|
||||
const capture = new AudioWorkletNode(audioCtx, 'whisper-capture');
|
||||
capture.port.onmessage = async (e) => {
|
||||
if (!e.data || !e.data.chunk) return;
|
||||
const pipeline = await ensureWhisperPipeline();
|
||||
if (!pipeline) return;
|
||||
try {
|
||||
const result = await pipeline(e.data.chunk, { sampling_rate: 16000 });
|
||||
const txt = (result && result.text ? result.text : '').trim();
|
||||
/* skip empty / placeholder transcriptions ("." " " "[BLANK_AUDIO]" etc.) */
|
||||
if (txt.length >= 2 && !/^\[/.test(txt)) appendTranscriptLine(uuid, txt);
|
||||
} catch (err){
|
||||
logLine('err', 'whisper transcribe '+uuid.slice(0,4)+': '+err.message);
|
||||
}
|
||||
};
|
||||
/* tap the source — capture runs in parallel with the worklet
|
||||
* chain, doesn't need to connect to destination (we just want
|
||||
* the chunks via port messages). */
|
||||
node.src.connect(capture);
|
||||
capture.port.postMessage({ cmd: 'start' });
|
||||
node.capture = capture;
|
||||
logLine('', 'whisper: capture started for '+uuid.slice(0,4));
|
||||
} catch (e){
|
||||
logLine('err', 'whisper capture install '+uuid.slice(0,4)+': '+e.message);
|
||||
}
|
||||
}
|
||||
|
||||
function stopCaptureForUuid(uuid){
|
||||
const node = listenerAudioNodes.get(uuid);
|
||||
if (!node || !node.capture) return;
|
||||
try { node.capture.port.postMessage({ cmd: 'stop' }); } catch(_){}
|
||||
try { node.capture.disconnect(); } catch(_){}
|
||||
try { node.src.disconnect(node.capture); } catch(_){}
|
||||
node.capture = null;
|
||||
}
|
||||
|
||||
async function toggleTranscribe(){
|
||||
transcribeEnabled = !transcribeEnabled;
|
||||
const btn = document.getElementById('btn-transcribe');
|
||||
const sec = document.getElementById('sec-transcript');
|
||||
if (btn) btn.classList.toggle('on', transcribeEnabled);
|
||||
if (btn) btn.textContent = transcribeEnabled ? 'transcribe (on)' : 'transcribe (off)';
|
||||
if (sec) sec.classList.toggle('hidden', !transcribeEnabled);
|
||||
if (transcribeEnabled){
|
||||
/* pre-warm the pipeline so the first chunk doesn't wait. */
|
||||
ensureWhisperPipeline().catch(()=>{});
|
||||
for (const [uuid] of listenerAudioNodes){
|
||||
startCaptureForUuid(uuid).catch(e => logLine('err', 'transcribe start '+uuid+': '+e.message));
|
||||
}
|
||||
} else {
|
||||
for (const [uuid] of listenerAudioNodes) stopCaptureForUuid(uuid);
|
||||
}
|
||||
}
|
||||
function installJitterBuffer(uuid, node){
|
||||
if (!node || node.jbuf || !workletReady) return;
|
||||
const target = node.targetSeconds || RECV_PLAYOUT_DELAY_SEC;
|
||||
|
|
@ -2426,6 +2652,12 @@ function attachAudioStreamViaWorklet(uuid, stream, targetSeconds){
|
|||
* existing stream is swapped through the buffer (see loadJitterWorklet) */
|
||||
loadJitterWorklet(audioCtx);
|
||||
if (workletReady) installJitterBuffer(uuid, node);
|
||||
/* If transcribe is on, install a parallel capture node for this
|
||||
* new speaker so their voice immediately starts flowing into the
|
||||
* transcript log. */
|
||||
if (transcribeEnabled){
|
||||
startCaptureForUuid(uuid).catch(e => logLine('err', 'transcribe '+uuid.slice(0,4)+': '+e.message));
|
||||
}
|
||||
logLine('', 'audio via AudioContext '+uuid.slice(0,4)+' target='+targetSeconds+'s ctxState='+audioCtx.state);
|
||||
return true;
|
||||
}
|
||||
|
|
@ -2467,6 +2699,7 @@ function detachListenerStream(uuid){
|
|||
if (!node) return;
|
||||
try { node.src.disconnect(); } catch(_){}
|
||||
try { if (node.jbuf) node.jbuf.disconnect(); } catch(_){}
|
||||
try { if (node.capture) node.capture.disconnect(); } catch(_){}
|
||||
try { node.gain.disconnect(); } catch(_){}
|
||||
listenerAudioNodes.delete(uuid);
|
||||
}
|
||||
|
|
@ -6848,8 +7081,8 @@ logLine('', 'ready — pick a handle, type a rendezvous code, enter the space');
|
|||
|
||||
<footer style="margin:2.2rem auto 0;font-size:0.65rem;color:#999;line-height:1.7;word-break:break-all;font-family:monospace">
|
||||
<span id="pi-seal" style="color:#777;cursor:default;user-select:none" title="">page integrity</span> · built <span class="stamp-date">2026-06-05</span><br>
|
||||
md5 <span class="stamp-md5">9728607e6d8187fed0bfa947f93cccaa</span><br>
|
||||
sha256 <span class="stamp-sha">26680e413131920d5c0dc981b20460b1c087d08193f1f5d89925d22e3013ec11</span><br>
|
||||
md5 <span class="stamp-md5">4591b92519ec28db69de15bb20eb2300</span><br>
|
||||
sha256 <span class="stamp-sha">0b792e6dd0d9bc69f888c239468c57506bc1d0feb9ded8b78bb650b886501c9b</span><br>
|
||||
<span style="color:#bbb">hashes are of this page with these two fields zeroed — to verify, blank them and re-hash</span><br>
|
||||
<span style="color:#bbb">one self-contained file — <strong>save a copy</strong> and verify against these hashes; point at your own servers with ?signal= and ?turncred=, or <a href="host-your-own.html" style="color:#999">host your own community</a></span>
|
||||
</footer>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue