diff --git a/web/zebra-spaces.html b/web/zebra-spaces.html index 24ca444..89fb3b5 100644 --- a/web/zebra-spaces.html +++ b/web/zebra-spaces.html @@ -1780,6 +1780,14 @@ class JitterBufferProcessor extends AudioWorkletProcessor { * tear down playback, or a 4s re-buffer kicks in every time — * which is what made the phone choppy. */ this.rearmThresholdBlocks = 100; + /* periodic buffer-depth report so JS can drive the lip-sync + * algorithm (video.playoutDelayHint must match audio total delay + * including this worklet's cushion, else mouths move ~4s ahead + * of the words on listeners). 256 blocks * 128 samples / 48000Hz + * ≈ 683ms — frequent enough to track real changes, sparse enough + * to not flood the port. */ + this.bufferedReportEvery = 256; + this.bufferedReportCounter = 0; this.queue = []; this.buffered = 0; this.started = false; @@ -1851,6 +1859,10 @@ class JitterBufferProcessor extends AudioWorkletProcessor { } } } + if (++this.bufferedReportCounter >= this.bufferedReportEvery){ + this.bufferedReportCounter = 0; + try { this.port.postMessage({ cmd: 'buffered', seconds: this.buffered / sampleRate }); } catch(_){} + } return true; } } @@ -1893,9 +1905,16 @@ function installJitterBuffer(uuid, node){ * we mark the listener buffer ready on the FIRST one (audio is * audible by then). */ jbuf.port.onmessage = (e) => { - if (e.data && e.data.cmd === 'started'){ + if (!e.data) return; + if (e.data.cmd === 'started'){ logLine('', 'jitter-buffer started uuid='+uuid.slice(0,4)+' target='+(e.data.targetSeconds||target)+'s'); onWorkletStarted(uuid); + } else if (e.data.cmd === 'buffered'){ + /* worklet's current buffer depth (seconds). Used by lip-sync + * to make video receivers track the audio's total delay. */ + const n = listenerAudioNodes.get(uuid); + if (n) n.bufferedSeconds = e.data.seconds; + refreshLipSyncForUuid(uuid); } }; try { node.src.disconnect(node.gain); } catch(_){} @@ -1933,6 +1952,93 @@ function resetListenerBufferReady(){ listenerBufferReady = false; } +/* ================================================================== + * lip-sync — per-publisher dynamic video playoutDelayHint + * + * Without this, listener video leads listener audio by up to 4s on + * music streams: video receivers honor playoutDelayHint=4s natively, + * but audio adds the worklet's 4s cushion ON TOP of whatever the + * native audio buffer is doing — total audio delay = native_jbuf + + * worklet_buffered_seconds, video delay = native_jbuf only. + * + * Algorithm: every time the worklet reports its current buffer depth + * (every ~683ms via {cmd:'buffered'}), recompute the audio total delay + * for that publisher, then set each of that publisher's video + * receivers' playoutDelayHint + jitterBufferTarget to match. + * + * FEC-style smoothing per fox 2026-06-04 request: maintain a rolling + * history of the last LIP_SYNC_HISTORY samples; apply the MEDIAN, not + * the latest. Single-sample outliers from network jitter or getStats + * noise can't whip the video target. Median-of-5 is a poor man's + * Hamming for a control signal — needs ≥3 of 5 samples to agree + * before a new target locks in. Hysteresis on top: only retarget if + * the new median differs by more than LIP_SYNC_THRESHOLD from the + * last applied value. */ +const lipSync = new Map(); /* pubHex → state */ +const LIP_SYNC_HISTORY = 5; +const LIP_SYNC_THRESHOLD = 0.05; /* 50ms — below this is within perception noise */ + +function lipSyncEntry(pubHex){ + let e = lipSync.get(pubHex); + if (!e){ + e = { audioUuid: null, audioReceiver: null, + nativeJbufSec: 0, videoReceivers: new Map(), + history: [], lastApplied: 0 }; + lipSync.set(pubHex, e); + } + return e; +} +function registerLipSyncAudio(pubHex, uuid, receiver){ + const e = lipSyncEntry(pubHex); + e.audioUuid = uuid; + e.audioReceiver = receiver; +} +function registerLipSyncVideo(pubHex, kind, receiver){ + const e = lipSyncEntry(pubHex); + e.videoReceivers.set(kind, receiver); +} +function medianOf(arr){ + if (arr.length === 0) return 0; + const s = arr.slice().sort((a,b) => a-b); + return s[Math.floor(s.length/2)]; +} +async function refreshLipSyncForUuid(uuid){ + /* find which publisher this audio uuid belongs to */ + let pubHex = null, e = null; + for (const [ph, st] of lipSync){ + if (st.audioUuid === uuid){ pubHex = ph; e = st; break; } + } + if (!e || !e.audioReceiver || e.videoReceivers.size === 0) return; + const node = listenerAudioNodes.get(uuid); + if (!node) return; + const workletBufferedSec = node.bufferedSeconds || 0; + /* refresh native jbuf via getStats; this is a slow path but we + * only do it on the ~683ms cadence the worklet posts, and getStats + * is cheap (~1-2ms on Firefox Android). */ + try { + const stats = await e.audioReceiver.getStats(); + stats.forEach(r => { + if (r.type === 'inbound-rtp' && r.kind === 'audio'){ + if (r.jitterBufferEmittedCount > 0){ + e.nativeJbufSec = r.jitterBufferDelay / r.jitterBufferEmittedCount; + } + } + }); + } catch(_){} + const audioTotal = e.nativeJbufSec + workletBufferedSec; + e.history.push(audioTotal); + while (e.history.length > LIP_SYNC_HISTORY) e.history.shift(); + if (e.history.length < 3) return; /* wait for ≥3 samples */ + const med = medianOf(e.history); + if (Math.abs(med - e.lastApplied) < LIP_SYNC_THRESHOLD) return; + for (const [kind, rx] of e.videoReceivers){ + try { rx.playoutDelayHint = med; } catch(_){} + try { rx.jitterBufferTarget = med * 1000; } catch(_){} + } + e.lastApplied = med; + logLine('', 'lip-sync pub='+pubHex.slice(0,4)+' delay='+med.toFixed(2)+'s ('+e.videoReceivers.size+' video rx, history median of '+e.history.length+')'); +} + /* Shared audio attach path. Every role routes through here now so the * worklet can apply role-appropriate buffer depth uniformly. Listener * gets a fat 4s cushion (lean-back, latency doesn't matter); speakers @@ -3185,6 +3291,10 @@ function handleRemoteSfuTrack(ev){ * the other way around. */ try { if (ev.receiver) ev.receiver.playoutDelayHint = playoutDelayForRole(myRole); } catch(_){} try { if (ev.receiver) ev.receiver.jitterBufferTarget = playoutDelayForRole(myRole) * 1000; } catch(_){} + /* register for the dynamic lip-sync algorithm — the next worklet + * 'buffered' message will recompute this video receiver's target + * to match the audio's total delay (native jbuf + worklet). */ + if (ev.receiver) registerLipSyncVideo(pubHex, kind, ev.receiver); } /* MSID-supplant safety: the SFU re-uses the same streamID * (`shortPub-kind`) when a publisher supplants themselves. WebRTC @@ -3253,6 +3363,13 @@ function handleRemoteSfuTrack(ev){ for (const [uuid, mm] of members){ try { if (mm.pubkey && hex(unb64(mm.pubkey)) === pubHex){ + /* register the audio receiver for lip-sync. We use the audio + * receiver's getStats jbuf + the worklet's buffer depth to set + * matching video target on the same publisher's screen/camera + * receivers. Register here (mic kind) BEFORE attachSfuTrack + * so the first worklet 'buffered' message lands on a paired + * publisher state. */ + if (ev.receiver) registerLipSyncAudio(pubHex, uuid, ev.receiver); /* speakers get their peers' audio via mesh (lower latency) * AT THE TIMES THE MESH PC IS CONNECTED. A stale or failing * mesh PC must NOT block the SFU fallback — that's how the @@ -6254,8 +6371,8 @@ logLine('', 'ready — pick a handle, type a rendezvous code, enter the space');