zebra-spaces: lip-sync — dynamic video playoutDelayHint matches audio total delay
Fox 2026-06-04: "lips on the video share are not synced with the audio."
Source of the desync:
- Audio total delay = native_jbuf + worklet_buffered (because the
AudioWorklet adds its 4s cushion ON TOP of whatever the native
receiver does)
- Video total delay = native_jbuf only (no worklet downstream)
- Net: video leads audio by up to ~4s on music listeners; by less on
voice (where native honors the hint, ~3s); audible mismatch in either.
Algorithm: per-publisher dynamic match. Every ~683ms the worklet
posts {cmd:'buffered', seconds: this.buffered/sampleRate}. JS-side
handler refreshes lip-sync for that uuid:
1. Resolve uuid → publisher pubHex
2. Read audio receiver's getStats jbuf (native part)
3. audioTotal = native_jbuf + worklet_buffered
4. Push to per-publisher history (size 5)
5. Compute median (FEC-style: 3-of-5 must agree before lock-in)
6. If |median - lastApplied| > 0.05s, set every video receiver for
that publisher: playoutDelayHint = median, jitterBufferTarget = median*1000
Hamming-spirit on a control signal per fox's request: median-of-5
rejects single-sample outliers from network jitter or getStats noise.
50ms hysteresis below perception threshold so the video target
doesn't whip on tiny shifts.
Registers:
- registerLipSyncAudio(pubHex, uuid, receiver) — at mic-kind ontrack
- registerLipSyncVideo(pubHex, kind, receiver) — at screen/camera/game ontrack
- Map: pubHex → {audioUuid, audioReceiver, nativeJbufSec, videoReceivers, history, lastApplied}
The role-aware base delay (playoutDelayForRole) is still applied at
attach as a sane initial value; lip-sync then refines per-publisher
within a few seconds.
This commit is contained in:
parent
1f6fde2a3e
commit
8d18216854
1 changed files with 120 additions and 3 deletions
|
|
@ -1780,6 +1780,14 @@ class JitterBufferProcessor extends AudioWorkletProcessor {
|
|||
* tear down playback, or a 4s re-buffer kicks in every time —
|
||||
* which is what made the phone choppy. */
|
||||
this.rearmThresholdBlocks = 100;
|
||||
/* periodic buffer-depth report so JS can drive the lip-sync
|
||||
* algorithm (video.playoutDelayHint must match audio total delay
|
||||
* including this worklet's cushion, else mouths move ~4s ahead
|
||||
* of the words on listeners). 256 blocks * 128 samples / 48000Hz
|
||||
* ≈ 683ms — frequent enough to track real changes, sparse enough
|
||||
* to not flood the port. */
|
||||
this.bufferedReportEvery = 256;
|
||||
this.bufferedReportCounter = 0;
|
||||
this.queue = [];
|
||||
this.buffered = 0;
|
||||
this.started = false;
|
||||
|
|
@ -1851,6 +1859,10 @@ class JitterBufferProcessor extends AudioWorkletProcessor {
|
|||
}
|
||||
}
|
||||
}
|
||||
if (++this.bufferedReportCounter >= this.bufferedReportEvery){
|
||||
this.bufferedReportCounter = 0;
|
||||
try { this.port.postMessage({ cmd: 'buffered', seconds: this.buffered / sampleRate }); } catch(_){}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -1893,9 +1905,16 @@ function installJitterBuffer(uuid, node){
|
|||
* we mark the listener buffer ready on the FIRST one (audio is
|
||||
* audible by then). */
|
||||
jbuf.port.onmessage = (e) => {
|
||||
if (e.data && e.data.cmd === 'started'){
|
||||
if (!e.data) return;
|
||||
if (e.data.cmd === 'started'){
|
||||
logLine('', 'jitter-buffer started uuid='+uuid.slice(0,4)+' target='+(e.data.targetSeconds||target)+'s');
|
||||
onWorkletStarted(uuid);
|
||||
} else if (e.data.cmd === 'buffered'){
|
||||
/* worklet's current buffer depth (seconds). Used by lip-sync
|
||||
* to make video receivers track the audio's total delay. */
|
||||
const n = listenerAudioNodes.get(uuid);
|
||||
if (n) n.bufferedSeconds = e.data.seconds;
|
||||
refreshLipSyncForUuid(uuid);
|
||||
}
|
||||
};
|
||||
try { node.src.disconnect(node.gain); } catch(_){}
|
||||
|
|
@ -1933,6 +1952,93 @@ function resetListenerBufferReady(){
|
|||
listenerBufferReady = false;
|
||||
}
|
||||
|
||||
/* ==================================================================
|
||||
* lip-sync — per-publisher dynamic video playoutDelayHint
|
||||
*
|
||||
* Without this, listener video leads listener audio by up to 4s on
|
||||
* music streams: video receivers honor playoutDelayHint=4s natively,
|
||||
* but audio adds the worklet's 4s cushion ON TOP of whatever the
|
||||
* native audio buffer is doing — total audio delay = native_jbuf +
|
||||
* worklet_buffered_seconds, video delay = native_jbuf only.
|
||||
*
|
||||
* Algorithm: every time the worklet reports its current buffer depth
|
||||
* (every ~683ms via {cmd:'buffered'}), recompute the audio total delay
|
||||
* for that publisher, then set each of that publisher's video
|
||||
* receivers' playoutDelayHint + jitterBufferTarget to match.
|
||||
*
|
||||
* FEC-style smoothing per fox 2026-06-04 request: maintain a rolling
|
||||
* history of the last LIP_SYNC_HISTORY samples; apply the MEDIAN, not
|
||||
* the latest. Single-sample outliers from network jitter or getStats
|
||||
* noise can't whip the video target. Median-of-5 is a poor man's
|
||||
* Hamming for a control signal — needs ≥3 of 5 samples to agree
|
||||
* before a new target locks in. Hysteresis on top: only retarget if
|
||||
* the new median differs by more than LIP_SYNC_THRESHOLD from the
|
||||
* last applied value. */
|
||||
const lipSync = new Map(); /* pubHex → state */
|
||||
const LIP_SYNC_HISTORY = 5;
|
||||
const LIP_SYNC_THRESHOLD = 0.05; /* 50ms — below this is within perception noise */
|
||||
|
||||
function lipSyncEntry(pubHex){
|
||||
let e = lipSync.get(pubHex);
|
||||
if (!e){
|
||||
e = { audioUuid: null, audioReceiver: null,
|
||||
nativeJbufSec: 0, videoReceivers: new Map(),
|
||||
history: [], lastApplied: 0 };
|
||||
lipSync.set(pubHex, e);
|
||||
}
|
||||
return e;
|
||||
}
|
||||
function registerLipSyncAudio(pubHex, uuid, receiver){
|
||||
const e = lipSyncEntry(pubHex);
|
||||
e.audioUuid = uuid;
|
||||
e.audioReceiver = receiver;
|
||||
}
|
||||
function registerLipSyncVideo(pubHex, kind, receiver){
|
||||
const e = lipSyncEntry(pubHex);
|
||||
e.videoReceivers.set(kind, receiver);
|
||||
}
|
||||
function medianOf(arr){
|
||||
if (arr.length === 0) return 0;
|
||||
const s = arr.slice().sort((a,b) => a-b);
|
||||
return s[Math.floor(s.length/2)];
|
||||
}
|
||||
async function refreshLipSyncForUuid(uuid){
|
||||
/* find which publisher this audio uuid belongs to */
|
||||
let pubHex = null, e = null;
|
||||
for (const [ph, st] of lipSync){
|
||||
if (st.audioUuid === uuid){ pubHex = ph; e = st; break; }
|
||||
}
|
||||
if (!e || !e.audioReceiver || e.videoReceivers.size === 0) return;
|
||||
const node = listenerAudioNodes.get(uuid);
|
||||
if (!node) return;
|
||||
const workletBufferedSec = node.bufferedSeconds || 0;
|
||||
/* refresh native jbuf via getStats; this is a slow path but we
|
||||
* only do it on the ~683ms cadence the worklet posts, and getStats
|
||||
* is cheap (~1-2ms on Firefox Android). */
|
||||
try {
|
||||
const stats = await e.audioReceiver.getStats();
|
||||
stats.forEach(r => {
|
||||
if (r.type === 'inbound-rtp' && r.kind === 'audio'){
|
||||
if (r.jitterBufferEmittedCount > 0){
|
||||
e.nativeJbufSec = r.jitterBufferDelay / r.jitterBufferEmittedCount;
|
||||
}
|
||||
}
|
||||
});
|
||||
} catch(_){}
|
||||
const audioTotal = e.nativeJbufSec + workletBufferedSec;
|
||||
e.history.push(audioTotal);
|
||||
while (e.history.length > LIP_SYNC_HISTORY) e.history.shift();
|
||||
if (e.history.length < 3) return; /* wait for ≥3 samples */
|
||||
const med = medianOf(e.history);
|
||||
if (Math.abs(med - e.lastApplied) < LIP_SYNC_THRESHOLD) return;
|
||||
for (const [kind, rx] of e.videoReceivers){
|
||||
try { rx.playoutDelayHint = med; } catch(_){}
|
||||
try { rx.jitterBufferTarget = med * 1000; } catch(_){}
|
||||
}
|
||||
e.lastApplied = med;
|
||||
logLine('', 'lip-sync pub='+pubHex.slice(0,4)+' delay='+med.toFixed(2)+'s ('+e.videoReceivers.size+' video rx, history median of '+e.history.length+')');
|
||||
}
|
||||
|
||||
/* Shared audio attach path. Every role routes through here now so the
|
||||
* worklet can apply role-appropriate buffer depth uniformly. Listener
|
||||
* gets a fat 4s cushion (lean-back, latency doesn't matter); speakers
|
||||
|
|
@ -3185,6 +3291,10 @@ function handleRemoteSfuTrack(ev){
|
|||
* the other way around. */
|
||||
try { if (ev.receiver) ev.receiver.playoutDelayHint = playoutDelayForRole(myRole); } catch(_){}
|
||||
try { if (ev.receiver) ev.receiver.jitterBufferTarget = playoutDelayForRole(myRole) * 1000; } catch(_){}
|
||||
/* register for the dynamic lip-sync algorithm — the next worklet
|
||||
* 'buffered' message will recompute this video receiver's target
|
||||
* to match the audio's total delay (native jbuf + worklet). */
|
||||
if (ev.receiver) registerLipSyncVideo(pubHex, kind, ev.receiver);
|
||||
}
|
||||
/* MSID-supplant safety: the SFU re-uses the same streamID
|
||||
* (`shortPub-kind`) when a publisher supplants themselves. WebRTC
|
||||
|
|
@ -3253,6 +3363,13 @@ function handleRemoteSfuTrack(ev){
|
|||
for (const [uuid, mm] of members){
|
||||
try {
|
||||
if (mm.pubkey && hex(unb64(mm.pubkey)) === pubHex){
|
||||
/* register the audio receiver for lip-sync. We use the audio
|
||||
* receiver's getStats jbuf + the worklet's buffer depth to set
|
||||
* matching video target on the same publisher's screen/camera
|
||||
* receivers. Register here (mic kind) BEFORE attachSfuTrack
|
||||
* so the first worklet 'buffered' message lands on a paired
|
||||
* publisher state. */
|
||||
if (ev.receiver) registerLipSyncAudio(pubHex, uuid, ev.receiver);
|
||||
/* speakers get their peers' audio via mesh (lower latency)
|
||||
* AT THE TIMES THE MESH PC IS CONNECTED. A stale or failing
|
||||
* mesh PC must NOT block the SFU fallback — that's how the
|
||||
|
|
@ -6254,8 +6371,8 @@ logLine('', 'ready — pick a handle, type a rendezvous code, enter the space');
|
|||
|
||||
<footer style="margin:2.2rem auto 0;font-size:0.65rem;color:#999;line-height:1.7;word-break:break-all;font-family:monospace">
|
||||
<span id="pi-seal" style="color:#777;cursor:default;user-select:none" title="">page integrity</span> · built <span class="stamp-date">2026-06-04</span><br>
|
||||
md5 <span class="stamp-md5">a8928d81125e2276e6724cc39d28d782</span><br>
|
||||
sha256 <span class="stamp-sha">1f7f5b419172ae5a15edcbb63b160d9bb63d88ec7809e30e22b9ea75533e9c72</span><br>
|
||||
md5 <span class="stamp-md5">544ed25e7616ced6a24bbc8c29bf13ae</span><br>
|
||||
sha256 <span class="stamp-sha">1bb6283957a5a00cc6e9e52c427d85791345bd6383d95f24bca1016497205f81</span><br>
|
||||
<span style="color:#bbb">hashes are of this page with these two fields zeroed — to verify, blank them and re-hash</span><br>
|
||||
<span style="color:#bbb">one self-contained file — <strong>save a copy</strong> and verify against these hashes; point at your own servers with ?signal= and ?turncred=, or <a href="host-your-own.html" style="color:#999">host your own community</a></span>
|
||||
</footer>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue