From e45db971aad72255d3905225bd14551473ba35c4 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sat, 6 Jun 2026 15:21:29 -0400 Subject: [PATCH] zebra-report: deploy shrink-retarget audio + lip-sync coupling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pulls in zebra-report 9215eaa: JitterBufferProcessor now drops the queue down to targetSamples (not 1.5×target) on a shrinking retarget, eliminating the residual 6%-cap speed-up phase fox heard as "janky slow" during listener-to-speaker promotion. The processor reports the dropped sample count back to JS so installJitterBuffer's port handler zeros every paired video receiver's jitterBufferTarget + playoutDelayHint — video frames drop in lockstep with the audio skip so lip-sync survives the jump. Fox 2026-06-06: "if we skip ahead from whatever listener is at to speaker speed, we need to make sure the video skips ahead the same amount or rate to keep the lips synced". Behavior change is isolated to the role-change moment (4s→0.5s on promote). Steady-state audio and listener-only sessions unchanged. --- zebra-report/zebra-spaces.html | 66 ++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/zebra-report/zebra-spaces.html b/zebra-report/zebra-spaces.html index bf86062..f5b802f 100644 --- a/zebra-report/zebra-spaces.html +++ b/zebra-report/zebra-spaces.html @@ -1878,14 +1878,37 @@ class JitterBufferProcessor extends AudioWorkletProcessor { if (e.data.cmd === 'retarget'){ const t = +e.data.targetSeconds; if (!isFinite(t) || t <= 0) return; + const prevTargetSamples = this.targetSamples; this.targetSeconds = t; this.maxSeconds = t * 1.5; this.targetSamples = Math.round(this.targetSeconds * sampleRate); this.maxSamples = Math.round(this.maxSeconds * sampleRate); - while (this.buffered > this.maxSamples && this.queue.length > 0){ + /* On a SHRINKING retarget (listener-to-speaker promotion: 4s + * → 0.5s), drop down to targetSamples directly instead of + * maxSamples. The maxSamples threshold leaves a 1.5× target + * overhang that triggers an audible 6%-cap speed-up phase + * lasting ~4 seconds — what fox 2026-06-06 called "janky" + * during promotion. One hard skip-forward, then normal + * playback — no residual adjustment. + * + * Report the skip back to JS so paired video receivers can + * advance the same amount: an audio jump without a matching + * video jump = broken lip-sync for the duration of the + * native video jbuf's gradual drain. fox 2026-06-06: "if we + * skip ahead from whatever listener is at to speaker speed, + * we need to make sure the video skips ahead the same amount + * or rate to keep the lips synced". */ + const shrinking = this.targetSamples < prevTargetSamples; + const dropTo = shrinking ? this.targetSamples : this.maxSamples; + let skipped = 0; + while (this.buffered > dropTo && this.queue.length > 0){ const drop = this.queue.shift(); this.buffered -= drop[0].length; this.dropped += drop[0].length; + skipped += drop[0].length; + } + if (skipped > 0){ + try { this.port.postMessage({ cmd: 'dropped', samples: skipped }); } catch(_){} } } else if (e.data.cmd === 'lock_rate'){ /* listeners get this — explicitly forbid time-stretching so @@ -2542,6 +2565,43 @@ function installJitterBuffer(uuid, node){ * enough to catch a wiggle within ~2s of it starting, * vs the 5s tick path which can be 5-10s late. */ ddNoteWorkletBuffered(uuid, e.data.seconds); + } else if (e.data.cmd === 'dropped'){ + /* The worklet just skipped audio forward (shrinking retarget + * — listener→speaker promotion drops 4s→0.5s buffer). Drag + * the paired video receivers forward by the same amount so + * lip-sync survives the jump. fox 2026-06-06: "if we skip + * ahead from whatever listener is at to speaker speed, we + * need to make sure the video skips ahead the same amount + * or rate to keep the lips synced". + * + * No native "skip ahead" API exists for RTCRtpReceiver, but + * jitterBufferTarget is a hard target the browser converges + * to. Setting it to 0 forces aggressive frame-drop until the + * native video jbuf drains; the next refreshLipSyncForUuid + * 'buffered' tick (~2s) restores the proper role-appropriate + * target. Net: audio jumps instantly + video jumps almost + * instantly (browser frame-drop is fast) = lip-sync stays + * within ~tens of ms. */ + try { + const mm = members.get(uuid); + const pubHex = mm && mm.pubkey ? hex(unb64(mm.pubkey)) : null; + if (pubHex){ + const ls = lipSync.get(pubHex); + if (ls && ls.videoReceivers){ + for (const [, rx] of ls.videoReceivers){ + try { rx.jitterBufferTarget = 0; } catch(_){} + try { rx.playoutDelayHint = 0; } catch(_){} + } + /* clear lastApplied so the next refreshLipSyncForUuid + * actually re-applies the role's target — without this + * the threshold check could see "no significant change" + * and leave video stuck at 0. */ + ls.lastApplied = 0; + } + } + const skippedSec = e.data.samples / (audioCtx ? audioCtx.sampleRate : 48000); + logLine('', 'jitter-buffer skipped '+skippedSec.toFixed(2)+'s uuid='+uuid.slice(0,4)+' — video re-targeting'); + } catch(_){} } }; /* All roles now allow time-stretching — listeners benefit too @@ -7904,8 +7964,8 @@ logLine('', 'ready — pick a handle, type a rendezvous code, enter the space');