zebra-spaces: listener video — playoutDelayHint=0, no lip-sync, instant tile switching
Fox 2026-06-05: "we should switch feeds immediately I don't want any algo slowing that down." The previous lip-sync algorithm set video receiver playoutDelayHint to match audio's worklet delay (up to 4s on listeners). That made spotlight-switching take up to 4s — the new tile's <video> element had to wait for a keyframe to appear inside the 4s-deep receiver buffer before showing the first frame. Changes (listener role only): - handleRemoteSfuTrack screen/camera/game branch: vDelay = 0 for listener (browser-lowest). Speaker/cohost/host unchanged. - refreshLipSyncForUuid: early-return for myRole === 'listener'. No more retargeting video to match audio for listeners. - retargetAllReceivers (called on role change): audio retargets per role; VIDEO retargets to 0 for listener, role-aware for others. Listener video now plays at browser-default minimum delay. Tile switches are instant — keyframe wait + native buffer pre-roll only, typically <300ms. Tradeoff accepted: listener mouth-leads-voice by approximately the audio worklet's current cushion (1.3–4s adaptive). For consume- content mode (lean-back listener) this is correct; switching shares is far more important than per-syllable lip-sync. Speakers/cohosts/hosts unaffected — their audio cushion is ~0.5s, video matches it via lip-sync, mouths and voice align naturally.
This commit is contained in:
parent
86df9622d1
commit
209a15a1a9
1 changed files with 33 additions and 18 deletions
|
|
@ -2323,6 +2323,14 @@ async function refreshLipSyncForUuid(uuid){
|
|||
* use a fixed HTTP_STREAM_DELAY_SEC estimate (set by startStream).
|
||||
* Apply it directly with the same threshold/hysteresis as the
|
||||
* worklet path. */
|
||||
/* Listener role: skip the video-receiver retarget entirely. Their
|
||||
* tiles need to switch instantly when the user spotlights a
|
||||
* different share — applying a 4s buffer to video defeats that.
|
||||
* Audio stays at full cushion via the worklet; listener accepts
|
||||
* mouth-leads-voice as the price of responsive switching.
|
||||
* Fox 2026-06-05: "we should switch feeds immediately I don't
|
||||
* want any algo slowing that down." */
|
||||
if (myRole === 'listener') return;
|
||||
const override = httpLipSyncOverride.get(pubHex);
|
||||
if (typeof override === 'number'){
|
||||
if (Math.abs(override - e.lastApplied) < LIP_SYNC_THRESHOLD) return;
|
||||
|
|
@ -3933,17 +3941,20 @@ function handleRemoteSfuTrack(ev){
|
|||
if (kind === 'screen' || kind === 'camera' || kind === 'game'){
|
||||
logLine('', 'sfu ontrack: kind=' + kind + ' pub=' + pubHex +
|
||||
' track=' + ev.track.kind + ' mute=' + ev.track.muted + ' state=' + ev.track.readyState);
|
||||
/* Video receivers (screen / camera / game) match the audio receiver's
|
||||
* playoutDelayHint so the picture stays in sync with the voice. The
|
||||
* mic jitter buffer is 4s; without a matching hint on video, mouse
|
||||
* clicks / mouth movement / keystrokes lead the voice by ~4s. Audio
|
||||
* is more vital than video — video adapts to audio's delay, never
|
||||
* the other way around. */
|
||||
try { if (ev.receiver) ev.receiver.playoutDelayHint = playoutDelayForRole(myRole); } catch(_){}
|
||||
try { if (ev.receiver) ev.receiver.jitterBufferTarget = playoutDelayForRole(myRole) * 1000; } catch(_){}
|
||||
/* register for the dynamic lip-sync algorithm — the next worklet
|
||||
* 'buffered' message will recompute this video receiver's target
|
||||
* to match the audio's total delay (native jbuf + worklet). */
|
||||
/* Video receivers: speakers/cohosts/hosts match audio's delay for
|
||||
* lip-sync. Listeners get playoutDelayHint=0 (browser-lowest)
|
||||
* so spotlight tile switches are instant — fox 2026-06-05: "we
|
||||
* should switch feeds immediately I don't want any algo slowing
|
||||
* that down." Tradeoff: listener mouths lead voice by ~audio
|
||||
* cushion (1.3–4s adaptive). Acceptable for content-consumption
|
||||
* mode where switching shares is more critical than per-syllable
|
||||
* lip-sync. */
|
||||
const vDelay = (myRole === 'listener') ? 0 : playoutDelayForRole(myRole);
|
||||
try { if (ev.receiver) ev.receiver.playoutDelayHint = vDelay; } catch(_){}
|
||||
try { if (ev.receiver) ev.receiver.jitterBufferTarget = vDelay * 1000; } catch(_){}
|
||||
/* Register for lip-sync — but the refresh loop also skips video
|
||||
* apply for listeners, so this is a no-op for listener tiles.
|
||||
* Kept registered in case role changes mid-session (promote). */
|
||||
if (ev.receiver) registerLipSyncVideo(pubHex, kind, ev.receiver);
|
||||
}
|
||||
/* MSID-supplant safety: the SFU re-uses the same streamID
|
||||
|
|
@ -5593,15 +5604,19 @@ function retargetAllReceivers(role){
|
|||
try { node.jbuf.port.postMessage({ cmd: 'retarget', targetSeconds: target }); } catch(_){}
|
||||
}
|
||||
}
|
||||
/* SFU sub PC receivers — audio (the native side, downstream of which
|
||||
* the worklet sits) AND video (which sits directly on the receiver). */
|
||||
/* SFU sub PC receivers. Audio gets the role-aware target (matching
|
||||
* the worklet downstream). Video for listeners gets 0 — instant tile
|
||||
* switching. Video for non-listeners matches audio. */
|
||||
const vTarget = role === 'listener' ? 0 : target;
|
||||
if (sfuSubPC && typeof sfuSubPC.getReceivers === 'function'){
|
||||
for (const r of sfuSubPC.getReceivers()){
|
||||
try { r.playoutDelayHint = target; } catch(_){}
|
||||
try { r.jitterBufferTarget = target * 1000; } catch(_){}
|
||||
const isVideo = r.track && r.track.kind === 'video';
|
||||
const t = isVideo ? vTarget : target;
|
||||
try { r.playoutDelayHint = t; } catch(_){}
|
||||
try { r.jitterBufferTarget = t * 1000; } catch(_){}
|
||||
}
|
||||
}
|
||||
logLine('', 'retarget all receivers → '+target+'s (role='+role+')');
|
||||
logLine('', 'retarget all receivers → aud='+target+'s vid='+vTarget+'s (role='+role+')');
|
||||
}
|
||||
|
||||
async function onRoleChanged(prev, next){
|
||||
|
|
@ -7081,8 +7096,8 @@ logLine('', 'ready — pick a handle, type a rendezvous code, enter the space');
|
|||
|
||||
<footer style="margin:2.2rem auto 0;font-size:0.65rem;color:#999;line-height:1.7;word-break:break-all;font-family:monospace">
|
||||
<span id="pi-seal" style="color:#777;cursor:default;user-select:none" title="">page integrity</span> · built <span class="stamp-date">2026-06-05</span><br>
|
||||
md5 <span class="stamp-md5">4591b92519ec28db69de15bb20eb2300</span><br>
|
||||
sha256 <span class="stamp-sha">0b792e6dd0d9bc69f888c239468c57506bc1d0feb9ded8b78bb650b886501c9b</span><br>
|
||||
md5 <span class="stamp-md5">563396c600c7710a1c213c2b22c314fc</span><br>
|
||||
sha256 <span class="stamp-sha">39e3a4bd33947eecfd33b8ae7cc552858cedef97b24a79e017ebb496b48f16ca</span><br>
|
||||
<span style="color:#bbb">hashes are of this page with these two fields zeroed — to verify, blank them and re-hash</span><br>
|
||||
<span style="color:#bbb">one self-contained file — <strong>save a copy</strong> and verify against these hashes; point at your own servers with ?signal= and ?turncred=, or <a href="host-your-own.html" style="color:#999">host your own community</a></span>
|
||||
</footer>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue