zebra-spaces: speaker buffer 4s + HTTP /stream lip-sync override
Two changes: 1. SPEAKER_PLAYOUT_DELAY_SEC: 0.5 → 4.0. Cohost on Fedora Chrome was glitching during host's X11 wiggles because 0.5s mesh cushion was shorter than typical wiggle stalls. Fox 2026-06-04: "we needed 4 secs before for the wiggle. it was at least 4 secs for 20 sec wiggles; we didn't test less." Speaker conversation latency goes from ~500ms to ~4s round-trip — accepted trade for wiggle-immunity. Future twin-stream double-dragon controller (mesh + HTTP /stream parallel, adaptive switching based on observed loss/jitter + hardware/network/feed performance) will reclaim conversational latency while keeping the glitch floor. 2. HTTP /stream lip-sync override. When the per-speaker stream toggle is ON for a publisher, audio comes from an <audio> element with its own deep buffer (~2.5s), not the worklet. Previously the lip-sync algorithm kept driving from the (silent) worklet's measurement → video target stayed at the worklet's ~4s while audio actually was ~2.5s → 1.5s mouth- leads-voice. Now startStream sets httpLipSyncOverride[pubHex] = HTTP_STREAM_DELAY_SEC and forces an immediate retarget via applyLipSyncForPub. stopStream clears the override and the next worklet 'buffered' message restores worklet-driven targeting. Constants kept conservative — HTTP delay is hardcoded at 2.5s; could be made dynamic by reading audio.buffered.end(0) - audio.currentTime plus an Ogg-granule offset, but the spread is small relative to the 80ms perception threshold. Fox 2026-06-04 framing: "think of it like a CD that is literally being skipped in a physical disc man and solve it with two lasers one moving fast mesh as fast as possible and one for the broadcast also as fast as possible dynamic based on not hardware fingerprint but hardware performance and network performance and feed performance double headed hydra! double dragon!" — Phase 2 lands the auto-engage controller per that spec.
This commit is contained in:
parent
c3ff58ca39
commit
e0bf064629
1 changed files with 86 additions and 14 deletions
|
|
@ -1976,6 +1976,7 @@ function resetListenerBufferReady(){
|
|||
* last applied value. */
|
||||
const lipSync = new Map(); /* pubHex → state */
|
||||
const sfuAudioReceivers = new Map(); /* pubHex → RTCRtpReceiver (SFU audio) — cached so we can restore as the lip-sync source after mesh fail */
|
||||
const httpLipSyncOverride = new Map(); /* pubHex → fixed delay (sec). When set, lip-sync uses this directly and skips worklet-derived measurement — for HTTP /stream toggle ON case where the audio source isn't the worklet anymore. */
|
||||
const LIP_SYNC_HISTORY = 5;
|
||||
const LIP_SYNC_THRESHOLD = 0.05; /* 50ms — below this is within perception noise */
|
||||
|
||||
|
|
@ -2009,7 +2010,26 @@ async function refreshLipSyncForUuid(uuid){
|
|||
for (const [ph, st] of lipSync){
|
||||
if (st.audioUuid === uuid){ pubHex = ph; e = st; break; }
|
||||
}
|
||||
if (!e || !e.audioReceiver || e.videoReceivers.size === 0) return;
|
||||
if (!e || e.videoReceivers.size === 0) return;
|
||||
/* HTTP /stream override path: when the per-speaker stream toggle is
|
||||
* ON for this publisher, the audio is coming from an <audio>
|
||||
* element with its own deep buffer instead of the worklet. We don't
|
||||
* have a clean way to read that element's effective delay, so we
|
||||
* use a fixed HTTP_STREAM_DELAY_SEC estimate (set by startStream).
|
||||
* Apply it directly with the same threshold/hysteresis as the
|
||||
* worklet path. */
|
||||
const override = httpLipSyncOverride.get(pubHex);
|
||||
if (typeof override === 'number'){
|
||||
if (Math.abs(override - e.lastApplied) < LIP_SYNC_THRESHOLD) return;
|
||||
for (const [kind, rx] of e.videoReceivers){
|
||||
try { rx.playoutDelayHint = override; } catch(_){}
|
||||
try { rx.jitterBufferTarget = override * 1000; } catch(_){}
|
||||
}
|
||||
e.lastApplied = override;
|
||||
logLine('', 'lip-sync pub='+pubHex.slice(0,4)+' delay='+override.toFixed(2)+'s (HTTP /stream override)');
|
||||
return;
|
||||
}
|
||||
if (!e.audioReceiver) return;
|
||||
const node = listenerAudioNodes.get(uuid);
|
||||
if (!node) return;
|
||||
const workletBufferedSec = node.bufferedSeconds || 0;
|
||||
|
|
@ -2040,6 +2060,25 @@ async function refreshLipSyncForUuid(uuid){
|
|||
logLine('', 'lip-sync pub='+pubHex.slice(0,4)+' delay='+med.toFixed(2)+'s ('+e.videoReceivers.size+' video rx, history median of '+e.history.length+')');
|
||||
}
|
||||
|
||||
/* Force a one-shot lip-sync application for a specific publisher.
|
||||
* Bypasses the worklet-driven 'buffered' cadence — used when an
|
||||
* external event (HTTP /stream toggle) changes the audio source and
|
||||
* we want the video target to update immediately rather than wait
|
||||
* for the next worklet message. */
|
||||
function applyLipSyncForPub(pubHex){
|
||||
const e = lipSync.get(pubHex);
|
||||
if (!e || e.videoReceivers.size === 0) return;
|
||||
const override = httpLipSyncOverride.get(pubHex);
|
||||
if (typeof override !== 'number') return;
|
||||
for (const [kind, rx] of e.videoReceivers){
|
||||
try { rx.playoutDelayHint = override; } catch(_){}
|
||||
try { rx.jitterBufferTarget = override * 1000; } catch(_){}
|
||||
}
|
||||
e.lastApplied = override;
|
||||
e.history = [];
|
||||
logLine('', 'lip-sync pub='+pubHex.slice(0,4)+' forced delay='+override.toFixed(2)+'s (HTTP /stream toggle)');
|
||||
}
|
||||
|
||||
/* Shared audio attach path. Every role routes through here now so the
|
||||
* worklet can apply role-appropriate buffer depth uniformly. Listener
|
||||
* gets a fat 4s cushion (lean-back, latency doesn't matter); speakers
|
||||
|
|
@ -2324,12 +2363,28 @@ const VIDEO_REMOVE_MUTE_WINDOW_SCREEN_MS = 120000;
|
|||
* out wiggle-stalls without a glitch. Big enough to survive any
|
||||
* realistic publisher-side hiccup. */
|
||||
const RECV_PLAYOUT_DELAY_SEC = 4.0;
|
||||
/* Speaker/cohost/host cushion — they're in active conversation, so
|
||||
* the buffer trades a tiny bit of jitter smoothing for sub-second
|
||||
* round-trip. ~500ms feels natural; 4s would make every back-and-forth
|
||||
* impossible. Native receiver jitterBufferTarget honors this for
|
||||
* voice-rate Opus and the worklet matches it for music-rate. */
|
||||
const SPEAKER_PLAYOUT_DELAY_SEC = 0.5;
|
||||
/* Speaker/cohost/host cushion — empirically validated at 4s for
|
||||
* 20-second wiggle absorption. Previous 0.5s was conversation-friendly
|
||||
* but cohost-on-Fedora-Chrome glitched during host's X11 wiggles;
|
||||
* smaller buffers untested. 4s pays a conversation-latency cost but
|
||||
* gives same wiggle-immunity as listeners. Fox 2026-06-04: "we needed
|
||||
* 4 secs before for the wiggle. it was at least 4 secs for 20 sec
|
||||
* wiggles; we didn't test less."
|
||||
*
|
||||
* Future: twin-stream double-dragon controller (mesh + HTTP /stream
|
||||
* running in parallel, adaptive switching based on mesh loss/jitter
|
||||
* stats + hardware/network/feed performance). Until that lands, this
|
||||
* single-source buffer is the floor that keeps everyone glitch-free
|
||||
* under the most common stalls. */
|
||||
const SPEAKER_PLAYOUT_DELAY_SEC = 4.0;
|
||||
/* HTTP /stream estimated end-to-end delay used for lip-sync when the
|
||||
* per-speaker stream toggle is ON for a publisher. The actual delay
|
||||
* varies (~1-3s depending on browser buffer + network), so this is a
|
||||
* conservative middle estimate. We could derive it more precisely
|
||||
* from audio.buffered.end(0) - audio.currentTime + an Ogg-granule
|
||||
* offset, but the spread is small enough relative to lip-sync
|
||||
* tolerance (~80ms) that a fixed value suffices. */
|
||||
const HTTP_STREAM_DELAY_SEC = 2.5;
|
||||
function playoutDelayForRole(role){
|
||||
return role === 'listener' ? RECV_PLAYOUT_DELAY_SEC : SPEAKER_PLAYOUT_DELAY_SEC;
|
||||
}
|
||||
|
|
@ -5485,6 +5540,13 @@ async function startStream(uuid, pubHex){
|
|||
rampWorkletGain(uuid, 0, 100);
|
||||
const meshEl = remoteAudio.get(uuid);
|
||||
if (meshEl) try { meshEl.muted = true; } catch(_){}
|
||||
/* Lip-sync: video target locks to HTTP's deeper buffer while the
|
||||
* stream toggle is ON. applyLipSyncForPub forces an immediate
|
||||
* update so the video catches up to audio rather than drifting
|
||||
* for 4s waiting for the worklet's next 'buffered' message. The
|
||||
* worklet isn't audible during HTTP mode anyway. */
|
||||
httpLipSyncOverride.set(pubHex, HTTP_STREAM_DELAY_SEC);
|
||||
applyLipSyncForPub(pubHex);
|
||||
let a = streamAudio.get(uuid);
|
||||
/* Idempotent: second toggle-on for the same pubHex while we're
|
||||
* already loading/playing is a no-op. Resetting .src abort the
|
||||
|
|
@ -5550,12 +5612,22 @@ function stopStream(uuid){
|
|||
* while DJ mode was active. */
|
||||
const w = remoteAudio.get(uuid);
|
||||
if (w) try { w.muted = false; } catch(_){}
|
||||
/* Ramp the SFU worklet back up (100ms). If mesh is currently active
|
||||
* for this peer, the mesh <audio> element above carries the audible
|
||||
* output and the worklet plays into a destination that nobody hears
|
||||
* — same as before, no behavior change. If mesh isn't active, the
|
||||
* SFU worklet IS the live path and the ramp restores audio. */
|
||||
/* Ramp the worklet back up (100ms). The worklet is the live audio
|
||||
* path now (mesh or SFU stream feeding it via setWorkletStream). */
|
||||
rampWorkletGain(uuid, 1, 100);
|
||||
/* Drop the HTTP lip-sync override so the worklet-driven update
|
||||
* takes over again on the next 'buffered' message. The video
|
||||
* target will retarget back from ~2.5s to whatever audio total
|
||||
* delay the worklet currently has (~4s for a speaker on mesh). */
|
||||
try {
|
||||
const mm = members.get(uuid);
|
||||
const pubHex = mm && mm.pubkey ? hex(unb64(mm.pubkey)) : null;
|
||||
if (pubHex){
|
||||
httpLipSyncOverride.delete(pubHex);
|
||||
const e = lipSync.get(pubHex);
|
||||
if (e) e.history = []; /* discard stale samples so the next median is fresh */
|
||||
}
|
||||
} catch(_){}
|
||||
}
|
||||
function toggleStreamFor(uuid, pubHex){
|
||||
if (streamMode.has(pubHex)){
|
||||
|
|
@ -6492,8 +6564,8 @@ logLine('', 'ready — pick a handle, type a rendezvous code, enter the space');
|
|||
|
||||
<footer style="margin:2.2rem auto 0;font-size:0.65rem;color:#999;line-height:1.7;word-break:break-all;font-family:monospace">
|
||||
<span id="pi-seal" style="color:#777;cursor:default;user-select:none" title="">page integrity</span> · built <span class="stamp-date">2026-06-04</span><br>
|
||||
md5 <span class="stamp-md5">801600028d330022de2e867f4490802d</span><br>
|
||||
sha256 <span class="stamp-sha">abc6ba58d773e53243dab444b1fb5d4ddd75cecdace78ecadfe5a77f6f8eef55</span><br>
|
||||
md5 <span class="stamp-md5">bb1d3911fdc9a91535d0042523fbd5be</span><br>
|
||||
sha256 <span class="stamp-sha">becc93e6ff940bcebf1b3423c945090f971f1889f7a44730448797350b086fc2</span><br>
|
||||
<span style="color:#bbb">hashes are of this page with these two fields zeroed — to verify, blank them and re-hash</span><br>
|
||||
<span style="color:#bbb">one self-contained file — <strong>save a copy</strong> and verify against these hashes; point at your own servers with ?signal= and ?turncred=, or <a href="host-your-own.html" style="color:#999">host your own community</a></span>
|
||||
</footer>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue