zebra-spaces: deploy mesh-through-worklet + lip-sync receiver rebind
This commit is contained in:
parent
5dea6156ba
commit
01abc88446
1 changed files with 101 additions and 43 deletions
|
|
@ -1975,6 +1975,7 @@ function resetListenerBufferReady(){
|
|||
* the new median differs by more than LIP_SYNC_THRESHOLD from the
|
||||
* last applied value. */
|
||||
const lipSync = new Map(); /* pubHex → state */
|
||||
const sfuAudioReceivers = new Map(); /* pubHex → RTCRtpReceiver (SFU audio) — cached so we can restore as the lip-sync source after mesh fail */
|
||||
const LIP_SYNC_HISTORY = 5;
|
||||
const LIP_SYNC_THRESHOLD = 0.05; /* 50ms — below this is within perception noise */
|
||||
|
||||
|
|
@ -2119,21 +2120,44 @@ function detachListenerStream(uuid){
|
|||
listenerAudioNodes.delete(uuid);
|
||||
}
|
||||
|
||||
/* Smooth crossfade for the SFU worklet path. The worklet stays running
|
||||
/* In-place stream swap on an existing worklet. Both the SFU stream
|
||||
* and any mesh stream from the same publisher contain the same
|
||||
* encoded content at slightly different network delays — so when we
|
||||
* disconnect the old source from the worklet and connect a new one,
|
||||
* the worklet's queued samples continue emitting seamlessly while
|
||||
* the new source fills the queue. No buffer drain, no audible gap.
|
||||
*
|
||||
* This is how we make a mesh peer's audio benefit from the worklet's
|
||||
* cushion: route mesh through the same worklet the SFU was using, so
|
||||
* the 0.5s buffer absorbs the same wiggle-glitches it does for SFU.
|
||||
* Fox 2026-06-04: "the cohost on fedora chrome has the wiggle issue
|
||||
* when host is messing with tabs … can we sync the other stream and
|
||||
* recover the glitches assuming it is still buffering where the low-
|
||||
* latency version is glitched."
|
||||
*
|
||||
* Returns true on successful swap, false if no existing worklet. */
|
||||
function setWorkletStream(uuid, newStream){
|
||||
const node = listenerAudioNodes.get(uuid);
|
||||
if (!node || !audioCtx) return false;
|
||||
let newSrc;
|
||||
try { newSrc = audioCtx.createMediaStreamSource(newStream); }
|
||||
catch(e){ logLine('err', 'setWorkletStream src '+uuid.slice(0,4)+': '+e.message); return false; }
|
||||
try { node.src.disconnect(); } catch(_){}
|
||||
if (node.jbuf) newSrc.connect(node.jbuf);
|
||||
else newSrc.connect(node.gain);
|
||||
node.src = newSrc;
|
||||
node.stream = newStream;
|
||||
logLine('', 'worklet source swapped for '+uuid.slice(0,4));
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Smooth crossfade for the worklet path. The worklet stays running
|
||||
* (continues decoding samples at small CPU cost); only its GainNode
|
||||
* value moves. We use Web Audio's linearRampToValueAtTime for click-
|
||||
* free transitions — settings audio.volume directly causes audible
|
||||
* zipper noise on Android.
|
||||
* value moves. Used by the HTTP /stream toggle to cleanly cut the
|
||||
* worklet while the HTTP <audio> takes over (and back again).
|
||||
*
|
||||
* Called by:
|
||||
* - mesh ontrack → ramp to 0 (mesh <audio> takes over)
|
||||
* - mesh fail → ramp to 1 (SFU comes back, no reattach needed)
|
||||
* - startStream → ramp to 0 (HTTP /stream takes over)
|
||||
* - stopStream → ramp to 1 (back to live SFU)
|
||||
*
|
||||
* Default 100ms ramp = below the gap between syllables, masks the
|
||||
* different time-offsets of the two sources (SFU worklet ~0.5s vs
|
||||
* mesh ~50ms). Listener barely notices the pivot. */
|
||||
* SFU↔mesh transitions don't use this anymore — they share the same
|
||||
* worklet via setWorkletStream above (single buffer, swapped source). */
|
||||
function rampWorkletGain(uuid, target, durationMs){
|
||||
const node = listenerAudioNodes.get(uuid);
|
||||
if (!node || !node.gain) return;
|
||||
|
|
@ -3400,8 +3424,17 @@ function handleRemoteSfuTrack(ev){
|
|||
* matching video target on the same publisher's screen/camera
|
||||
* receivers. Register here (mic kind) BEFORE attachSfuTrack
|
||||
* so the first worklet 'buffered' message lands on a paired
|
||||
* publisher state. */
|
||||
if (ev.receiver) registerLipSyncAudio(pubHex, uuid, ev.receiver);
|
||||
* publisher state.
|
||||
*
|
||||
* Cache the SFU receiver separately too — when a mesh ontrack
|
||||
* swaps the worklet's source to mesh, we re-register lip-sync
|
||||
* with the mesh receiver, and on mesh fail we restore from
|
||||
* this cache. Lip-sync video always tracks the CURRENT audio
|
||||
* source's jbuf. */
|
||||
if (ev.receiver){
|
||||
registerLipSyncAudio(pubHex, uuid, ev.receiver);
|
||||
sfuAudioReceivers.set(pubHex, ev.receiver);
|
||||
}
|
||||
/* speakers get their peers' audio via mesh (lower latency)
|
||||
* AT THE TIMES THE MESH PC IS CONNECTED. A stale or failing
|
||||
* mesh PC must NOT block the SFU fallback — that's how the
|
||||
|
|
@ -5109,30 +5142,42 @@ async function connectToPeer(uuid, weOffer){
|
|||
for (const tr of micStream.getTracks()){ tagTrack(tr); pc.addTrack(tr, micStream); }
|
||||
setSenderBitrate(pc.getSenders().find(s=>s.track && s.track.kind==='audio'));
|
||||
pc.ontrack = (ev) => {
|
||||
/* Smooth pivot SFU → mesh. We DON'T tear down the SFU worklet
|
||||
* path — it keeps decoding samples in the background. Instead
|
||||
* we ramp its GainNode to 0 over 100ms while the mesh <audio>
|
||||
* starts unmuted. On mesh failure later, ramping SFU gain back
|
||||
* to 1 restores audio without any reattach work. Fox 2026-06-04:
|
||||
* "a speaker should be able to pivot smoothly between the two
|
||||
* feeds." */
|
||||
rampWorkletGain(uuid, 0, 100);
|
||||
let a = remoteAudio.get(uuid);
|
||||
if (!a){
|
||||
a = document.createElement('audio'); a.autoplay = true;
|
||||
document.body.appendChild(a); remoteAudio.set(uuid, a);
|
||||
applySinkTo(a);
|
||||
/* SFU and mesh both feed the SAME worklet now. The SFU stream
|
||||
* was already routed there by attachSfuTrack; we swap in the
|
||||
* mesh stream so the worklet's buffer absorbs wiggle-glitches
|
||||
* for mesh too. Without this, mesh's native ~50ms buffer was no
|
||||
* cushion at all — a 200ms host stall caused mesh listeners to
|
||||
* glitch while SFU listeners didn't. Now both share the worklet
|
||||
* cushion (0.5s for speakers). Source swap is seamless because
|
||||
* the worklet's queue holds 0.5s of decoded samples and both
|
||||
* sources contain identical content at slightly different
|
||||
* network delays. */
|
||||
const stream = ev.streams[0] || new MediaStream([ev.track]);
|
||||
if (!setWorkletStream(uuid, stream)){
|
||||
/* fallback: no existing worklet (AudioContext failed at SFU
|
||||
* attach time). Create one fresh with mesh as the source.
|
||||
* Small initial silence while the buffer fills. */
|
||||
attachAudioStreamViaWorklet(uuid, stream, SPEAKER_PLAYOUT_DELAY_SEC);
|
||||
}
|
||||
a.srcObject = ev.streams[0] || new MediaStream([ev.track]);
|
||||
/* mesh path is peer-to-peer between two speakers (you'd never be
|
||||
* in mesh as a pure listener). Always conversational latency
|
||||
* here — fixed at SPEAKER_PLAYOUT_DELAY_SEC, no role check
|
||||
* needed. */
|
||||
try { ev.receiver.playoutDelayHint = SPEAKER_PLAYOUT_DELAY_SEC; } catch(_){}
|
||||
try { ev.receiver.jitterBufferTarget = SPEAKER_PLAYOUT_DELAY_SEC * 1000; } catch(_){}
|
||||
try { a.muted = false; } catch(_){}
|
||||
stopMeter(uuid); startMeter(uuid, a.srcObject);
|
||||
logLine('', 'mesh audio attached for '+uuid.slice(0,4)+' — SFU worklet faded out');
|
||||
/* lip-sync: rebind the audio receiver for this publisher to the
|
||||
* MESH receiver since mesh is now what's feeding the worklet.
|
||||
* Its jbuf (~50ms native) + worklet (0.5s) ≈ ~0.55s total audio
|
||||
* delay. Video receivers will re-target to match on the next
|
||||
* worklet 'buffered' message. Fox 2026-06-04: "video would need
|
||||
* to be slid depending on the mode to keep it in sync." */
|
||||
try {
|
||||
const mm = members.get(uuid);
|
||||
const pubHex = mm && mm.pubkey ? hex(unb64(mm.pubkey)) : null;
|
||||
if (pubHex && ev.receiver) registerLipSyncAudio(pubHex, uuid, ev.receiver);
|
||||
} catch(_){}
|
||||
stopMeter(uuid); startMeter(uuid, stream);
|
||||
logLine('', 'mesh stream swapped into worklet for '+uuid.slice(0,4)+' — buffer cushion now applies to mesh too');
|
||||
};
|
||||
pc.onicecandidate = (ev) => { /* using waitForIceGathering pattern, candidates ignored */ };
|
||||
pc.onconnectionstatechange = () => {
|
||||
|
|
@ -5147,15 +5192,28 @@ async function connectToPeer(uuid, weOffer){
|
|||
const attempts = (peerMeshRetries.get(uuid) || 0) + 1;
|
||||
peerMeshRetries.set(uuid, attempts);
|
||||
tearPeer(uuid);
|
||||
/* Mesh PC just died. The SFU worklet path was never torn down —
|
||||
* we just faded its GainNode to 0 when mesh took over. Ramp
|
||||
* it back to 1 (100ms) so audio returns smoothly. The legacy
|
||||
* attachCachedSfuStreamFor is still called as a fallback in
|
||||
* case the worklet wasn't present (e.g. AudioContext failed
|
||||
* earlier and we landed on the <audio>-element fallback path
|
||||
* at attach time). */
|
||||
rampWorkletGain(uuid, 1, 100);
|
||||
try { attachCachedSfuStreamFor(uuid); } catch(_){}
|
||||
/* Mesh PC died. Swap the worklet's source back to the cached
|
||||
* SFU stream so the user keeps hearing the publisher without a
|
||||
* hiccup. The worklet's existing queue covers the swap latency
|
||||
* — by the time the queue drains 0.5s of (now-stale) mesh
|
||||
* samples, SFU samples are flowing in. attachCachedSfuStreamFor
|
||||
* is kept as a fallback for the no-worklet case (AudioContext
|
||||
* failed earlier). */
|
||||
try {
|
||||
const mm = members.get(uuid);
|
||||
const pubHex = mm && mm.pubkey ? hex(unb64(mm.pubkey)) : null;
|
||||
const sfuStream = pubHex ? sfuStreamsByPubHex.get(pubHex) : null;
|
||||
if (sfuStream && setWorkletStream(uuid, sfuStream)){
|
||||
/* lip-sync: rebind back to the SFU audio receiver. Worklet
|
||||
* source is SFU again → video should target SFU's native
|
||||
* jbuf + worklet (~0.5s + 0.5s ≈ 1s) instead of mesh's. */
|
||||
const sfuRx = pubHex ? sfuAudioReceivers.get(pubHex) : null;
|
||||
if (sfuRx) registerLipSyncAudio(pubHex, uuid, sfuRx);
|
||||
logLine('', 'mesh failed → worklet swapped back to SFU stream for '+uuid.slice(0,4));
|
||||
} else {
|
||||
attachCachedSfuStreamFor(uuid);
|
||||
}
|
||||
} catch(_){ try { attachCachedSfuStreamFor(uuid); } catch(_){} }
|
||||
|
||||
if (attempts >= PEER_MESH_MAX_RETRIES){
|
||||
peerMeshGiveUp.add(uuid);
|
||||
|
|
@ -6434,8 +6492,8 @@ logLine('', 'ready — pick a handle, type a rendezvous code, enter the space');
|
|||
|
||||
<footer style="margin:2.2rem auto 0;font-size:0.65rem;color:#999;line-height:1.7;word-break:break-all;font-family:monospace">
|
||||
<span id="pi-seal" style="color:#777;cursor:default;user-select:none" title="">page integrity</span> · built <span class="stamp-date">2026-06-04</span><br>
|
||||
md5 <span class="stamp-md5">f77d16e2213f40156ea05cf0983587b4</span><br>
|
||||
sha256 <span class="stamp-sha">2ad85516c4eaee2daa910790f3e9e8fe626339ccbd5704fa4d0a49b429bffba2</span><br>
|
||||
md5 <span class="stamp-md5">801600028d330022de2e867f4490802d</span><br>
|
||||
sha256 <span class="stamp-sha">abc6ba58d773e53243dab444b1fb5d4ddd75cecdace78ecadfe5a77f6f8eef55</span><br>
|
||||
<span style="color:#bbb">hashes are of this page with these two fields zeroed — to verify, blank them and re-hash</span><br>
|
||||
<span style="color:#bbb">one self-contained file — <strong>save a copy</strong> and verify against these hashes; point at your own servers with ?signal= and ?turncred=, or <a href="host-your-own.html" style="color:#999">host your own community</a></span>
|
||||
</footer>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue