From e0bf0646296c7e3ab26ad0e2ef52e5b77db9012f Mon Sep 17 00:00:00 2001
From: Russell Ballestrini <russell.ballestrini@gmail.com>
Date: Thu, 4 Jun 2026 18:21:49 -0400
Subject: [PATCH] zebra-spaces: speaker buffer 4s + HTTP /stream lip-sync
 override
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two changes:

1. SPEAKER_PLAYOUT_DELAY_SEC: 0.5 → 4.0.
   Cohost on Fedora Chrome was glitching during host's X11 wiggles
   because 0.5s mesh cushion was shorter than typical wiggle stalls.
   Fox 2026-06-04: "we needed 4 secs before for the wiggle. it was at
   least 4 secs for 20 sec wiggles; we didn't test less." Speaker
   conversation latency goes from ~500ms to ~4s round-trip — accepted
   trade for wiggle-immunity. Future twin-stream double-dragon
   controller (mesh + HTTP /stream parallel, adaptive switching based
   on observed loss/jitter + hardware/network/feed performance) will
   reclaim conversational latency while keeping the glitch floor.

2. HTTP /stream lip-sync override.
   When the per-speaker stream toggle is ON for a publisher, audio
   comes from an <audio> element with its own deep buffer (~2.5s),
   not the worklet. Previously the lip-sync algorithm kept driving
   from the (silent) worklet's measurement → video target stayed at
   the worklet's ~4s while audio actually was ~2.5s → 1.5s mouth-
   leads-voice. Now startStream sets httpLipSyncOverride[pubHex] =
   HTTP_STREAM_DELAY_SEC and forces an immediate retarget via
   applyLipSyncForPub. stopStream clears the override and the next
   worklet 'buffered' message restores worklet-driven targeting.

Constants kept conservative — HTTP delay is hardcoded at 2.5s; could
be made dynamic by reading audio.buffered.end(0) - audio.currentTime
plus an Ogg-granule offset, but the spread is small relative to the
80ms perception threshold.

Fox 2026-06-04 framing: "think of it like a CD that is literally
being skipped in a physical disc man and solve it with two lasers
one moving fast mesh as fast as possible and one for the broadcast
also as fast as possible dynamic based on not hardware fingerprint
but hardware performance and network performance and feed performance
double headed hydra! double dragon!" — Phase 2 lands the auto-engage
controller per that spec.
---
 web/zebra-spaces.html | 100 ++++++++++++++++++++++++++++++++++++------
 1 file changed, 86 insertions(+), 14 deletions(-)

diff --git a/web/zebra-spaces.html b/web/zebra-spaces.html
index 37fc6f5..8d1c46d 100644
--- a/web/zebra-spaces.html
+++ b/web/zebra-spaces.html
@@ -1976,6 +1976,7 @@ function resetListenerBufferReady(){
  * last applied value. */
 const lipSync = new Map(); /* pubHex → state */
 const sfuAudioReceivers = new Map(); /* pubHex → RTCRtpReceiver (SFU audio) — cached so we can restore as the lip-sync source after mesh fail */
+const httpLipSyncOverride = new Map(); /* pubHex → fixed delay (sec). When set, lip-sync uses this directly and skips worklet-derived measurement — for HTTP /stream toggle ON case where the audio source isn't the worklet anymore. */
 const LIP_SYNC_HISTORY = 5;
 const LIP_SYNC_THRESHOLD = 0.05;   /* 50ms — below this is within perception noise */
 
@@ -2009,7 +2010,26 @@ async function refreshLipSyncForUuid(uuid){
   for (const [ph, st] of lipSync){
     if (st.audioUuid === uuid){ pubHex = ph; e = st; break; }
   }
-  if (!e || !e.audioReceiver || e.videoReceivers.size === 0) return;
+  if (!e || e.videoReceivers.size === 0) return;
+  /* HTTP /stream override path: when the per-speaker stream toggle is
+   * ON for this publisher, the audio is coming from an <audio>
+   * element with its own deep buffer instead of the worklet. We don't
+   * have a clean way to read that element's effective delay, so we
+   * use a fixed HTTP_STREAM_DELAY_SEC estimate (set by startStream).
+   * Apply it directly with the same threshold/hysteresis as the
+   * worklet path. */
+  const override = httpLipSyncOverride.get(pubHex);
+  if (typeof override === 'number'){
+    if (Math.abs(override - e.lastApplied) < LIP_SYNC_THRESHOLD) return;
+    for (const [kind, rx] of e.videoReceivers){
+      try { rx.playoutDelayHint    = override; } catch(_){}
+      try { rx.jitterBufferTarget  = override * 1000; } catch(_){}
+    }
+    e.lastApplied = override;
+    logLine('', 'lip-sync pub='+pubHex.slice(0,4)+' delay='+override.toFixed(2)+'s (HTTP /stream override)');
+    return;
+  }
+  if (!e.audioReceiver) return;
   const node = listenerAudioNodes.get(uuid);
   if (!node) return;
   const workletBufferedSec = node.bufferedSeconds || 0;
@@ -2040,6 +2060,25 @@ async function refreshLipSyncForUuid(uuid){
   logLine('', 'lip-sync pub='+pubHex.slice(0,4)+' delay='+med.toFixed(2)+'s ('+e.videoReceivers.size+' video rx, history median of '+e.history.length+')');
 }
 
+/* Force a one-shot lip-sync application for a specific publisher.
+ * Bypasses the worklet-driven 'buffered' cadence — used when an
+ * external event (HTTP /stream toggle) changes the audio source and
+ * we want the video target to update immediately rather than wait
+ * for the next worklet message. */
+function applyLipSyncForPub(pubHex){
+  const e = lipSync.get(pubHex);
+  if (!e || e.videoReceivers.size === 0) return;
+  const override = httpLipSyncOverride.get(pubHex);
+  if (typeof override !== 'number') return;
+  for (const [kind, rx] of e.videoReceivers){
+    try { rx.playoutDelayHint    = override; } catch(_){}
+    try { rx.jitterBufferTarget  = override * 1000; } catch(_){}
+  }
+  e.lastApplied = override;
+  e.history = [];
+  logLine('', 'lip-sync pub='+pubHex.slice(0,4)+' forced delay='+override.toFixed(2)+'s (HTTP /stream toggle)');
+}
+
 /* Shared audio attach path. Every role routes through here now so the
  * worklet can apply role-appropriate buffer depth uniformly. Listener
  * gets a fat 4s cushion (lean-back, latency doesn't matter); speakers
@@ -2324,12 +2363,28 @@ const VIDEO_REMOVE_MUTE_WINDOW_SCREEN_MS = 120000;
  * out wiggle-stalls without a glitch. Big enough to survive any
  * realistic publisher-side hiccup. */
 const RECV_PLAYOUT_DELAY_SEC = 4.0;
-/* Speaker/cohost/host cushion — they're in active conversation, so
- * the buffer trades a tiny bit of jitter smoothing for sub-second
- * round-trip. ~500ms feels natural; 4s would make every back-and-forth
- * impossible. Native receiver jitterBufferTarget honors this for
- * voice-rate Opus and the worklet matches it for music-rate. */
-const SPEAKER_PLAYOUT_DELAY_SEC = 0.5;
+/* Speaker/cohost/host cushion — empirically validated at 4s for
+ * 20-second wiggle absorption. Previous 0.5s was conversation-friendly
+ * but cohost-on-Fedora-Chrome glitched during host's X11 wiggles;
+ * smaller buffers untested. 4s pays a conversation-latency cost but
+ * gives same wiggle-immunity as listeners. Fox 2026-06-04: "we needed
+ * 4 secs before for the wiggle. it was at least 4 secs for 20 sec
+ * wiggles; we didn't test less."
+ *
+ * Future: twin-stream double-dragon controller (mesh + HTTP /stream
+ * running in parallel, adaptive switching based on mesh loss/jitter
+ * stats + hardware/network/feed performance). Until that lands, this
+ * single-source buffer is the floor that keeps everyone glitch-free
+ * under the most common stalls. */
+const SPEAKER_PLAYOUT_DELAY_SEC = 4.0;
+/* HTTP /stream estimated end-to-end delay used for lip-sync when the
+ * per-speaker stream toggle is ON for a publisher. The actual delay
+ * varies (~1-3s depending on browser buffer + network), so this is a
+ * conservative middle estimate. We could derive it more precisely
+ * from audio.buffered.end(0) - audio.currentTime + an Ogg-granule
+ * offset, but the spread is small enough relative to lip-sync
+ * tolerance (~80ms) that a fixed value suffices. */
+const HTTP_STREAM_DELAY_SEC = 2.5;
 function playoutDelayForRole(role){
   return role === 'listener' ? RECV_PLAYOUT_DELAY_SEC : SPEAKER_PLAYOUT_DELAY_SEC;
 }
@@ -5485,6 +5540,13 @@ async function startStream(uuid, pubHex){
   rampWorkletGain(uuid, 0, 100);
   const meshEl = remoteAudio.get(uuid);
   if (meshEl) try { meshEl.muted = true; } catch(_){}
+  /* Lip-sync: video target locks to HTTP's deeper buffer while the
+   * stream toggle is ON. applyLipSyncForPub forces an immediate
+   * update so the video catches up to audio rather than drifting
+   * for 4s waiting for the worklet's next 'buffered' message. The
+   * worklet isn't audible during HTTP mode anyway. */
+  httpLipSyncOverride.set(pubHex, HTTP_STREAM_DELAY_SEC);
+  applyLipSyncForPub(pubHex);
   let a = streamAudio.get(uuid);
   /* Idempotent: second toggle-on for the same pubHex while we're
    * already loading/playing is a no-op. Resetting .src abort the
@@ -5550,12 +5612,22 @@ function stopStream(uuid){
    * while DJ mode was active. */
   const w = remoteAudio.get(uuid);
   if (w) try { w.muted = false; } catch(_){}
-  /* Ramp the SFU worklet back up (100ms). If mesh is currently active
-   * for this peer, the mesh <audio> element above carries the audible
-   * output and the worklet plays into a destination that nobody hears
-   * — same as before, no behavior change. If mesh isn't active, the
-   * SFU worklet IS the live path and the ramp restores audio. */
+  /* Ramp the worklet back up (100ms). The worklet is the live audio
+   * path now (mesh or SFU stream feeding it via setWorkletStream). */
   rampWorkletGain(uuid, 1, 100);
+  /* Drop the HTTP lip-sync override so the worklet-driven update
+   * takes over again on the next 'buffered' message. The video
+   * target will retarget back from ~2.5s to whatever audio total
+   * delay the worklet currently has (~4s for a speaker on mesh). */
+  try {
+    const mm = members.get(uuid);
+    const pubHex = mm && mm.pubkey ? hex(unb64(mm.pubkey)) : null;
+    if (pubHex){
+      httpLipSyncOverride.delete(pubHex);
+      const e = lipSync.get(pubHex);
+      if (e) e.history = [];   /* discard stale samples so the next median is fresh */
+    }
+  } catch(_){}
 }
 function toggleStreamFor(uuid, pubHex){
   if (streamMode.has(pubHex)){
@@ -6492,8 +6564,8 @@ logLine('', 'ready — pick a handle, type a rendezvous code, enter the space');
 
 <footer style="margin:2.2rem auto 0;font-size:0.65rem;color:#999;line-height:1.7;word-break:break-all;font-family:monospace">
   <span id="pi-seal" style="color:#777;cursor:default;user-select:none" title="">page integrity</span> &nbsp;·&nbsp; built <span class="stamp-date">2026-06-04</span><br>
-  md5 <span class="stamp-md5">801600028d330022de2e867f4490802d</span><br>
-  sha256 <span class="stamp-sha">abc6ba58d773e53243dab444b1fb5d4ddd75cecdace78ecadfe5a77f6f8eef55</span><br>
+  md5 <span class="stamp-md5">bb1d3911fdc9a91535d0042523fbd5be</span><br>
+  sha256 <span class="stamp-sha">becc93e6ff940bcebf1b3423c945090f971f1889f7a44730448797350b086fc2</span><br>
   <span style="color:#bbb">hashes are of this page with these two fields zeroed — to verify, blank them and re-hash</span><br>
   <span style="color:#bbb">one self-contained file — <strong>save a copy</strong> and verify against these hashes; point at your own servers with ?signal= and ?turncred=, or <a href="host-your-own.html" style="color:#999">host your own community</a></span>
 </footer>