From 1d74af874fa562b6b128cc36735f40f62b14b769 Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Thu, 4 Jun 2026 18:26:17 -0400 Subject: [PATCH] zebra-spaces: Phase 2 Double Dragon auto-engage controller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-publisher health monitor runs once per 5s telemetry tick. Samples the active audio receiver's lost/jitter; transitions a per-pubHex state machine between 'mesh' (worklet) and 'http' (HTTP /stream). State machine: mesh (default) — listening via worklet (mesh source) http — auto-engaged HTTP /stream; worklet muted by existing startStream gain-ramp manual-on — user toggled HTTP manually; controller hands off manual-off — user toggled HTTP off; controller hands off Auto transitions (manual states never auto-flip): mesh → http: 2 consecutive samples show loss > 2/s OR jitter > 30ms http → mesh: 6 consecutive clean samples (~30s) Manual state set on toggleStreamFor — clicking the per-speaker toggle records the user's intent; controller respects it until they leave + re-enter. Auto engagement reuses the existing infrastructure: - startStream(uuid, pubHex) sets streamMode + ramps worklet to 0 + locks lip-sync override to HTTP_STREAM_DELAY_SEC - stopStream(uuid) clears the override + ramps worklet back Audible switch: ~2.5s time-jump per direction (mesh delay vs HTTP delay). Listener briefly hears past content on mesh→http or future content on http→mesh. Phase 3 (sample-aligned dual-decode via cross-correlation, fed through a 2-input worklet) eliminates the jump but needs a dedicated session. Telemetry will show: - "double-dragon pub=XXXX AUTO-ENGAGE HTTP (loss/s=N jitter=Nms)" - "double-dragon pub=XXXX AUTO-DISENGAGE (clean N samples)" - "double-dragon pub=XXXX → manual-on (manual)" on user toggle --- web/zebra-spaces.html | 134 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 2 deletions(-) diff --git a/web/zebra-spaces.html b/web/zebra-spaces.html index 8d1c46d..8848b55 100644 --- a/web/zebra-spaces.html +++ b/web/zebra-spaces.html @@ -2060,6 +2060,130 @@ async function refreshLipSyncForUuid(uuid){ logLine('', 'lip-sync pub='+pubHex.slice(0,4)+' delay='+med.toFixed(2)+'s ('+e.videoReceivers.size+' video rx, history median of '+e.history.length+')'); } +/* ================================================================== + * Double Dragon — twin-stream auto-engage controller. + * + * Fox 2026-06-04 framing: "think of it like a CD that is literally + * being skipped in a physical disc man and solve it with two lasers + * one moving fast mesh as fast as possible and one for the broadcast + * also as fast as possible … double headed hydra! double dragon!" + * + * Per-publisher health monitor. Every telemetry tick (5s), we look + * at the active mesh receiver's stats. If the publisher's network or + * encoder is hitting the listener with loss/jitter, auto-engage the + * HTTP /stream path (deeper buffer, glitch-free, ~2.5s behind). When + * the publisher's mesh path is clean for sustained time, auto- + * disengage and return to low-latency mesh. + * + * State per pubHex: + * 'mesh' — listening via worklet (mesh source), HTTP off + * 'http' — auto-engaged HTTP, worklet muted + * 'manual-on' — user toggled HTTP manually; we don't touch it + * 'manual-off' — user toggled HTTP off; we don't auto-engage + * + * Transitions (auto only): + * mesh → http: 2 consecutive samples show instability + * http → mesh: 6 consecutive clean samples (~30s) + * + * Manual state is sticky — if the user clicks the toggle, that + * intent overrides the controller until they leave + re-enter. + * + * The switch is audibly a ~2.5s time-jump (mesh latency vs HTTP + * latency). Listener hears past content briefly. Phase 3 (deferred) + * does sample-aligned dual-decode via cross-correlation to remove + * the jump. */ +const doubleDragon = new Map(); /* pubHex → state */ +const DD_LOSS_PER_SEC_THRESH = 2; /* > 2 loss/sec = unstable */ +const DD_JITTER_THRESH_SEC = 0.030; /* > 30ms = unstable */ +const DD_UNSTABLE_SAMPLES_NEEDED = 2; +const DD_CLEAN_SAMPLES_NEEDED = 6; /* 6 × 5s = 30s clean → recover */ +const DD_TELEMETRY_TICK_SEC = 5; + +function ddEntry(pubHex){ + let e = doubleDragon.get(pubHex); + if (!e){ + e = { state: 'mesh', unstableStreak: 0, cleanStreak: 0, + lastLost: 0, lastSeenStatsAt: 0 }; + doubleDragon.set(pubHex, e); + } + return e; +} +function ddNoteManualToggle(pubHex, isOn){ + const e = ddEntry(pubHex); + e.state = isOn ? 'manual-on' : 'manual-off'; + e.unstableStreak = 0; e.cleanStreak = 0; + logLine('', 'double-dragon pub='+pubHex.slice(0,4)+' → '+e.state+' (manual)'); +} + +async function evaluateDoubleDragonForPub(pubHex){ + const e = doubleDragon.get(pubHex) || ddEntry(pubHex); + /* manual state: hands off */ + if (e.state === 'manual-on' || e.state === 'manual-off') return; + /* find the publisher's audio receiver — prefer mesh receiver from + * lipSync entry (which gets updated to whichever source is currently + * feeding the worklet). Fall back to SFU. */ + const ls = lipSync.get(pubHex); + const audRx = (ls && ls.audioReceiver) || sfuAudioReceivers.get(pubHex); + if (!audRx) return; + /* find the uuid for the publisher (needed for start/stopStream) */ + let uuid = null; + for (const [u, mm] of members){ + try { + if (mm.pubkey && hex(unb64(mm.pubkey)) === pubHex){ uuid = u; break; } + } catch(_){} + } + if (!uuid) return; + /* read stats */ + let lossRate = 0, jitter = 0; + try { + const stats = await audRx.getStats(); + stats.forEach(r => { + if (r.type === 'inbound-rtp' && r.kind === 'audio'){ + const nowMs = Date.now(); + const dtSec = e.lastSeenStatsAt + ? (nowMs - e.lastSeenStatsAt) / 1000 + : DD_TELEMETRY_TICK_SEC; + const lostDelta = Math.max(0, (r.packetsLost|0) - (e.lastLost|0)); + lossRate = dtSec > 0 ? lostDelta / dtSec : 0; + e.lastLost = (r.packetsLost|0); + e.lastSeenStatsAt = nowMs; + jitter = r.jitter || 0; + } + }); + } catch(_){ return; } + const unstable = (lossRate > DD_LOSS_PER_SEC_THRESH) || + (jitter > DD_JITTER_THRESH_SEC); + if (unstable){ + e.unstableStreak++; e.cleanStreak = 0; + if (e.state === 'mesh' && e.unstableStreak >= DD_UNSTABLE_SAMPLES_NEEDED){ + e.state = 'http'; + logLine('', 'double-dragon pub='+pubHex.slice(0,4)+ + ' AUTO-ENGAGE HTTP (loss/s='+lossRate.toFixed(1)+ + ' jitter='+(jitter*1000).toFixed(0)+'ms)'); + streamMode.add(pubHex); + try { await startStream(uuid, pubHex); } catch(_){} + try { renderRoom(); } catch(_){} + } + } else { + e.cleanStreak++; e.unstableStreak = 0; + if (e.state === 'http' && e.cleanStreak >= DD_CLEAN_SAMPLES_NEEDED){ + e.state = 'mesh'; + logLine('', 'double-dragon pub='+pubHex.slice(0,4)+ + ' AUTO-DISENGAGE (clean '+e.cleanStreak+' samples)'); + streamMode.delete(pubHex); + try { stopStream(uuid); } catch(_){} + try { renderRoom(); } catch(_){} + } + } +} + +function evaluateDoubleDragon(){ + /* run once per telemetry tick for every publisher we know about */ + for (const pubHex of lipSync.keys()){ + evaluateDoubleDragonForPub(pubHex).catch(()=>{}); + } +} + /* Force a one-shot lip-sync application for a specific publisher. * Bypasses the worklet-driven 'buffered' cadence — used when an * external event (HTTP /stream toggle) changes the audio source and @@ -4361,6 +4485,10 @@ function stopAlivePings(){ const TELEMETRY_INTERVAL_MS = 5000; let telemetryTimer = null; async function dumpTelemetry(){ + /* Run the Double Dragon controller off the same 5s cadence — same + * getStats traffic is being read anyway, evaluating per-publisher + * stability here is essentially free. */ + try { evaluateDoubleDragon(); } catch(_){} const parts = []; parts.push('role=' + myRole); parts.push('sub=' + (sfuSubPC ? sfuSubPC.connectionState + '/' + sfuSubPC.iceConnectionState : 'none')); @@ -5633,9 +5761,11 @@ function toggleStreamFor(uuid, pubHex){ if (streamMode.has(pubHex)){ streamMode.delete(pubHex); stopStream(uuid); + ddNoteManualToggle(pubHex, false); } else { streamMode.add(pubHex); startStream(uuid, pubHex); + ddNoteManualToggle(pubHex, true); } renderRoom(); } @@ -6564,8 +6694,8 @@ logLine('', 'ready — pick a handle, type a rendezvous code, enter the space');