diff --git a/zebra-report/zebra-spaces.html b/zebra-report/zebra-spaces.html index 8848b55..6781985 100644 --- a/zebra-report/zebra-spaces.html +++ b/zebra-report/zebra-spaces.html @@ -1766,6 +1766,24 @@ const listenerAudioNodes = new Map(); /* uuid -> { src, gain, jbuf?, stream } */ * Loaded as a Blob URL because this is a single-file app — no * separate JS file shipped. */ const JITTER_BUFFER_WORKLET_CODE = ` +/* Adaptive jitter buffer with variable playback rate (Phase 3 of Double + * Dragon). The buffer's target depth is set from JS; the worklet + * automatically tunes its INTERNAL playback rate to drive the actual + * buffer level toward target. + * + * - buffered < target → stretchFactor > 1.0 (slow playback → buffer grows) + * - buffered > target → stretchFactor < 1.0 (fast playback → buffer shrinks) + * - buffered ≈ target → stretchFactor = 1.0 (normal) + * + * Rate change is capped at ±8% and ramped smoothly per-block so the + * pitch shift during adaptation is ≤1 semitone, brief, and centred. + * Listeners can disable stretching entirely via the 'lock_rate' command + * so music never gets resampled. + * + * Resampling: linear interpolation between adjacent input samples. Pure + * pitch-preserving WSOLA grain processing is ~10x more code; ship this + * first and upgrade if the brief pitch shift is audible enough to + * matter. */ class JitterBufferProcessor extends AudioWorkletProcessor { constructor(opts){ super(); @@ -1774,18 +1792,16 @@ class JitterBufferProcessor extends AudioWorkletProcessor { this.maxSeconds = o.maxSeconds || (this.targetSeconds * 1.5); this.targetSamples = Math.round(this.targetSeconds * sampleRate); this.maxSamples = Math.round(this.maxSeconds * sampleRate); - /* re-arm only after this many consecutive empty blocks. 128 samples - * per block at 48 kHz = 2.67 ms; 100 blocks ≈ 267 ms of silence. - * brief upstream drains (a single empty process() tick) MUST NOT - * tear down playback, or a 4s re-buffer kicks in every time — - * which is what made the phone choppy. */ this.rearmThresholdBlocks = 100; - /* periodic buffer-depth report so JS can drive the lip-sync - * algorithm (video.playoutDelayHint must match audio total delay - * including this worklet's cushion, else mouths move ~4s ahead - * of the words on listeners). 256 blocks * 128 samples / 48000Hz - * ≈ 683ms — frequent enough to track real changes, sparse enough - * to not flood the port. */ + /* Variable-rate playback state. cursor is the fractional sample + * index into the head of the queue. stretchFactor controls how + * fast the cursor advances per output sample. */ + this.cursor = 0; + this.stretchFactor = 1.0; + this.targetStretch = 1.0; + this.stretchRampPerBlock = 0.0002; /* ~0.075/sec ramp */ + this.maxStretch = 1.08; /* cap rate at ±8% */ + this.minStretch = 0.92; this.bufferedReportEvery = 256; this.bufferedReportCounter = 0; this.queue = []; @@ -1793,21 +1809,28 @@ class JitterBufferProcessor extends AudioWorkletProcessor { this.started = false; this.emptyStreak = 0; this.dropped = 0; - /* role-change retarget — JS posts {cmd:'retarget', targetSeconds} - * when the user is promoted/demoted; we recompute the sample - * targets and shrink the queue if the new max is smaller. */ this.port.onmessage = (e) => { - if (!e.data || e.data.cmd !== 'retarget') return; - const t = +e.data.targetSeconds; - if (!isFinite(t) || t <= 0) return; - this.targetSeconds = t; - this.maxSeconds = t * 1.5; - this.targetSamples = Math.round(this.targetSeconds * sampleRate); - this.maxSamples = Math.round(this.maxSeconds * sampleRate); - while (this.buffered > this.maxSamples && this.queue.length > 0){ - const drop = this.queue.shift(); - this.buffered -= drop[0].length; - this.dropped += drop[0].length; + if (!e.data) return; + if (e.data.cmd === 'retarget'){ + const t = +e.data.targetSeconds; + if (!isFinite(t) || t <= 0) return; + this.targetSeconds = t; + this.maxSeconds = t * 1.5; + this.targetSamples = Math.round(this.targetSeconds * sampleRate); + this.maxSamples = Math.round(this.maxSeconds * sampleRate); + while (this.buffered > this.maxSamples && this.queue.length > 0){ + const drop = this.queue.shift(); + this.buffered -= drop[0].length; + this.dropped += drop[0].length; + } + } else if (e.data.cmd === 'lock_rate'){ + /* listeners get this — explicitly forbid time-stretching so + * music playback stays at exactly 1.0 always. */ + const r = +e.data.rate; + if (isFinite(r) && r > 0){ + this.stretchFactor = this.targetStretch = r; + this.minStretch = this.maxStretch = r; + } } }; } @@ -1816,38 +1839,84 @@ class JitterBufferProcessor extends AudioWorkletProcessor { const outBlk = outputs[0]; if (!outBlk || outBlk.length === 0) return true; const nch = outBlk.length; - /* push the incoming block (must copy — host may reuse the buffer + const blkLen = outBlk[0].length; + /* push the incoming block (copy — host may reuse the buffer * after process() returns) */ if (inBlk && inBlk.length > 0 && inBlk[0] && inBlk[0].length > 0){ const copy = []; for (let c = 0; c < inBlk.length; c++) copy.push(new Float32Array(inBlk[c])); this.queue.push(copy); this.buffered += copy[0].length; - /* overflow guard — drop oldest if clock drift or network surge - * pushes us above the cap */ while (this.buffered > this.maxSamples && this.queue.length > 0){ const drop = this.queue.shift(); this.buffered -= drop[0].length; this.dropped += drop[0].length; } } - /* lock onto the buffer once it fills. Do NOT un-lock on a single - * empty queue tick — that's what made the phone choppy: any - * 2.67ms drain forced a full 4s re-buffer. emptyStreak tracks - * sustained silence and only re-arms after ~267ms. */ + /* Adaptive rate: target stretch from buffer depth ratio. */ + if (this.maxStretch !== this.minStretch){ + const ratio = this.buffered / Math.max(1, this.targetSamples); + if (ratio < 0.9){ + this.targetStretch = this.maxStretch; /* slow → grow */ + } else if (ratio > 1.1){ + this.targetStretch = this.minStretch; /* fast → shrink */ + } else { + this.targetStretch = 1.0; + } + if (this.stretchFactor < this.targetStretch){ + this.stretchFactor = Math.min(this.targetStretch, this.stretchFactor + this.stretchRampPerBlock); + } else if (this.stretchFactor > this.targetStretch){ + this.stretchFactor = Math.max(this.targetStretch, this.stretchFactor - this.stretchRampPerBlock); + } + } + /* lock onto buffer once it fills */ if (!this.started && this.buffered >= this.targetSamples){ this.started = true; - /* notify JS — listener UI stays in "buffering" state until the - * first started message arrives. */ try { this.port.postMessage({ cmd: 'started', targetSeconds: this.targetSeconds }); } catch(_){} } if (this.started && this.queue.length > 0){ - const head = this.queue.shift(); - this.buffered -= head[0].length; + const invStretch = 1.0 / this.stretchFactor; this.emptyStreak = 0; - for (let c = 0; c < nch; c++){ - const srcCh = head[c] || head[0]; /* mono → stereo: dup L→R */ - outBlk[c].set(srcCh.subarray(0, outBlk[c].length)); + /* generate blkLen output samples by reading at fractional cursor + * positions with linear interpolation between adjacent input + * samples. */ + for (let i = 0; i < blkLen; i++){ + const inFloat = this.cursor + i * invStretch; + const inIdx = Math.floor(inFloat); + const frac = inFloat - inIdx; + /* find which block + offset inIdx falls into */ + let offsetInBlock = inIdx; + let blockIdx = -1; + for (let b = 0; b < this.queue.length; b++){ + if (offsetInBlock < this.queue[b][0].length){ blockIdx = b; break; } + offsetInBlock -= this.queue[b][0].length; + } + if (blockIdx < 0){ + for (let c = 0; c < nch; c++) outBlk[c][i] = 0; + continue; + } + for (let c = 0; c < nch; c++){ + const srcCh = this.queue[blockIdx][c] || this.queue[blockIdx][0]; + const s1 = srcCh[offsetInBlock]; + let s2; + if (offsetInBlock + 1 < srcCh.length){ + s2 = srcCh[offsetInBlock + 1]; + } else if (blockIdx + 1 < this.queue.length){ + const nb = this.queue[blockIdx + 1]; + s2 = (nb[c] || nb[0])[0]; + } else { + s2 = s1; + } + outBlk[c][i] = s1 * (1 - frac) + s2 * frac; + } + } + this.cursor += blkLen * invStretch; + /* drop fully-consumed blocks from queue front, bring cursor back + * into the new head block's index space */ + while (this.queue.length > 0 && this.cursor >= this.queue[0][0].length){ + this.cursor -= this.queue[0][0].length; + this.buffered -= this.queue[0][0].length; + this.queue.shift(); } } else { for (let c = 0; c < nch; c++) outBlk[c].fill(0); @@ -1856,12 +1925,17 @@ class JitterBufferProcessor extends AudioWorkletProcessor { if (this.emptyStreak >= this.rearmThresholdBlocks){ this.started = false; this.emptyStreak = 0; + this.cursor = 0; } } } if (++this.bufferedReportCounter >= this.bufferedReportEvery){ this.bufferedReportCounter = 0; - try { this.port.postMessage({ cmd: 'buffered', seconds: this.buffered / sampleRate }); } catch(_){} + try { this.port.postMessage({ + cmd: 'buffered', + seconds: this.buffered / sampleRate, + stretchFactor: this.stretchFactor, + }); } catch(_){} } return true; } @@ -1911,12 +1985,25 @@ function installJitterBuffer(uuid, node){ onWorkletStarted(uuid); } else if (e.data.cmd === 'buffered'){ /* worklet's current buffer depth (seconds). Used by lip-sync - * to make video receivers track the audio's total delay. */ + * to make video receivers track the audio's total delay. + * stretchFactor is reported too — JS controller in Phase 3 + * uses it to drive the per-publisher target. */ const n = listenerAudioNodes.get(uuid); - if (n) n.bufferedSeconds = e.data.seconds; + if (n){ + n.bufferedSeconds = e.data.seconds; + n.stretchFactor = e.data.stretchFactor; + } refreshLipSyncForUuid(uuid); } }; + /* Listener-role worklets never time-stretch — music quality on the + * deep cushion is paramount, and the buffer drifts gradually with + * clock drift. Speaker / cohost / host worklets DO stretch + * adaptively to drive the per-publisher target the Double Dragon + * controller sets. */ + if (myRole === 'listener'){ + try { jbuf.port.postMessage({ cmd: 'lock_rate', rate: 1.0 }); } catch(_){} + } try { node.src.disconnect(node.gain); } catch(_){} node.src.connect(jbuf).connect(node.gain); node.jbuf = jbuf; @@ -2061,49 +2148,53 @@ async function refreshLipSyncForUuid(uuid){ } /* ================================================================== - * Double Dragon — twin-stream auto-engage controller. + * Double Dragon — adaptive time-stretching controller (Phase 3). * * Fox 2026-06-04 framing: "think of it like a CD that is literally - * being skipped in a physical disc man and solve it with two lasers - * one moving fast mesh as fast as possible and one for the broadcast - * also as fast as possible … double headed hydra! double dragon!" + * being skipped in a physical disc man … two lasers one moving fast + * mesh as fast as possible and one for the broadcast also as fast as + * possible dynamic based on … hardware performance and network + * performance and feed performance double headed hydra!" * - * Per-publisher health monitor. Every telemetry tick (5s), we look - * at the active mesh receiver's stats. If the publisher's network or - * encoder is hitting the listener with loss/jitter, auto-engage the - * HTTP /stream path (deeper buffer, glitch-free, ~2.5s behind). When - * the publisher's mesh path is clean for sustained time, auto- - * disengage and return to low-latency mesh. + * Per-publisher controller. Every 5s telemetry tick, we look at the + * active audio receiver's stats. If the publisher's network/encoder + * is hitting the listener with loss/jitter, we GROW the worklet's + * target buffer depth from DD_BASE_TARGET (0.5s, conversational) up + * toward DD_MAX_TARGET (4s, full wiggle cushion). The worklet's + * built-in adaptive resampling (±8% rate cap) slowly stretches + * playback to grow the actual buffer toward the new target without + * audible click or gap — pitch shifts by ≤1 semitone during the + * adaptation, returns to normal once the target is reached. + * + * When the publisher's mesh path is clean for sustained time, we + * SHRINK the target back to DD_BASE_TARGET. Worklet compresses + * playback (≤8% faster) to shrink the buffer. * * State per pubHex: - * 'mesh' — listening via worklet (mesh source), HTTP off - * 'http' — auto-engaged HTTP, worklet muted - * 'manual-on' — user toggled HTTP manually; we don't touch it - * 'manual-off' — user toggled HTTP off; we don't auto-engage + * 'auto' — controller-managed; target adjusts with stats + * 'manual-on' — user toggled HTTP /stream; controller hands off + * 'manual-off' — user toggled HTTP off; controller hands off * - * Transitions (auto only): - * mesh → http: 2 consecutive samples show instability - * http → mesh: 6 consecutive clean samples (~30s) + * Manual state is sticky — clicking the toggle overrides the + * controller until the user leaves + re-enters. * - * Manual state is sticky — if the user clicks the toggle, that - * intent overrides the controller until they leave + re-enter. - * - * The switch is audibly a ~2.5s time-jump (mesh latency vs HTTP - * latency). Listener hears past content briefly. Phase 3 (deferred) - * does sample-aligned dual-decode via cross-correlation to remove - * the jump. */ + * Phase 4 (deferred): grain-based PSOLA/WSOLA pitch-preservation in + * the worklet to remove the ≤1 semitone shift during adaptation. */ const doubleDragon = new Map(); /* pubHex → state */ const DD_LOSS_PER_SEC_THRESH = 2; /* > 2 loss/sec = unstable */ const DD_JITTER_THRESH_SEC = 0.030; /* > 30ms = unstable */ const DD_UNSTABLE_SAMPLES_NEEDED = 2; const DD_CLEAN_SAMPLES_NEEDED = 6; /* 6 × 5s = 30s clean → recover */ const DD_TELEMETRY_TICK_SEC = 5; +const DD_BASE_TARGET_SEC = 0.5; /* normal conversational cushion */ +const DD_MAX_TARGET_SEC = 4.0; /* maximum cushion under instability */ function ddEntry(pubHex){ let e = doubleDragon.get(pubHex); if (!e){ - e = { state: 'mesh', unstableStreak: 0, cleanStreak: 0, - lastLost: 0, lastSeenStatsAt: 0 }; + e = { state: 'auto', unstableStreak: 0, cleanStreak: 0, + lastLost: 0, lastSeenStatsAt: 0, + currentTargetSec: DD_BASE_TARGET_SEC }; doubleDragon.set(pubHex, e); } return e; @@ -2115,6 +2206,26 @@ function ddNoteManualToggle(pubHex, isOn){ logLine('', 'double-dragon pub='+pubHex.slice(0,4)+' → '+e.state+' (manual)'); } +/* Find the listener-audio node uuid for a publisher pubHex and post a + * retarget message to its worklet. The worklet's built-in rate-control + * loop will then stretch playback to drive its actual buffer toward + * the new target — no audible click, just a gradual time-stretch over + * ~5-10 seconds. */ +function ddSetTargetForPub(pubHex, targetSec){ + for (const [u, mm] of members){ + try { + if (mm.pubkey && hex(unb64(mm.pubkey)) === pubHex){ + const node = listenerAudioNodes.get(u); + if (node && node.jbuf && node.jbuf.port){ + node.targetSeconds = targetSec; + try { node.jbuf.port.postMessage({ cmd: 'retarget', targetSeconds: targetSec }); } catch(_){} + } + break; + } + } catch(_){} + } +} + async function evaluateDoubleDragonForPub(pubHex){ const e = doubleDragon.get(pubHex) || ddEntry(pubHex); /* manual state: hands off */ @@ -2125,14 +2236,6 @@ async function evaluateDoubleDragonForPub(pubHex){ const ls = lipSync.get(pubHex); const audRx = (ls && ls.audioReceiver) || sfuAudioReceivers.get(pubHex); if (!audRx) return; - /* find the uuid for the publisher (needed for start/stopStream) */ - let uuid = null; - for (const [u, mm] of members){ - try { - if (mm.pubkey && hex(unb64(mm.pubkey)) === pubHex){ uuid = u; break; } - } catch(_){} - } - if (!uuid) return; /* read stats */ let lossRate = 0, jitter = 0; try { @@ -2155,24 +2258,23 @@ async function evaluateDoubleDragonForPub(pubHex){ (jitter > DD_JITTER_THRESH_SEC); if (unstable){ e.unstableStreak++; e.cleanStreak = 0; - if (e.state === 'mesh' && e.unstableStreak >= DD_UNSTABLE_SAMPLES_NEEDED){ - e.state = 'http'; + if (e.unstableStreak >= DD_UNSTABLE_SAMPLES_NEEDED && + e.currentTargetSec < DD_MAX_TARGET_SEC){ + e.currentTargetSec = DD_MAX_TARGET_SEC; + ddSetTargetForPub(pubHex, DD_MAX_TARGET_SEC); logLine('', 'double-dragon pub='+pubHex.slice(0,4)+ - ' AUTO-ENGAGE HTTP (loss/s='+lossRate.toFixed(1)+ - ' jitter='+(jitter*1000).toFixed(0)+'ms)'); - streamMode.add(pubHex); - try { await startStream(uuid, pubHex); } catch(_){} - try { renderRoom(); } catch(_){} + ' GROW target → '+DD_MAX_TARGET_SEC+'s (loss/s='+ + lossRate.toFixed(1)+' jitter='+(jitter*1000).toFixed(0)+'ms)'); } } else { e.cleanStreak++; e.unstableStreak = 0; - if (e.state === 'http' && e.cleanStreak >= DD_CLEAN_SAMPLES_NEEDED){ - e.state = 'mesh'; + if (e.cleanStreak >= DD_CLEAN_SAMPLES_NEEDED && + e.currentTargetSec > DD_BASE_TARGET_SEC){ + e.currentTargetSec = DD_BASE_TARGET_SEC; + ddSetTargetForPub(pubHex, DD_BASE_TARGET_SEC); logLine('', 'double-dragon pub='+pubHex.slice(0,4)+ - ' AUTO-DISENGAGE (clean '+e.cleanStreak+' samples)'); - streamMode.delete(pubHex); - try { stopStream(uuid); } catch(_){} - try { renderRoom(); } catch(_){} + ' SHRINK target → '+DD_BASE_TARGET_SEC+'s (clean '+ + e.cleanStreak+' samples)'); } } } @@ -2487,20 +2589,14 @@ const VIDEO_REMOVE_MUTE_WINDOW_SCREEN_MS = 120000; * out wiggle-stalls without a glitch. Big enough to survive any * realistic publisher-side hiccup. */ const RECV_PLAYOUT_DELAY_SEC = 4.0; -/* Speaker/cohost/host cushion — empirically validated at 4s for - * 20-second wiggle absorption. Previous 0.5s was conversation-friendly - * but cohost-on-Fedora-Chrome glitched during host's X11 wiggles; - * smaller buffers untested. 4s pays a conversation-latency cost but - * gives same wiggle-immunity as listeners. Fox 2026-06-04: "we needed - * 4 secs before for the wiggle. it was at least 4 secs for 20 sec - * wiggles; we didn't test less." - * - * Future: twin-stream double-dragon controller (mesh + HTTP /stream - * running in parallel, adaptive switching based on mesh loss/jitter - * stats + hardware/network/feed performance). Until that lands, this - * single-source buffer is the floor that keeps everyone glitch-free - * under the most common stalls. */ -const SPEAKER_PLAYOUT_DELAY_SEC = 4.0; +/* Speaker/cohost/host INITIAL cushion — 0.5s for conversational + * latency, then the Double Dragon adaptive controller grows it to + * DD_MAX_TARGET_SEC (4s) when the publisher's network or encoder + * shows instability and shrinks it back to DD_BASE_TARGET_SEC (0.5s) + * during sustained clean periods. The worklet's ±8% rate-limited + * resampling makes the transition smooth: brief ≤1 semitone pitch + * shift during the ramp, silence-free, click-free. */ +const SPEAKER_PLAYOUT_DELAY_SEC = 0.5; /* HTTP /stream estimated end-to-end delay used for lip-sync when the * per-speaker stream toggle is ON for a publisher. The actual delay * varies (~1-3s depending on browser buffer + network), so this is a @@ -6694,8 +6790,8 @@ logLine('', 'ready — pick a handle, type a rendezvous code, enter the space');