diff --git a/Makefile b/Makefile index d4949ed..fb83fee 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,15 @@ test-listener-audio: test-sendbye: @node test/sendbye-fsm.test.js +# JitterBufferProcessor worklet — extracts the inline template literal +# from the page + sandboxes it in Node, then drives process() + port +# messages directly. Pins the shrink-retarget contract: on a +# listener-to-speaker promotion the queue must drop to targetSamples +# (not 1.5×target) and the processor must report the dropped sample +# count back to JS so paired video receivers can advance in lockstep. +test-jitter-worklet: + @node test/jitter-buffer-worklet.test.js + # zebra-spaces JS↔Go protocol parity + vault + ed25519 + (optionally) a live # server flow. The live-server tier auto-runs when proxy.unturf.com sits # alongside this checkout AND has a Go toolchain — we build the relay binary @@ -96,7 +105,7 @@ test-zebra-spaces: ZEBRA_SPACES_BINARY=$$bin node test/zebra-spaces.test.js; \ rc=$$?; rm -f /tmp/zspc-signal-test; exit $$rc -test-all: test/unit test/integration test/functional test-web test-fsm test-video-removal test-mesh test-mod-actions test-self-listener test-listener-audio test-sendbye test-zebra-spaces +test-all: test/unit test/integration test/functional test-web test-fsm test-video-removal test-mesh test-mod-actions test-self-listener test-listener-audio test-sendbye test-jitter-worklet test-zebra-spaces @echo "--- unit ---" @./test/unit @echo "--- integration ---" @@ -119,6 +128,8 @@ test-all: test/unit test/integration test/functional test-web test-fsm test-vide @node test/listener-audio-attach.test.js @echo "--- sendBye FSM ---" @node test/sendbye-fsm.test.js + @echo "--- jitter-buffer worklet ---" + @node test/jitter-buffer-worklet.test.js @echo "--- zebra-spaces ---" @$(MAKE) -s test-zebra-spaces diff --git a/test/jitter-buffer-worklet.test.js b/test/jitter-buffer-worklet.test.js new file mode 100644 index 0000000..fb817fc --- /dev/null +++ b/test/jitter-buffer-worklet.test.js @@ -0,0 +1,219 @@ +#!/usr/bin/env node +/* jitter-buffer worklet processor tests. + * + * node test/jitter-buffer-worklet.test.js + * + * Pins the JITTER_BUFFER_WORKLET_CODE processor contract: + * + * - retarget shrinking (e.g. listener-to-speaker promotion, 4s→0.5s): + * drop the queue down to targetSamples (NOT 1.5×target) and post + * a 'dropped' message back to JS so paired video receivers can + * advance the matching amount. Fox 2026-06-06: "if we skip ahead + * from whatever listener is at to speaker speed, we need to make + * sure the video skips ahead the same amount or rate to keep the + * lips synced". The shrink contract eliminates the residual 6% + * speed-up phase fox heard as "janky" during promotion. + * + * - retarget growing (speaker-to-listener demotion, 0.5s→4s): no + * drop (we don't have extra audio to throw away; the algo grows + * into the new target naturally). + * + * - retarget to same target: no-op drop. + * + * - 'started' posted on first fill to target. + * + * - 'buffered' periodic reports. + * + * - 'lock_rate' freezes stretchFactor (listener music mode). + * + * The worklet code is a template literal in the page. We extract it + * verbatim, inject sampleRate + registerProcessor + a port spy, and + * drive process() / message events directly. */ + +const fs = require('fs'); +const path = require('path'); +const src = fs.readFileSync(path.join(__dirname, '..', 'web', 'zebra-spaces.html'), 'utf8'); + +/* extract the `const JITTER_BUFFER_WORKLET_CODE = \`...\`;` literal */ +function extractTemplate(name){ + const head = new RegExp('const\\s+' + name + '\\s*=\\s*`'); + const m = src.match(head); + if (!m) throw new Error('could not find template ' + name); + const startBacktick = src.indexOf('`', m.index); + const endBacktick = src.indexOf('`;', startBacktick + 1); + if (startBacktick < 0 || endBacktick < 0) throw new Error('malformed template literal ' + name); + return src.slice(startBacktick + 1, endBacktick); +} + +const workletCode = extractTemplate('JITTER_BUFFER_WORKLET_CODE'); + +/* The worklet expects: + * - global `sampleRate` (provided by AudioWorkletGlobalScope) + * - global `registerProcessor(name, classRef)` + * - `AudioWorkletProcessor` base class (just needs `this.port`) + * + * In Node we build a stub world and `Function`-eval the code in it. */ +function buildWorld(sampleRate){ + const registered = {}; + function registerProcessor(name, cls){ registered[name] = cls; } + + /* AudioWorkletProcessor: constructor sets `this.port = portStub`. + * The processor's constructor calls super(), so we initialize the + * port here. */ + class AudioWorkletProcessor { + constructor(){ this.port = { onmessage: null, posted: [], postMessage(m){ this.posted.push(m); } }; } + } + + const factory = new Function( + 'sampleRate', 'registerProcessor', 'AudioWorkletProcessor', + workletCode + '\nreturn null;' + ); + factory(sampleRate, registerProcessor, AudioWorkletProcessor); + return registered; +} + +const SR = 48000; +const world = buildWorld(SR); +const JitterBufferProcessor = world['jitter-buffer']; +if (!JitterBufferProcessor) throw new Error('jitter-buffer processor did not register'); + +/* Helper: create a processor instance with options, build a fake input + * block, and run process() N times feeding it. Each input block has + * blkLen samples per channel (Web Audio standard 128). */ +function makeProcessor(targetSeconds){ + const p = new JitterBufferProcessor({ + processorOptions: { targetSeconds, maxSeconds: targetSeconds * 1.5 }, + }); + return p; +} + +const BLK = 128; +function inputBlock(channels){ + const ch = []; + for (let c = 0; c < channels; c++) ch.push(new Float32Array(BLK)); + return [ch]; /* inputs[0] = first input, channels */ +} +function outputBlock(channels){ + const ch = []; + for (let c = 0; c < channels; c++) ch.push(new Float32Array(BLK)); + return [ch]; +} +/* Feed N input blocks through process so the queue grows by ~N*BLK samples */ +function feed(p, n, channels){ + channels = channels || 2; + for (let i = 0; i < n; i++){ + p.process(inputBlock(channels), outputBlock(channels)); + } +} + +let pass = 0, fail = 0; +function test(name, fn){ + try { fn(); console.log(' ✓ ' + name); pass++; } + catch (e){ console.log(' ✗ ' + name + ' — ' + (e.message || e)); fail++; } +} +function eq(a, b, msg){ + if (a !== b) throw new Error((msg || 'expected') + ' — got ' + JSON.stringify(a) + ' want ' + JSON.stringify(b)); +} +function approxEq(a, b, tol, msg){ + if (Math.abs(a - b) > tol) throw new Error((msg || 'approxEq') + ' — got ' + a + ' want ~' + b + ' (tol=' + tol + ')'); +} +function truthy(v, msg){ if (!v) throw new Error(msg || 'expected truthy'); } +function ge(a, b, msg){ if (!(a >= b)) throw new Error((msg || 'expected >=') + ' — got ' + a + ' >= ' + b); } +function le(a, b, msg){ if (!(a <= b)) throw new Error((msg || 'expected <=') + ' — got ' + a + ' <= ' + b); } + +console.log('jitter-buffer worklet processor:'); + +/* -------- 'started' message on first fill -------- */ +test('"started" message fires once when the buffer reaches targetSamples', () => { + const p = makeProcessor(0.5); /* 0.5s × 48k = 24000 samples target */ + const blocks = Math.ceil(p.targetSamples / BLK) + 1; + feed(p, blocks); + const startedMsgs = p.port.posted.filter(m => m.cmd === 'started'); + eq(startedMsgs.length, 1, 'one started'); + eq(startedMsgs[0].targetSeconds, 0.5, 'reports target'); +}); + +/* -------- 'buffered' messages -------- */ +test('"buffered" messages report current depth periodically', () => { + const p = makeProcessor(0.5); + /* push enough blocks to cross the report-every threshold */ + feed(p, 400); + const buf = p.port.posted.filter(m => m.cmd === 'buffered'); + ge(buf.length, 1, 'at least one buffered'); + ge(buf[0].seconds, 0, 'seconds reported'); +}); + +/* -------- retarget SHRINK drops to targetSamples + reports 'dropped' -------- */ +test('retarget SHRINK (4s → 0.5s) drops queue to targetSamples and posts "dropped" with sample count', () => { + /* fill to ~4s, then retarget to 0.5s, assert the drop went all the + * way down to targetSamples (NOT 1.5×target=0.75s) and the dropped + * message carries the skip amount. */ + const p = makeProcessor(4.0); + const blocks = Math.ceil(4.0 * SR / BLK); /* ~1500 blocks */ + feed(p, blocks + 5); /* a little over target */ + /* now retarget */ + p.port.onmessage({ data: { cmd: 'retarget', targetSeconds: 0.5 } }); + /* targetSamples = 0.5 * 48000 = 24000 */ + le(p.buffered, p.targetSamples + BLK, 'buffered now within one block of targetSamples'); + ge(p.buffered, 0, 'buffered non-negative'); + /* a 'dropped' message must have fired */ + const dropMsgs = p.port.posted.filter(m => m.cmd === 'dropped'); + eq(dropMsgs.length, 1, 'one dropped message'); + ge(dropMsgs[0].samples, 1, 'samples > 0'); +}); + +test('retarget SHRINK does NOT leave a 1.5×target overhang (the residual that triggered the "janky 6%" phase)', () => { + const p = makeProcessor(4.0); + feed(p, Math.ceil(4.0 * SR / BLK) + 10); + p.port.onmessage({ data: { cmd: 'retarget', targetSeconds: 0.5 } }); + /* after a shrink, buffered should be at most targetSamples + one + * block — explicitly NOT 1.5×targetSamples. Pre-fix the drop was + * `while (buffered > maxSamples)` which left up to 0.75s in the + * queue → ~4s of 6% catch-up playback. */ + le(p.buffered, p.targetSamples + BLK, 'no 1.5× overhang'); +}); + +/* -------- retarget GROW does NOT drop -------- */ +test('retarget GROW (0.5s → 4s) does NOT drop anything', () => { + const p = makeProcessor(0.5); + feed(p, Math.ceil(0.5 * SR / BLK) + 5); + const buffBefore = p.buffered; + p.port.onmessage({ data: { cmd: 'retarget', targetSeconds: 4.0 } }); + eq(p.buffered, buffBefore, 'buffered unchanged on grow'); + const dropMsgs = p.port.posted.filter(m => m.cmd === 'dropped'); + eq(dropMsgs.length, 0, 'no dropped message'); +}); + +test('retarget to SAME target is a no-op (no drop, no message)', () => { + const p = makeProcessor(0.5); + feed(p, Math.ceil(0.5 * SR / BLK) + 5); + const buffBefore = p.buffered; + p.port.onmessage({ data: { cmd: 'retarget', targetSeconds: 0.5 } }); + eq(p.buffered, buffBefore, 'buffered unchanged on same-target retarget'); + const dropMsgs = p.port.posted.filter(m => m.cmd === 'dropped'); + eq(dropMsgs.length, 0, 'no dropped message'); +}); + +/* -------- retarget validation -------- */ +test('retarget with bogus targetSeconds is ignored (no NaN, no negative)', () => { + const p = makeProcessor(4.0); + feed(p, 50); + const targetBefore = p.targetSeconds; + p.port.onmessage({ data: { cmd: 'retarget', targetSeconds: NaN } }); + p.port.onmessage({ data: { cmd: 'retarget', targetSeconds: -1 } }); + p.port.onmessage({ data: { cmd: 'retarget', targetSeconds: 0 } }); + eq(p.targetSeconds, targetBefore, 'target unchanged'); +}); + +/* -------- lock_rate (listener music mode) -------- */ +test('lock_rate=1.0 pins stretchFactor at 1.0 and disables ±6% adaptation', () => { + const p = makeProcessor(4.0); + p.port.onmessage({ data: { cmd: 'lock_rate', rate: 1.0 } }); + eq(p.stretchFactor, 1.0); + eq(p.minStretch, 1.0, 'min === max → no adaptation'); + eq(p.maxStretch, 1.0); +}); + +console.log(''); +console.log((fail === 0 ? 'PASS' : 'FAIL') + ' — ' + pass + ' passed, ' + fail + ' failed'); +process.exit(fail === 0 ? 0 : 1); diff --git a/test/listener-audio-attach.test.js b/test/listener-audio-attach.test.js index 6b47e91..a8ebc29 100644 --- a/test/listener-audio-attach.test.js +++ b/test/listener-audio-attach.test.js @@ -308,11 +308,20 @@ function makeBrowser(role){ 'const VIDEO_REMOVE_MUTE_WINDOW_MS = 5000;\n' + 'const VIDEO_REMOVE_MUTE_WINDOW_SCREEN_MS = 8000;\n' + handleTrackSrc + '\n' + + /* test seam: seed lipSync.videoReceivers so the worklet 'dropped' + * handler has receivers to zero out. lipSync is in factory scope, + * so callers can't reach it directly. */ + 'function __seedLipSync(pubHex, kindRxMap){\n' + + ' let e = lipSync.get(pubHex);\n' + + ' if (!e){ e = { videoReceivers: new Map(), lastApplied: 0 }; lipSync.set(pubHex, e); }\n' + + ' for (const k in kindRxMap) e.videoReceivers.set(k, kindRxMap[k]);\n' + + '}\n' + 'return {\n' + ' attachAudioStreamViaWorklet, attachListenerStreamViaAudioContext,\n' + ' attachSfuTrack, flushSfuStreams, setWorkletStream, detachListenerStream,\n' + ' handleRemoteSfuTrack,\n' + ' triggerWorkletReady, setRole,\n' + + ' __seedLipSync,\n' + ' get audioCtx(){ return audioCtx; },\n' + ' get workletReady(){ return workletReady; },\n' + ' get audioCtxCreates(){ return audioCtxCreates; },\n' + @@ -805,6 +814,221 @@ test('listener: attach when audioCtx is already RUNNING does not call resume()', falsy(resumeCalled, 'resume() not called on running ctx'); }); +/* =================== video-coupling on shrink retarget =================== */ + +test('worklet "dropped" message: zeros jitterBufferTarget on every paired video receiver (audio-video lip-sync survives the skip)', () => { + /* The worklet posts {cmd:'dropped',samples:N} after a shrinking + * retarget (listener-to-speaker promotion). The JS handler in + * installJitterBuffer must walk the publisher's lipSync entry and + * force every video receiver's jitterBufferTarget + playoutDelayHint + * to 0 so the browser drops frames to match. Without this, audio + * jumps 3.25s forward instantly + video drains gradually = broken + * lip-sync. fox 2026-06-06: "if we skip ahead from whatever + * listener is at to speaker speed, we need to make sure the video + * skips ahead the same amount or rate to keep the lips synced." */ + const b = makeBrowser('listener'); + addMember(b, UUID_A, PUB_A); + b.api.attachAudioStreamViaWorklet(UUID_A, makeStream(), RECV); + b.api.triggerWorkletReady(); + const node = b.api.listenerAudioNodes.get(UUID_A); + truthy(node.jbuf, 'jbuf installed'); + + /* register two video receivers for the publisher (camera + screen), + * mirroring registerLipSyncVideo. Each receiver is a stub with + * mutable jitterBufferTarget + playoutDelayHint. */ + const camRx = { jitterBufferTarget: 100, playoutDelayHint: 0.5 }; + const scrRx = { jitterBufferTarget: 100, playoutDelayHint: 0.5 }; + /* drive registerLipSyncVideo via the lipSync state directly since + * the function isn't extracted into this sandbox. */ + b.api.members.get(UUID_A); + /* the lipSync map is in factory scope; expose via a known shape. We + * mirror lipSyncEntry by inserting a pubHex entry. */ + /* eslint-disable */ + const factoryHack = new Function( + 'b', 'PUB_A', 'camRx', 'scrRx', + 'this.lipSync || null; return null;' + ); + /* eslint-enable */ + /* Simpler: call a small helper we attach to the API for tests. */ + b.api.__seedLipSyncForTest = b.api.__seedLipSyncForTest || function(pubHex, kindRxPairs){ + /* no-op stub; we rely on the path below instead */ + }; + + /* Drive the dropped-message handler by posting to the worklet's + * port. The handler closures over `uuid` (UUID_A here). It looks + * up members → pubHex → lipSync entry → video receivers. We seed + * the lipSync map by reaching into the sandbox via the test + * helper exposed below. */ + /* lipSync map seed: insert a manual entry under PUB_A with both + * receivers wired in. */ + /* We need a sandbox-side helper to do this since lipSync is closed + * over. Add `__seedLip` to the api factory's return object. */ + /* The simplest path: the listener-audio sandbox already exposes + * b.api by capture; we attach the seed via b.api.__internalLipMap() + * once. */ + + /* Fall back to a direct factory-side seed. We can't introspect the + * closure-bound lipSync map from the outside, so we test the + * behavior indirectly: ensure the worklet handler doesn't throw + * even when lipSync is empty (defensive path), AND ensure that + * when we seed the map via the API surface, receivers get zeroed. */ + + /* Path 1: empty lipSync — must not throw. */ + let threw = false; + try { + node.jbuf.port.onmessage({ data: { cmd: 'dropped', samples: 192000 } }); + } catch(_){ threw = true; } + falsy(threw, 'dropped handler is defensive against missing lipSync entry'); + + /* Path 2: seed lipSync from inside the sandbox via __seedLipSync helper + * that we add in makeBrowser (see harness extension below). */ + if (b.api.__seedLipSync){ + b.api.__seedLipSync(PUB_A, { camera: camRx, screen: scrRx }); + node.jbuf.port.onmessage({ data: { cmd: 'dropped', samples: 192000 } }); + eq(camRx.jitterBufferTarget, 0, 'camera receiver target zeroed'); + eq(scrRx.jitterBufferTarget, 0, 'screen receiver target zeroed'); + eq(camRx.playoutDelayHint, 0, 'camera playoutDelayHint zeroed'); + eq(scrRx.playoutDelayHint, 0, 'screen playoutDelayHint zeroed'); + } +}); + +/* =================== speaker mesh+SFU double-attach =================== */ +/* On a speaker, handleRemoteSfuTrack's mic path checks the peer's + * mesh PC state and SKIPS the SFU attach if mesh is already + * 'connected'. This is the contract that keeps a speaker from getting + * two parallel chains (SFU + mesh) for the same publisher. Fox kicked + * fxhp-phone-as-speaker (hearing double); kick didn't clear it; + * closing Firefox + re-entering as listener cleared it. Speaker-role + * mesh+SFU collision is the suspect path — these tests pin the + * single-chain invariant. */ + +test('speaker: SFU ontrack arrives, mesh PC NOT connected → attach proceeds (mesh would take over later via setWorkletStream)', () => { + const b = makeBrowser('speaker'); + addMember(b, UUID_A, PUB_A); + /* no mesh PC yet at all */ + const stream = makeStream(); + b.api.handleRemoteSfuTrack(fakeOntrack(PUB_A, 'mic', stream)); + truthy(b.api.listenerAudioNodes.has(UUID_A), 'chain built via SFU'); + const node = b.api.listenerAudioNodes.get(UUID_A); + eq(node.targetSeconds, SPEAKER, '0.5s target for speaker'); + truthy(reachable(node.src, b.ctx.destination), 'reaches destination'); +}); + +test('speaker: SFU ontrack with mesh peer state=connected → SFU attach SKIPPED (no double-chain with mesh)', () => { + /* Pins the canSpeak()+meshState branch at handleRemoteSfuTrack + * ~line 4760. If this guard regresses, every speaker with a mesh + * peer ends up with two chains for the same publisher: one via + * SFU, one via mesh. fox: "fxhp-phone hearing double as speaker." + * Closing Firefox cleared it because the rebuilt PC didn't race + * the SFU ontrack as harshly. */ + const b = makeBrowser('speaker'); + addMember(b, UUID_A, PUB_A); + b.api.peers.set(UUID_A, { connectionState: 'connected' }); + const stream = makeStream(); + b.api.handleRemoteSfuTrack(fakeOntrack(PUB_A, 'mic', stream)); + falsy(b.api.listenerAudioNodes.has(UUID_A), 'no SFU chain — mesh owns this peer'); + /* but cache is still primed so a later mesh-fail can recover */ + truthy(b.api.sfuStreamsByPubHex.size > 0, 'SFU stream cached for fallback'); +}); + +test('speaker: SFU ontrack with mesh peer state=failed → attach PROCEEDS (mesh is dead, SFU must take over)', () => { + /* The "promoted but nobody hears them" regression — a stale 'failed' + * mesh entry must not block SFU. */ + const b = makeBrowser('speaker'); + addMember(b, UUID_A, PUB_A); + b.api.peers.set(UUID_A, { connectionState: 'failed' }); + const stream = makeStream(); + b.api.handleRemoteSfuTrack(fakeOntrack(PUB_A, 'mic', stream)); + truthy(b.api.listenerAudioNodes.has(UUID_A), 'SFU took over from failed mesh'); +}); + +test('speaker: SFU first then mesh — setWorkletStream swaps in place, single chain', () => { + /* This is the common mesh path: SFU subscribe brings audio first + * (because subscribe runs immediately), then mesh PC connects a + * second or two later and ontrack fires, swapping the source. */ + const b = makeBrowser('speaker'); + addMember(b, UUID_A, PUB_A); + const sfuStream = makeStream(); + b.api.attachSfuTrack(UUID_A, sfuStream); + const node = b.api.listenerAudioNodes.get(UUID_A); + const dest = b.ctx.destination; + truthy(node, 'chain built via SFU'); + + /* simulate mesh ontrack: swap source in place — exact code path the + * connectToPeer ontrack handler uses (web/zebra-spaces.html:6596) */ + const meshStream = makeStream(); + const swapOk = b.api.setWorkletStream(UUID_A, meshStream); + eq(swapOk, true, 'mesh swap succeeded'); + const after = b.api.listenerAudioNodes.get(UUID_A); + eq(after, node, 'same node — no rebuild'); + eq(after.gain, node.gain, 'same gain — no rewire to destination'); + eq(after.stream, meshStream, 'tracks mesh stream now'); + truthy(reachable(after.src, dest), 'mesh src reaches destination'); +}); + +test('speaker: mesh ontrack BEFORE SFU ontrack → mesh attaches via fallback, SFU then skips because mesh.connectionState=connected', () => { + /* Less common but possible: mesh connects faster than SFU subscribe + * negotiation (e.g. STUN binding cached). Mesh's ontrack runs + * setWorkletStream which fails (no existing chain), falls through + * to attachAudioStreamViaWorklet to build one. Then SFU ontrack + * fires; canSpeak+mesh.connected → skip. */ + const b = makeBrowser('speaker'); + addMember(b, UUID_A, PUB_A); + + /* fake mesh ontrack effect: chain doesn't exist yet, setWorkletStream + * returns false, fallback builds chain at SPEAKER target */ + const meshStream = makeStream(); + const swapOk = b.api.setWorkletStream(UUID_A, meshStream); + eq(swapOk, false, 'setWorkletStream false — no chain yet'); + b.api.attachAudioStreamViaWorklet(UUID_A, meshStream, SPEAKER); + truthy(b.api.listenerAudioNodes.has(UUID_A), 'chain built from mesh fallback'); + + /* now SFU ontrack arrives with mesh marked connected */ + b.api.peers.set(UUID_A, { connectionState: 'connected' }); + const sfuStream = makeStream(); + b.api.handleRemoteSfuTrack(fakeOntrack(PUB_A, 'mic', sfuStream)); + + /* still only one chain — the one mesh built. SFU should have skipped. */ + eq(b.api.listenerAudioNodes.size, 1, 'one chain total'); + eq(b.api.listenerAudioNodes.get(UUID_A).stream, meshStream, 'still pointing at mesh stream'); +}); + +test('speaker: rapid mesh re-ontracks for same publisher → single chain, no duplicate gains or sources stacked on destination', () => { + /* Mesh renegotiation or transceiver replay can fire multiple ontracks + * within a tick. Each must be a clean in-place swap; the final state + * is exactly one src and one gain feeding destination. */ + const b = makeBrowser('speaker'); + addMember(b, UUID_A, PUB_A); + b.api.attachAudioStreamViaWorklet(UUID_A, makeStream(), SPEAKER); + const dest = b.ctx.destination; + for (let i = 0; i < 5; i++){ + b.api.setWorkletStream(UUID_A, makeStream()); + } + /* count edges from any source-kind node into destination — exactly one + * gain should feed destination, fed by exactly one source. */ + const gainsFeedingDest = b.ctx._nodes.filter(n => + n.kind === 'gain' && n._outgoing.has(dest) + ); + eq(gainsFeedingDest.length, 1, 'exactly one gain feeds destination'); + const node = b.api.listenerAudioNodes.get(UUID_A); + /* the gain in listenerAudioNodes is the one feeding dest */ + eq(gainsFeedingDest[0], node.gain, 'and it is the chain gain'); +}); + +test('speaker: SFU cached stream remains in sfuStreamsByPubHex even when mesh wins the race — needed for mesh-failed fallback', () => { + /* setWorkletStream(uuid, sfuCachedStream) on mesh-failed (line + * ~6645) reads sfuStreamsByPubHex.get(pubHex). If the cache is + * empty (because we never went through handleRemoteSfuTrack's + * caching branch), mesh-fail leaves the user silent. */ + const b = makeBrowser('speaker'); + addMember(b, UUID_A, PUB_A); + b.api.peers.set(UUID_A, { connectionState: 'connected' }); + const stream = makeStream(); + b.api.handleRemoteSfuTrack(fakeOntrack(PUB_A, 'mic', stream)); + /* the SFU attach was skipped, but the CACHE must still be primed */ + truthy(b.api.sfuStreamsByPubHex.size > 0, 'cached for later mesh-fail'); +}); + /* -------- summary -------- */ console.log(''); console.log((fail === 0 ? 'PASS' : 'FAIL') + ' — ' + pass + ' passed, ' + fail + ' failed'); diff --git a/web/zebra-spaces.html b/web/zebra-spaces.html index bf86062..f5b802f 100644 --- a/web/zebra-spaces.html +++ b/web/zebra-spaces.html @@ -1878,14 +1878,37 @@ class JitterBufferProcessor extends AudioWorkletProcessor { if (e.data.cmd === 'retarget'){ const t = +e.data.targetSeconds; if (!isFinite(t) || t <= 0) return; + const prevTargetSamples = this.targetSamples; this.targetSeconds = t; this.maxSeconds = t * 1.5; this.targetSamples = Math.round(this.targetSeconds * sampleRate); this.maxSamples = Math.round(this.maxSeconds * sampleRate); - while (this.buffered > this.maxSamples && this.queue.length > 0){ + /* On a SHRINKING retarget (listener-to-speaker promotion: 4s + * → 0.5s), drop down to targetSamples directly instead of + * maxSamples. The maxSamples threshold leaves a 1.5× target + * overhang that triggers an audible 6%-cap speed-up phase + * lasting ~4 seconds — what fox 2026-06-06 called "janky" + * during promotion. One hard skip-forward, then normal + * playback — no residual adjustment. + * + * Report the skip back to JS so paired video receivers can + * advance the same amount: an audio jump without a matching + * video jump = broken lip-sync for the duration of the + * native video jbuf's gradual drain. fox 2026-06-06: "if we + * skip ahead from whatever listener is at to speaker speed, + * we need to make sure the video skips ahead the same amount + * or rate to keep the lips synced". */ + const shrinking = this.targetSamples < prevTargetSamples; + const dropTo = shrinking ? this.targetSamples : this.maxSamples; + let skipped = 0; + while (this.buffered > dropTo && this.queue.length > 0){ const drop = this.queue.shift(); this.buffered -= drop[0].length; this.dropped += drop[0].length; + skipped += drop[0].length; + } + if (skipped > 0){ + try { this.port.postMessage({ cmd: 'dropped', samples: skipped }); } catch(_){} } } else if (e.data.cmd === 'lock_rate'){ /* listeners get this — explicitly forbid time-stretching so @@ -2542,6 +2565,43 @@ function installJitterBuffer(uuid, node){ * enough to catch a wiggle within ~2s of it starting, * vs the 5s tick path which can be 5-10s late. */ ddNoteWorkletBuffered(uuid, e.data.seconds); + } else if (e.data.cmd === 'dropped'){ + /* The worklet just skipped audio forward (shrinking retarget + * — listener→speaker promotion drops 4s→0.5s buffer). Drag + * the paired video receivers forward by the same amount so + * lip-sync survives the jump. fox 2026-06-06: "if we skip + * ahead from whatever listener is at to speaker speed, we + * need to make sure the video skips ahead the same amount + * or rate to keep the lips synced". + * + * No native "skip ahead" API exists for RTCRtpReceiver, but + * jitterBufferTarget is a hard target the browser converges + * to. Setting it to 0 forces aggressive frame-drop until the + * native video jbuf drains; the next refreshLipSyncForUuid + * 'buffered' tick (~2s) restores the proper role-appropriate + * target. Net: audio jumps instantly + video jumps almost + * instantly (browser frame-drop is fast) = lip-sync stays + * within ~tens of ms. */ + try { + const mm = members.get(uuid); + const pubHex = mm && mm.pubkey ? hex(unb64(mm.pubkey)) : null; + if (pubHex){ + const ls = lipSync.get(pubHex); + if (ls && ls.videoReceivers){ + for (const [, rx] of ls.videoReceivers){ + try { rx.jitterBufferTarget = 0; } catch(_){} + try { rx.playoutDelayHint = 0; } catch(_){} + } + /* clear lastApplied so the next refreshLipSyncForUuid + * actually re-applies the role's target — without this + * the threshold check could see "no significant change" + * and leave video stuck at 0. */ + ls.lastApplied = 0; + } + } + const skippedSec = e.data.samples / (audioCtx ? audioCtx.sampleRate : 48000); + logLine('', 'jitter-buffer skipped '+skippedSec.toFixed(2)+'s uuid='+uuid.slice(0,4)+' — video re-targeting'); + } catch(_){} } }; /* All roles now allow time-stretching — listeners benefit too @@ -7904,8 +7964,8 @@ logLine('', 'ready — pick a handle, type a rendezvous code, enter the space');