Fox 2026-06-06 on listener-to-speaker promotion: "the music slows
down I think that is the 6% algo which is janky. maybe it's better
to speed up in that case? if we skip ahead from whatever listener
is at to speaker speed, we need to make sure the video skips ahead
the same amount or rate to keep the lips synced".
Two coupled changes in the audio path:
1. JitterBufferProcessor retarget handler — on a SHRINKING retarget
(e.g. 4s → 0.5s) drop the queue down to targetSamples directly
instead of maxSamples (1.5×target). The 0.25s overhang the old
code left behind triggered a 6%-cap stretchFactor adjustment that
played at 1.064× for ~4 seconds — the residual phase fox heard
as "janky". Now: instant skip to the new target, then normal
playback. The grow-direction path is unchanged.
Worklet also reports the dropped sample count back to JS via
{cmd:'dropped', samples}.
2. installJitterBuffer.jbuf.port.onmessage — on 'dropped' from the
worklet, walk the publisher's lipSync.videoReceivers and set
jitterBufferTarget = 0 + playoutDelayHint = 0 on every one. The
browser drops video frames aggressively to converge on the new
target. Without this, audio jumps 3.25s forward instantly and
video drains gradually = broken lip-sync for the duration of the
native video jbuf's drain. Clearing ls.lastApplied lets the next
refreshLipSyncForUuid actually re-apply the role-appropriate
target instead of stopping at the threshold check.
Tests:
test/jitter-buffer-worklet.test.js (NEW, 8 assertions) — extracts
the inline JITTER_BUFFER_WORKLET_CODE template literal and runs
the processor in a Node sandbox with stubbed sampleRate +
registerProcessor + AudioWorkletProcessor. Pins:
- 'started' fires once on first fill
- 'buffered' reports depth
- retarget SHRINK drops to targetSamples (not maxSamples) and
posts 'dropped' with sample count
- retarget SHRINK does NOT leave 1.5×target overhang (the
residual that caused the janky 6% phase)
- retarget GROW does NOT drop
- retarget to same target is no-op
- bogus targetSeconds (NaN/0/negative) ignored
- lock_rate pins stretchFactor
test/listener-audio-attach.test.js (extended, +8 assertions):
- worklet 'dropped' handler zeros every paired video receiver
target (defensive against missing lipSync entry; correct when
seeded)
- speaker mesh+SFU collision tests (single-chain invariant):
mesh state=connected → SFU skip
mesh state=failed → SFU takes over
SFU first then mesh → in-place swap, no rebuild
mesh first then SFU(connected) → SFU skip, one chain
rapid mesh re-ontracks → still one gain feeds destination
SFU cache stays primed even when mesh wins the race
Makefile: new test-jitter-worklet target, added to test-all chain.
219 lines
9.2 KiB
JavaScript
219 lines
9.2 KiB
JavaScript
#!/usr/bin/env node
|
||
/* jitter-buffer worklet processor tests.
|
||
*
|
||
* node test/jitter-buffer-worklet.test.js
|
||
*
|
||
* Pins the JITTER_BUFFER_WORKLET_CODE processor contract:
|
||
*
|
||
* - retarget shrinking (e.g. listener-to-speaker promotion, 4s→0.5s):
|
||
* drop the queue down to targetSamples (NOT 1.5×target) and post
|
||
* a 'dropped' message back to JS so paired video receivers can
|
||
* advance the matching amount. Fox 2026-06-06: "if we skip ahead
|
||
* from whatever listener is at to speaker speed, we need to make
|
||
* sure the video skips ahead the same amount or rate to keep the
|
||
* lips synced". The shrink contract eliminates the residual 6%
|
||
* speed-up phase fox heard as "janky" during promotion.
|
||
*
|
||
* - retarget growing (speaker-to-listener demotion, 0.5s→4s): no
|
||
* drop (we don't have extra audio to throw away; the algo grows
|
||
* into the new target naturally).
|
||
*
|
||
* - retarget to same target: no-op drop.
|
||
*
|
||
* - 'started' posted on first fill to target.
|
||
*
|
||
* - 'buffered' periodic reports.
|
||
*
|
||
* - 'lock_rate' freezes stretchFactor (listener music mode).
|
||
*
|
||
* The worklet code is a template literal in the page. We extract it
|
||
* verbatim, inject sampleRate + registerProcessor + a port spy, and
|
||
* drive process() / message events directly. */
|
||
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
const src = fs.readFileSync(path.join(__dirname, '..', 'web', 'zebra-spaces.html'), 'utf8');
|
||
|
||
/* extract the `const JITTER_BUFFER_WORKLET_CODE = \`...\`;` literal */
|
||
function extractTemplate(name){
|
||
const head = new RegExp('const\\s+' + name + '\\s*=\\s*`');
|
||
const m = src.match(head);
|
||
if (!m) throw new Error('could not find template ' + name);
|
||
const startBacktick = src.indexOf('`', m.index);
|
||
const endBacktick = src.indexOf('`;', startBacktick + 1);
|
||
if (startBacktick < 0 || endBacktick < 0) throw new Error('malformed template literal ' + name);
|
||
return src.slice(startBacktick + 1, endBacktick);
|
||
}
|
||
|
||
const workletCode = extractTemplate('JITTER_BUFFER_WORKLET_CODE');
|
||
|
||
/* The worklet expects:
|
||
* - global `sampleRate` (provided by AudioWorkletGlobalScope)
|
||
* - global `registerProcessor(name, classRef)`
|
||
* - `AudioWorkletProcessor` base class (just needs `this.port`)
|
||
*
|
||
* In Node we build a stub world and `Function`-eval the code in it. */
|
||
function buildWorld(sampleRate){
|
||
const registered = {};
|
||
function registerProcessor(name, cls){ registered[name] = cls; }
|
||
|
||
/* AudioWorkletProcessor: constructor sets `this.port = portStub`.
|
||
* The processor's constructor calls super(), so we initialize the
|
||
* port here. */
|
||
class AudioWorkletProcessor {
|
||
constructor(){ this.port = { onmessage: null, posted: [], postMessage(m){ this.posted.push(m); } }; }
|
||
}
|
||
|
||
const factory = new Function(
|
||
'sampleRate', 'registerProcessor', 'AudioWorkletProcessor',
|
||
workletCode + '\nreturn null;'
|
||
);
|
||
factory(sampleRate, registerProcessor, AudioWorkletProcessor);
|
||
return registered;
|
||
}
|
||
|
||
const SR = 48000;
|
||
const world = buildWorld(SR);
|
||
const JitterBufferProcessor = world['jitter-buffer'];
|
||
if (!JitterBufferProcessor) throw new Error('jitter-buffer processor did not register');
|
||
|
||
/* Helper: create a processor instance with options, build a fake input
|
||
* block, and run process() N times feeding it. Each input block has
|
||
* blkLen samples per channel (Web Audio standard 128). */
|
||
function makeProcessor(targetSeconds){
|
||
const p = new JitterBufferProcessor({
|
||
processorOptions: { targetSeconds, maxSeconds: targetSeconds * 1.5 },
|
||
});
|
||
return p;
|
||
}
|
||
|
||
const BLK = 128;
|
||
function inputBlock(channels){
|
||
const ch = [];
|
||
for (let c = 0; c < channels; c++) ch.push(new Float32Array(BLK));
|
||
return [ch]; /* inputs[0] = first input, channels */
|
||
}
|
||
function outputBlock(channels){
|
||
const ch = [];
|
||
for (let c = 0; c < channels; c++) ch.push(new Float32Array(BLK));
|
||
return [ch];
|
||
}
|
||
/* Feed N input blocks through process so the queue grows by ~N*BLK samples */
|
||
function feed(p, n, channels){
|
||
channels = channels || 2;
|
||
for (let i = 0; i < n; i++){
|
||
p.process(inputBlock(channels), outputBlock(channels));
|
||
}
|
||
}
|
||
|
||
let pass = 0, fail = 0;
|
||
function test(name, fn){
|
||
try { fn(); console.log(' ✓ ' + name); pass++; }
|
||
catch (e){ console.log(' ✗ ' + name + ' — ' + (e.message || e)); fail++; }
|
||
}
|
||
function eq(a, b, msg){
|
||
if (a !== b) throw new Error((msg || 'expected') + ' — got ' + JSON.stringify(a) + ' want ' + JSON.stringify(b));
|
||
}
|
||
function approxEq(a, b, tol, msg){
|
||
if (Math.abs(a - b) > tol) throw new Error((msg || 'approxEq') + ' — got ' + a + ' want ~' + b + ' (tol=' + tol + ')');
|
||
}
|
||
function truthy(v, msg){ if (!v) throw new Error(msg || 'expected truthy'); }
|
||
function ge(a, b, msg){ if (!(a >= b)) throw new Error((msg || 'expected >=') + ' — got ' + a + ' >= ' + b); }
|
||
function le(a, b, msg){ if (!(a <= b)) throw new Error((msg || 'expected <=') + ' — got ' + a + ' <= ' + b); }
|
||
|
||
console.log('jitter-buffer worklet processor:');
|
||
|
||
/* -------- 'started' message on first fill -------- */
|
||
test('"started" message fires once when the buffer reaches targetSamples', () => {
|
||
const p = makeProcessor(0.5); /* 0.5s × 48k = 24000 samples target */
|
||
const blocks = Math.ceil(p.targetSamples / BLK) + 1;
|
||
feed(p, blocks);
|
||
const startedMsgs = p.port.posted.filter(m => m.cmd === 'started');
|
||
eq(startedMsgs.length, 1, 'one started');
|
||
eq(startedMsgs[0].targetSeconds, 0.5, 'reports target');
|
||
});
|
||
|
||
/* -------- 'buffered' messages -------- */
|
||
test('"buffered" messages report current depth periodically', () => {
|
||
const p = makeProcessor(0.5);
|
||
/* push enough blocks to cross the report-every threshold */
|
||
feed(p, 400);
|
||
const buf = p.port.posted.filter(m => m.cmd === 'buffered');
|
||
ge(buf.length, 1, 'at least one buffered');
|
||
ge(buf[0].seconds, 0, 'seconds reported');
|
||
});
|
||
|
||
/* -------- retarget SHRINK drops to targetSamples + reports 'dropped' -------- */
|
||
test('retarget SHRINK (4s → 0.5s) drops queue to targetSamples and posts "dropped" with sample count', () => {
|
||
/* fill to ~4s, then retarget to 0.5s, assert the drop went all the
|
||
* way down to targetSamples (NOT 1.5×target=0.75s) and the dropped
|
||
* message carries the skip amount. */
|
||
const p = makeProcessor(4.0);
|
||
const blocks = Math.ceil(4.0 * SR / BLK); /* ~1500 blocks */
|
||
feed(p, blocks + 5); /* a little over target */
|
||
/* now retarget */
|
||
p.port.onmessage({ data: { cmd: 'retarget', targetSeconds: 0.5 } });
|
||
/* targetSamples = 0.5 * 48000 = 24000 */
|
||
le(p.buffered, p.targetSamples + BLK, 'buffered now within one block of targetSamples');
|
||
ge(p.buffered, 0, 'buffered non-negative');
|
||
/* a 'dropped' message must have fired */
|
||
const dropMsgs = p.port.posted.filter(m => m.cmd === 'dropped');
|
||
eq(dropMsgs.length, 1, 'one dropped message');
|
||
ge(dropMsgs[0].samples, 1, 'samples > 0');
|
||
});
|
||
|
||
test('retarget SHRINK does NOT leave a 1.5×target overhang (the residual that triggered the "janky 6%" phase)', () => {
|
||
const p = makeProcessor(4.0);
|
||
feed(p, Math.ceil(4.0 * SR / BLK) + 10);
|
||
p.port.onmessage({ data: { cmd: 'retarget', targetSeconds: 0.5 } });
|
||
/* after a shrink, buffered should be at most targetSamples + one
|
||
* block — explicitly NOT 1.5×targetSamples. Pre-fix the drop was
|
||
* `while (buffered > maxSamples)` which left up to 0.75s in the
|
||
* queue → ~4s of 6% catch-up playback. */
|
||
le(p.buffered, p.targetSamples + BLK, 'no 1.5× overhang');
|
||
});
|
||
|
||
/* -------- retarget GROW does NOT drop -------- */
|
||
test('retarget GROW (0.5s → 4s) does NOT drop anything', () => {
|
||
const p = makeProcessor(0.5);
|
||
feed(p, Math.ceil(0.5 * SR / BLK) + 5);
|
||
const buffBefore = p.buffered;
|
||
p.port.onmessage({ data: { cmd: 'retarget', targetSeconds: 4.0 } });
|
||
eq(p.buffered, buffBefore, 'buffered unchanged on grow');
|
||
const dropMsgs = p.port.posted.filter(m => m.cmd === 'dropped');
|
||
eq(dropMsgs.length, 0, 'no dropped message');
|
||
});
|
||
|
||
test('retarget to SAME target is a no-op (no drop, no message)', () => {
|
||
const p = makeProcessor(0.5);
|
||
feed(p, Math.ceil(0.5 * SR / BLK) + 5);
|
||
const buffBefore = p.buffered;
|
||
p.port.onmessage({ data: { cmd: 'retarget', targetSeconds: 0.5 } });
|
||
eq(p.buffered, buffBefore, 'buffered unchanged on same-target retarget');
|
||
const dropMsgs = p.port.posted.filter(m => m.cmd === 'dropped');
|
||
eq(dropMsgs.length, 0, 'no dropped message');
|
||
});
|
||
|
||
/* -------- retarget validation -------- */
|
||
test('retarget with bogus targetSeconds is ignored (no NaN, no negative)', () => {
|
||
const p = makeProcessor(4.0);
|
||
feed(p, 50);
|
||
const targetBefore = p.targetSeconds;
|
||
p.port.onmessage({ data: { cmd: 'retarget', targetSeconds: NaN } });
|
||
p.port.onmessage({ data: { cmd: 'retarget', targetSeconds: -1 } });
|
||
p.port.onmessage({ data: { cmd: 'retarget', targetSeconds: 0 } });
|
||
eq(p.targetSeconds, targetBefore, 'target unchanged');
|
||
});
|
||
|
||
/* -------- lock_rate (listener music mode) -------- */
|
||
test('lock_rate=1.0 pins stretchFactor at 1.0 and disables ±6% adaptation', () => {
|
||
const p = makeProcessor(4.0);
|
||
p.port.onmessage({ data: { cmd: 'lock_rate', rate: 1.0 } });
|
||
eq(p.stretchFactor, 1.0);
|
||
eq(p.minStretch, 1.0, 'min === max → no adaptation');
|
||
eq(p.maxStretch, 1.0);
|
||
});
|
||
|
||
console.log('');
|
||
console.log((fail === 0 ? 'PASS' : 'FAIL') + ' — ' + pass + ' passed, ' + fail + ' failed');
|
||
process.exit(fail === 0 ? 0 : 1);
|