zebra-spaces: stop re-attaching every speaker on peer-joined (kill fxhp double-audio)
This commit is contained in:
parent
c2d6016c8b
commit
269ba777ca
1 changed files with 262 additions and 8 deletions
|
|
@ -2616,6 +2616,226 @@ function registerLipSyncVideo(pubHex, kind, receiver){
|
|||
const e = lipSyncEntry(pubHex);
|
||||
e.videoReceivers.set(kind, receiver);
|
||||
}
|
||||
|
||||
/* ====== perceptual lip-sync ===============================================
|
||||
* The buffer-derived target above knows what was BUFFERED, not what reached
|
||||
* the user. PipeWire / kernel resamplers / the listener's mobile audio stack
|
||||
* add hidden delay the worklet can't see. Result: visible drift in both
|
||||
* directions depending on which side has more local indirection that hour.
|
||||
*
|
||||
* This module measures the actual visual-vs-audio offset by cross-correlating
|
||||
* video motion energy against audio RMS over a rolling 5 s window. Output:
|
||||
* a per-pub correction (ms) added to refreshLipSyncForUuid's playoutDelay
|
||||
* target.
|
||||
*
|
||||
* Three modes, auto-detected per pub:
|
||||
* TALKING_HEAD — high motion↔audio correlation → apply correction
|
||||
* DETACHED_MEDIA — motion exists but uncorrelated (movie / slides) → idle
|
||||
* AUDIO_ONLY — no useful motion → idle, save battery
|
||||
*/
|
||||
const perceptualSync = new Map(); /* pubHex → state */
|
||||
const PERCEPTUAL_SAMPLE_MS = 100;
|
||||
const PERCEPTUAL_BUF_LEN = 50; /* 5 s window */
|
||||
const PERCEPTUAL_MAX_LAG = 5; /* ±500 ms search range */
|
||||
const PERCEPTUAL_CORR_GATE = 0.35; /* TALKING_HEAD entry threshold */
|
||||
const PERCEPTUAL_MOTION_GATE = 0.004; /* DETACHED vs AUDIO_ONLY */
|
||||
const PERCEPTUAL_ENTER_DEBOUNCE_MS = 500;
|
||||
const PERCEPTUAL_LEAVE_DEBOUNCE_MS = 3000;
|
||||
const PERCEPTUAL_OFFSET_CLAMP_MS = 300;
|
||||
const PERCEPTUAL_EMA_ALPHA = 0.3;
|
||||
const PERCEPTUAL_LOG_THROTTLE_MS = 5000;
|
||||
|
||||
function attachPerceptualSync(pubHex, videoEl, retries){
|
||||
retries = retries || 0;
|
||||
if (!videoEl || !audioCtx) return;
|
||||
const existing = perceptualSync.get(pubHex);
|
||||
if (existing){
|
||||
if (existing.videoEl === videoEl) return;
|
||||
detachPerceptualSync(pubHex);
|
||||
}
|
||||
const e = lipSync.get(pubHex);
|
||||
if (!e || !e.audioUuid){
|
||||
/* audio side not registered yet — retry a few times */
|
||||
if (retries < 10){
|
||||
setTimeout(() => attachPerceptualSync(pubHex, videoEl, retries + 1), 500);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const node = listenerAudioNodes.get(e.audioUuid);
|
||||
if (!node || !node.gain){
|
||||
if (retries < 10){
|
||||
setTimeout(() => attachPerceptualSync(pubHex, videoEl, retries + 1), 500);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let analyser;
|
||||
try {
|
||||
analyser = audioCtx.createAnalyser();
|
||||
analyser.fftSize = 1024;
|
||||
analyser.smoothingTimeConstant = 0;
|
||||
node.gain.connect(analyser);
|
||||
} catch(err){
|
||||
logLine('err','perceptual analyser failed: '+err.message);
|
||||
return;
|
||||
}
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = 32; canvas.height = 32;
|
||||
const ctx2d = canvas.getContext('2d', { willReadFrequently: true });
|
||||
const st = {
|
||||
pubHex,
|
||||
videoEl,
|
||||
analyser,
|
||||
audioSource: node.gain,
|
||||
canvas, ctx2d,
|
||||
audioBuf: new Float32Array(PERCEPTUAL_BUF_LEN),
|
||||
motionBuf: new Float32Array(PERCEPTUAL_BUF_LEN),
|
||||
prevPixels: null,
|
||||
idx: 0, filled: 0,
|
||||
mode: 'AUDIO_ONLY',
|
||||
pendingMode: null, pendingSince: 0,
|
||||
offsetMs: 0,
|
||||
lastCorrAt: 0,
|
||||
lastLogAt: 0,
|
||||
sampleScratch: new Float32Array(analyser.fftSize),
|
||||
};
|
||||
st.interval = setInterval(() => perceptualTick(pubHex), PERCEPTUAL_SAMPLE_MS);
|
||||
perceptualSync.set(pubHex, st);
|
||||
}
|
||||
function detachPerceptualSync(pubHex){
|
||||
const st = perceptualSync.get(pubHex);
|
||||
if (!st) return;
|
||||
try { clearInterval(st.interval); } catch(_){}
|
||||
try { st.audioSource.disconnect(st.analyser); } catch(_){}
|
||||
try { st.analyser.disconnect(); } catch(_){}
|
||||
perceptualSync.delete(pubHex);
|
||||
}
|
||||
function perceptualOffsetMs(pubHex){
|
||||
const st = perceptualSync.get(pubHex);
|
||||
if (!st || st.mode !== 'TALKING_HEAD') return null;
|
||||
return st.offsetMs;
|
||||
}
|
||||
function perceptualTick(pubHex){
|
||||
const st = perceptualSync.get(pubHex);
|
||||
if (!st) return;
|
||||
const v = st.videoEl;
|
||||
if (!v || v.readyState < 2 || v.videoWidth === 0){
|
||||
perceptualTransition(st, 'AUDIO_ONLY');
|
||||
return;
|
||||
}
|
||||
try { st.analyser.getFloatTimeDomainData(st.sampleScratch); }
|
||||
catch(_){ return; }
|
||||
let sum = 0;
|
||||
for (let i = 0; i < st.sampleScratch.length; i++){
|
||||
sum += st.sampleScratch[i] * st.sampleScratch[i];
|
||||
}
|
||||
const audioE = Math.sqrt(sum / st.sampleScratch.length);
|
||||
let motionE = 0;
|
||||
try {
|
||||
st.ctx2d.drawImage(v, 0, 0, 32, 32);
|
||||
const img = st.ctx2d.getImageData(0, 0, 32, 32).data;
|
||||
if (st.prevPixels && st.prevPixels.length === img.length){
|
||||
let d = 0;
|
||||
for (let i = 0; i < img.length; i += 4){
|
||||
const lumNow = 0.299*img[i] + 0.587*img[i+1] + 0.114*img[i+2];
|
||||
const lumPrev = 0.299*st.prevPixels[i] + 0.587*st.prevPixels[i+1] + 0.114*st.prevPixels[i+2];
|
||||
d += Math.abs(lumNow - lumPrev);
|
||||
}
|
||||
motionE = d / (img.length / 4) / 255;
|
||||
}
|
||||
st.prevPixels = img;
|
||||
} catch(_){
|
||||
/* cross-origin tainted MediaStream or detached video — stop trying */
|
||||
perceptualTransition(st, 'AUDIO_ONLY');
|
||||
return;
|
||||
}
|
||||
st.audioBuf[st.idx] = audioE;
|
||||
st.motionBuf[st.idx] = motionE;
|
||||
st.idx = (st.idx + 1) % PERCEPTUAL_BUF_LEN;
|
||||
if (st.filled < PERCEPTUAL_BUF_LEN) st.filled++;
|
||||
if (st.filled < PERCEPTUAL_BUF_LEN) return;
|
||||
const now = performance.now();
|
||||
if (now - st.lastCorrAt < 1000) return;
|
||||
st.lastCorrAt = now;
|
||||
const corr = perceptualCorrelate(st.audioBuf, st.motionBuf, st.idx, PERCEPTUAL_MAX_LAG);
|
||||
let motionRms = 0;
|
||||
for (let i = 0; i < PERCEPTUAL_BUF_LEN; i++){
|
||||
motionRms += st.motionBuf[i] * st.motionBuf[i];
|
||||
}
|
||||
motionRms = Math.sqrt(motionRms / PERCEPTUAL_BUF_LEN);
|
||||
let desired = 'AUDIO_ONLY';
|
||||
if (corr.peak > PERCEPTUAL_CORR_GATE) desired = 'TALKING_HEAD';
|
||||
else if (motionRms > PERCEPTUAL_MOTION_GATE) desired = 'DETACHED_MEDIA';
|
||||
perceptualTransition(st, desired);
|
||||
if (st.mode === 'TALKING_HEAD'){
|
||||
/* +lag = audio leads motion samples → video late → reduce delay */
|
||||
const rawMs = -corr.bestLag * PERCEPTUAL_SAMPLE_MS;
|
||||
const clamped = Math.max(-PERCEPTUAL_OFFSET_CLAMP_MS,
|
||||
Math.min(PERCEPTUAL_OFFSET_CLAMP_MS, rawMs));
|
||||
st.offsetMs = PERCEPTUAL_EMA_ALPHA * clamped + (1 - PERCEPTUAL_EMA_ALPHA) * st.offsetMs;
|
||||
if (now - st.lastLogAt > PERCEPTUAL_LOG_THROTTLE_MS){
|
||||
st.lastLogAt = now;
|
||||
logLine('', 'perceptual pub='+pubHex.slice(0,4)+
|
||||
' mode=TALKING corr='+corr.peak.toFixed(2)+
|
||||
' lag='+(corr.bestLag * PERCEPTUAL_SAMPLE_MS)+'ms'+
|
||||
' offset='+st.offsetMs.toFixed(0)+'ms');
|
||||
}
|
||||
} else if (st.offsetMs !== 0){
|
||||
st.offsetMs = 0;
|
||||
}
|
||||
}
|
||||
function perceptualTransition(st, desired){
|
||||
if (st.mode === desired){
|
||||
st.pendingMode = null;
|
||||
return;
|
||||
}
|
||||
const now = performance.now();
|
||||
const debounce = (desired === 'TALKING_HEAD')
|
||||
? PERCEPTUAL_ENTER_DEBOUNCE_MS
|
||||
: PERCEPTUAL_LEAVE_DEBOUNCE_MS;
|
||||
if (st.pendingMode !== desired){
|
||||
st.pendingMode = desired;
|
||||
st.pendingSince = now;
|
||||
return;
|
||||
}
|
||||
if (now - st.pendingSince < debounce) return;
|
||||
const prev = st.mode;
|
||||
st.mode = desired;
|
||||
st.pendingMode = null;
|
||||
if (desired !== 'TALKING_HEAD') st.offsetMs = 0;
|
||||
logLine('', 'perceptual pub='+st.pubHex.slice(0,4)+' mode '+prev+' → '+desired);
|
||||
}
|
||||
function perceptualCorrelate(audioCirc, motionCirc, idx, maxLag){
|
||||
const N = audioCirc.length;
|
||||
const a = new Float32Array(N), m = new Float32Array(N);
|
||||
for (let i = 0; i < N; i++){
|
||||
a[i] = audioCirc[(idx + i) % N];
|
||||
m[i] = motionCirc[(idx + i) % N];
|
||||
}
|
||||
let am = 0, mm = 0;
|
||||
for (let i = 0; i < N; i++){ am += a[i]; mm += m[i]; }
|
||||
am /= N; mm /= N;
|
||||
let va = 0, vm = 0;
|
||||
for (let i = 0; i < N; i++){
|
||||
a[i] -= am; m[i] -= mm;
|
||||
va += a[i]*a[i]; vm += m[i]*m[i];
|
||||
}
|
||||
const denom = Math.sqrt(va * vm);
|
||||
if (denom < 1e-9) return { peak: 0, bestLag: 0 };
|
||||
let peak = 0, bestLag = 0;
|
||||
for (let L = -maxLag; L <= maxLag; L++){
|
||||
let s = 0, n = 0;
|
||||
for (let i = 0; i < N; i++){
|
||||
const j = i + L;
|
||||
if (j < 0 || j >= N) continue;
|
||||
s += a[i] * m[j];
|
||||
n++;
|
||||
}
|
||||
if (n < N / 2) continue;
|
||||
const c = s / denom;
|
||||
if (c > peak){ peak = c; bestLag = L; }
|
||||
}
|
||||
return { peak, bestLag };
|
||||
}
|
||||
function medianOf(arr){
|
||||
if (arr.length === 0) return 0;
|
||||
const s = arr.slice().sort((a,b) => a-b);
|
||||
|
|
@ -2685,13 +2905,19 @@ async function refreshLipSyncForUuid(uuid){
|
|||
while (e.history.length > LIP_SYNC_HISTORY) e.history.shift();
|
||||
if (e.history.length < 3) return; /* wait for ≥3 samples */
|
||||
const med = medianOf(e.history);
|
||||
if (Math.abs(med - e.lastApplied) < LIP_SYNC_THRESHOLD) return;
|
||||
/* perceptual correction: ground-truth offset from motion↔audio correlation.
|
||||
* Null when no TALKING_HEAD signal — we then trust the buffer median alone. */
|
||||
const percOff = perceptualOffsetMs(pubHex);
|
||||
const adj = (percOff !== null) ? med + percOff / 1000 : med;
|
||||
const target = Math.max(0.05, Math.min(5.0, adj));
|
||||
if (Math.abs(target - e.lastApplied) < LIP_SYNC_THRESHOLD) return;
|
||||
for (const [kind, rx] of e.videoReceivers){
|
||||
try { rx.playoutDelayHint = med; } catch(_){}
|
||||
try { rx.jitterBufferTarget = med * 1000; } catch(_){}
|
||||
try { rx.playoutDelayHint = target; } catch(_){}
|
||||
try { rx.jitterBufferTarget = target * 1000; } catch(_){}
|
||||
}
|
||||
e.lastApplied = med;
|
||||
logLine('', 'lip-sync pub='+pubHex.slice(0,4)+' delay='+med.toFixed(2)+'s ('+e.videoReceivers.size+' video rx, history median of '+e.history.length+')');
|
||||
e.lastApplied = target;
|
||||
const percTag = (percOff !== null) ? ' perc='+percOff.toFixed(0)+'ms' : '';
|
||||
logLine('', 'lip-sync pub='+pubHex.slice(0,4)+' delay='+target.toFixed(2)+'s ('+e.videoReceivers.size+' video rx, median='+med.toFixed(2)+'s'+percTag+')');
|
||||
}
|
||||
|
||||
/* ==================================================================
|
||||
|
|
@ -3195,11 +3421,23 @@ function attachCachedSfuStreamFor(uuid){
|
|||
if (stream) attachSfuTrack(uuid, stream);
|
||||
}
|
||||
function flushSfuStreams(){
|
||||
/* Skip if EITHER attach map already has the uuid: remoteAudio is the
|
||||
* <audio>-element fallback path (only populated when AudioContext
|
||||
* fails); listenerAudioNodes is the modern worklet path used by
|
||||
* every role today. Pre-fix this guard only checked remoteAudio,
|
||||
* which is empty on listeners using the worklet path, so every
|
||||
* peer-joined event re-attached EVERY speaker — racing teardown vs
|
||||
* fresh-attach on the same uuid and producing audible overlap.
|
||||
* Fox 2026-06-05 telemetry: "fxhp-android-firefox is hearing two
|
||||
* streams now as listener" — fix the wrong guard, not double-audio
|
||||
* via dedup. */
|
||||
for (const [pubHex, stream] of sfuStreamsByPubHex){
|
||||
for (const [uuid, mm] of members){
|
||||
try {
|
||||
if (mm.pubkey && hex(unb64(mm.pubkey)) === pubHex){
|
||||
if (!remoteAudio.has(uuid)) attachSfuTrack(uuid, stream);
|
||||
if (!remoteAudio.has(uuid) && !listenerAudioNodes.has(uuid)){
|
||||
attachSfuTrack(uuid, stream);
|
||||
}
|
||||
break;
|
||||
}
|
||||
} catch(_){}
|
||||
|
|
@ -3733,6 +3971,12 @@ function renderVideoTile(kind, pubHex, stream, opts){
|
|||
logLine('err','play threw: '+e.message);
|
||||
entry.tile.classList.add('needs-tap');
|
||||
}
|
||||
/* perceptual sync — start measuring motion↔audio offset for this video.
|
||||
* Idempotent for the same videoEl; tears down + re-inits if videoEl
|
||||
* changed (MSID-supplant via swapFreshVideoElement). DETACHED_MEDIA
|
||||
* (screen-share with movie) and AUDIO_ONLY (static frame) modes
|
||||
* auto-quiesce, so blanket-attach is safe. */
|
||||
if (!local) try { attachPerceptualSync(pubHex, entry.video); } catch(_){}
|
||||
/* Spotlight uses DOM-move 2026-06-05 — spotlight.tile === entry.tile
|
||||
* when the same publisher is currently spotlit. The entry swap above
|
||||
* already replaced the live video element in the DOM, so we just
|
||||
|
|
@ -3770,6 +4014,16 @@ function removeVideoTile(kind, pubHex){
|
|||
const entry = k.store.get(pubHex);
|
||||
if (!entry) return;
|
||||
const wasSpotlight = spotlight && spotlight.kind === kind && spotlight.pubHex === pubHex;
|
||||
/* tear down perceptual sync if this pub has no other video tiles left.
|
||||
* Camera and screen for the same pub share one perceptualSync entry
|
||||
* keyed by pubHex; if one tile goes away but another remains, keep
|
||||
* the sync alive on the surviving tile. */
|
||||
const stillHasOther = ['camera','screen','game'].some(otherKind => {
|
||||
if (otherKind === kind) return false;
|
||||
const ok = TILE_KINDS[otherKind];
|
||||
return ok && ok.store.has(pubHex);
|
||||
});
|
||||
if (!stillHasOther) try { detachPerceptualSync(pubHex); } catch(_){}
|
||||
try { entry.video.srcObject = null; entry.tile.remove(); } catch(_){}
|
||||
k.store.delete(pubHex);
|
||||
k.streams.delete(pubHex);
|
||||
|
|
@ -7532,8 +7786,8 @@ logLine('', 'ready — pick a handle, type a rendezvous code, enter the space');
|
|||
|
||||
<footer style="margin:2.2rem auto 0;font-size:0.65rem;color:#999;line-height:1.7;word-break:break-all;font-family:monospace">
|
||||
<span id="pi-seal" style="color:#777;cursor:default;user-select:none" title="">page integrity</span> · built <span class="stamp-date">2026-06-05</span><br>
|
||||
md5 <span class="stamp-md5">9aca7c642e3a0f7ea5ea104cc114cbde</span><br>
|
||||
sha256 <span class="stamp-sha">c295a568dfcf5df8567bce78e5ab90503bcec52fa9ae3a64d433000803dabc2d</span><br>
|
||||
md5 <span class="stamp-md5">00e6a52b14259a52ff7edda1c7617906</span><br>
|
||||
sha256 <span class="stamp-sha">98d5e1f1fe606f4fe3261ea116278d0a584440bdd91a9a01472b07c07e71d78e</span><br>
|
||||
<span style="color:#bbb">hashes are of this page with these two fields zeroed — to verify, blank them and re-hash</span><br>
|
||||
<span style="color:#bbb">one self-contained file — <strong>save a copy</strong> and verify against these hashes; point at your own servers with ?signal= and ?turncred=, or <a href="host-your-own.html" style="color:#999">host your own community</a></span>
|
||||
</footer>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue