diff --git a/web/zebra-spaces.html b/web/zebra-spaces.html
index 77bd97d..8c48c75 100644
--- a/web/zebra-spaces.html
+++ b/web/zebra-spaces.html
@@ -2616,6 +2616,226 @@ function registerLipSyncVideo(pubHex, kind, receiver){
const e = lipSyncEntry(pubHex);
e.videoReceivers.set(kind, receiver);
}
+
+/* ====== perceptual lip-sync ===============================================
+ * The buffer-derived target above knows what was BUFFERED, not what reached
+ * the user. PipeWire / kernel resamplers / the listener's mobile audio stack
+ * add hidden delay the worklet can't see. Result: visible drift in both
+ * directions depending on which side has more local indirection that hour.
+ *
+ * This module measures the actual visual-vs-audio offset by cross-correlating
+ * video motion energy against audio RMS over a rolling 5 s window. Output:
+ * a per-pub correction (ms) added to refreshLipSyncForUuid's playoutDelay
+ * target.
+ *
+ * Three modes, auto-detected per pub:
+ * TALKING_HEAD — high motion↔audio correlation → apply correction
+ * DETACHED_MEDIA — motion exists but uncorrelated (movie / slides) → idle
+ * AUDIO_ONLY — no useful motion → idle, save battery
+ */
+const perceptualSync = new Map(); /* pubHex → state */
+const PERCEPTUAL_SAMPLE_MS = 100;
+const PERCEPTUAL_BUF_LEN = 50; /* 5 s window */
+const PERCEPTUAL_MAX_LAG = 5; /* ±500 ms search range */
+const PERCEPTUAL_CORR_GATE = 0.35; /* TALKING_HEAD entry threshold */
+const PERCEPTUAL_MOTION_GATE = 0.004; /* DETACHED vs AUDIO_ONLY */
+const PERCEPTUAL_ENTER_DEBOUNCE_MS = 500;
+const PERCEPTUAL_LEAVE_DEBOUNCE_MS = 3000;
+const PERCEPTUAL_OFFSET_CLAMP_MS = 300;
+const PERCEPTUAL_EMA_ALPHA = 0.3;
+const PERCEPTUAL_LOG_THROTTLE_MS = 5000;
+
+function attachPerceptualSync(pubHex, videoEl, retries){
+ retries = retries || 0;
+ if (!videoEl || !audioCtx) return;
+ const existing = perceptualSync.get(pubHex);
+ if (existing){
+ if (existing.videoEl === videoEl) return;
+ detachPerceptualSync(pubHex);
+ }
+ const e = lipSync.get(pubHex);
+ if (!e || !e.audioUuid){
+ /* audio side not registered yet — retry a few times */
+ if (retries < 10){
+ setTimeout(() => attachPerceptualSync(pubHex, videoEl, retries + 1), 500);
+ }
+ return;
+ }
+ const node = listenerAudioNodes.get(e.audioUuid);
+ if (!node || !node.gain){
+ if (retries < 10){
+ setTimeout(() => attachPerceptualSync(pubHex, videoEl, retries + 1), 500);
+ }
+ return;
+ }
+ let analyser;
+ try {
+ analyser = audioCtx.createAnalyser();
+ analyser.fftSize = 1024;
+ analyser.smoothingTimeConstant = 0;
+ node.gain.connect(analyser);
+ } catch(err){
+ logLine('err','perceptual analyser failed: '+err.message);
+ return;
+ }
+ const canvas = document.createElement('canvas');
+ canvas.width = 32; canvas.height = 32;
+ const ctx2d = canvas.getContext('2d', { willReadFrequently: true });
+ const st = {
+ pubHex,
+ videoEl,
+ analyser,
+ audioSource: node.gain,
+ canvas, ctx2d,
+ audioBuf: new Float32Array(PERCEPTUAL_BUF_LEN),
+ motionBuf: new Float32Array(PERCEPTUAL_BUF_LEN),
+ prevPixels: null,
+ idx: 0, filled: 0,
+ mode: 'AUDIO_ONLY',
+ pendingMode: null, pendingSince: 0,
+ offsetMs: 0,
+ lastCorrAt: 0,
+ lastLogAt: 0,
+ sampleScratch: new Float32Array(analyser.fftSize),
+ };
+ st.interval = setInterval(() => perceptualTick(pubHex), PERCEPTUAL_SAMPLE_MS);
+ perceptualSync.set(pubHex, st);
+}
+function detachPerceptualSync(pubHex){
+ const st = perceptualSync.get(pubHex);
+ if (!st) return;
+ try { clearInterval(st.interval); } catch(_){}
+ try { st.audioSource.disconnect(st.analyser); } catch(_){}
+ try { st.analyser.disconnect(); } catch(_){}
+ perceptualSync.delete(pubHex);
+}
+function perceptualOffsetMs(pubHex){
+ const st = perceptualSync.get(pubHex);
+ if (!st || st.mode !== 'TALKING_HEAD') return null;
+ return st.offsetMs;
+}
+function perceptualTick(pubHex){
+ const st = perceptualSync.get(pubHex);
+ if (!st) return;
+ const v = st.videoEl;
+ if (!v || v.readyState < 2 || v.videoWidth === 0){
+ perceptualTransition(st, 'AUDIO_ONLY');
+ return;
+ }
+ try { st.analyser.getFloatTimeDomainData(st.sampleScratch); }
+ catch(_){ return; }
+ let sum = 0;
+ for (let i = 0; i < st.sampleScratch.length; i++){
+ sum += st.sampleScratch[i] * st.sampleScratch[i];
+ }
+ const audioE = Math.sqrt(sum / st.sampleScratch.length);
+ let motionE = 0;
+ try {
+ st.ctx2d.drawImage(v, 0, 0, 32, 32);
+ const img = st.ctx2d.getImageData(0, 0, 32, 32).data;
+ if (st.prevPixels && st.prevPixels.length === img.length){
+ let d = 0;
+ for (let i = 0; i < img.length; i += 4){
+ const lumNow = 0.299*img[i] + 0.587*img[i+1] + 0.114*img[i+2];
+ const lumPrev = 0.299*st.prevPixels[i] + 0.587*st.prevPixels[i+1] + 0.114*st.prevPixels[i+2];
+ d += Math.abs(lumNow - lumPrev);
+ }
+ motionE = d / (img.length / 4) / 255;
+ }
+ st.prevPixels = img;
+ } catch(_){
+ /* cross-origin tainted MediaStream or detached video — stop trying */
+ perceptualTransition(st, 'AUDIO_ONLY');
+ return;
+ }
+ st.audioBuf[st.idx] = audioE;
+ st.motionBuf[st.idx] = motionE;
+ st.idx = (st.idx + 1) % PERCEPTUAL_BUF_LEN;
+ if (st.filled < PERCEPTUAL_BUF_LEN) st.filled++;
+ if (st.filled < PERCEPTUAL_BUF_LEN) return;
+ const now = performance.now();
+ if (now - st.lastCorrAt < 1000) return;
+ st.lastCorrAt = now;
+ const corr = perceptualCorrelate(st.audioBuf, st.motionBuf, st.idx, PERCEPTUAL_MAX_LAG);
+ let motionRms = 0;
+ for (let i = 0; i < PERCEPTUAL_BUF_LEN; i++){
+ motionRms += st.motionBuf[i] * st.motionBuf[i];
+ }
+ motionRms = Math.sqrt(motionRms / PERCEPTUAL_BUF_LEN);
+ let desired = 'AUDIO_ONLY';
+ if (corr.peak > PERCEPTUAL_CORR_GATE) desired = 'TALKING_HEAD';
+ else if (motionRms > PERCEPTUAL_MOTION_GATE) desired = 'DETACHED_MEDIA';
+ perceptualTransition(st, desired);
+ if (st.mode === 'TALKING_HEAD'){
+ /* +lag = audio leads motion samples → video late → reduce delay */
+ const rawMs = -corr.bestLag * PERCEPTUAL_SAMPLE_MS;
+ const clamped = Math.max(-PERCEPTUAL_OFFSET_CLAMP_MS,
+ Math.min(PERCEPTUAL_OFFSET_CLAMP_MS, rawMs));
+ st.offsetMs = PERCEPTUAL_EMA_ALPHA * clamped + (1 - PERCEPTUAL_EMA_ALPHA) * st.offsetMs;
+ if (now - st.lastLogAt > PERCEPTUAL_LOG_THROTTLE_MS){
+ st.lastLogAt = now;
+ logLine('', 'perceptual pub='+pubHex.slice(0,4)+
+ ' mode=TALKING corr='+corr.peak.toFixed(2)+
+ ' lag='+(corr.bestLag * PERCEPTUAL_SAMPLE_MS)+'ms'+
+ ' offset='+st.offsetMs.toFixed(0)+'ms');
+ }
+ } else if (st.offsetMs !== 0){
+ st.offsetMs = 0;
+ }
+}
+function perceptualTransition(st, desired){
+ if (st.mode === desired){
+ st.pendingMode = null;
+ return;
+ }
+ const now = performance.now();
+ const debounce = (desired === 'TALKING_HEAD')
+ ? PERCEPTUAL_ENTER_DEBOUNCE_MS
+ : PERCEPTUAL_LEAVE_DEBOUNCE_MS;
+ if (st.pendingMode !== desired){
+ st.pendingMode = desired;
+ st.pendingSince = now;
+ return;
+ }
+ if (now - st.pendingSince < debounce) return;
+ const prev = st.mode;
+ st.mode = desired;
+ st.pendingMode = null;
+ if (desired !== 'TALKING_HEAD') st.offsetMs = 0;
+ logLine('', 'perceptual pub='+st.pubHex.slice(0,4)+' mode '+prev+' → '+desired);
+}
+function perceptualCorrelate(audioCirc, motionCirc, idx, maxLag){
+ const N = audioCirc.length;
+ const a = new Float32Array(N), m = new Float32Array(N);
+ for (let i = 0; i < N; i++){
+ a[i] = audioCirc[(idx + i) % N];
+ m[i] = motionCirc[(idx + i) % N];
+ }
+ let am = 0, mm = 0;
+ for (let i = 0; i < N; i++){ am += a[i]; mm += m[i]; }
+ am /= N; mm /= N;
+ let va = 0, vm = 0;
+ for (let i = 0; i < N; i++){
+ a[i] -= am; m[i] -= mm;
+ va += a[i]*a[i]; vm += m[i]*m[i];
+ }
+ const denom = Math.sqrt(va * vm);
+ if (denom < 1e-9) return { peak: 0, bestLag: 0 };
+ let peak = 0, bestLag = 0;
+ for (let L = -maxLag; L <= maxLag; L++){
+ let s = 0, n = 0;
+ for (let i = 0; i < N; i++){
+ const j = i + L;
+ if (j < 0 || j >= N) continue;
+ s += a[i] * m[j];
+ n++;
+ }
+ if (n < N / 2) continue;
+ const c = s / denom;
+ if (c > peak){ peak = c; bestLag = L; }
+ }
+ return { peak, bestLag };
+}
function medianOf(arr){
if (arr.length === 0) return 0;
const s = arr.slice().sort((a,b) => a-b);
@@ -2685,13 +2905,19 @@ async function refreshLipSyncForUuid(uuid){
while (e.history.length > LIP_SYNC_HISTORY) e.history.shift();
if (e.history.length < 3) return; /* wait for ≥3 samples */
const med = medianOf(e.history);
- if (Math.abs(med - e.lastApplied) < LIP_SYNC_THRESHOLD) return;
+ /* perceptual correction: ground-truth offset from motion↔audio correlation.
+ * Null when no TALKING_HEAD signal — we then trust the buffer median alone. */
+ const percOff = perceptualOffsetMs(pubHex);
+ const adj = (percOff !== null) ? med + percOff / 1000 : med;
+ const target = Math.max(0.05, Math.min(5.0, adj));
+ if (Math.abs(target - e.lastApplied) < LIP_SYNC_THRESHOLD) return;
for (const [kind, rx] of e.videoReceivers){
- try { rx.playoutDelayHint = med; } catch(_){}
- try { rx.jitterBufferTarget = med * 1000; } catch(_){}
+ try { rx.playoutDelayHint = target; } catch(_){}
+ try { rx.jitterBufferTarget = target * 1000; } catch(_){}
}
- e.lastApplied = med;
- logLine('', 'lip-sync pub='+pubHex.slice(0,4)+' delay='+med.toFixed(2)+'s ('+e.videoReceivers.size+' video rx, history median of '+e.history.length+')');
+ e.lastApplied = target;
+ const percTag = (percOff !== null) ? ' perc='+percOff.toFixed(0)+'ms' : '';
+ logLine('', 'lip-sync pub='+pubHex.slice(0,4)+' delay='+target.toFixed(2)+'s ('+e.videoReceivers.size+' video rx, median='+med.toFixed(2)+'s'+percTag+')');
}
/* ==================================================================
@@ -3195,11 +3421,23 @@ function attachCachedSfuStreamFor(uuid){
if (stream) attachSfuTrack(uuid, stream);
}
function flushSfuStreams(){
+ /* Skip if EITHER attach map already has the uuid: remoteAudio is the
+ *