From 867333f6a5e3e04b6d71b57f8bcbfe48cbabb11c Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Wed, 3 Jun 2026 15:15:48 -0400 Subject: [PATCH] zebra-spaces: exponential backoff + give-up cap on mesh PC retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Telemetry showed Willdabeast's mesh PC (peer edc257b4...) failing every ~20s in a perfect loop for 4+ minutes — the old 1.5s flat retry was firing connectToPeer over and over against a NAT we couldn't traverse. Each cycle ate CPU, network, signaling churn, and contributed to the robot-voice + lost-mic noise we've been chasing. New behavior: - 2s -> 4s -> 8s -> 16s backoff (capped) between attempts - Hard cap at PEER_MESH_MAX_RETRIES = 4 attempts. After that, peerMeshGiveUp.add(uuid): connectToPeer becomes a no-op for that uuid and audio rides on SFU permanently. - Successful 'connected' transition resets the retry counter so a much-later transient blip gets a fresh budget. - peer-left / peer-booted clear all retry state via clearMeshRetryState() so a rejoin from the same uuid starts over. Log lines surface the decision so the next pathological mesh peer shows up in CLIENT_LOG as 'mesh failed Nx — giving up, audio stays on SFU' rather than a wall of identical 'failed — reconnecting' lines. Side benefit: less mesh churn means the SFU mic fallback path (the 2f1e481 + 441c086 fix) gets to settle and stay settled, so receivers don't keep flipping their audio elements between mesh and SFU streams under a failing peer. --- web/zebra-spaces.html | 72 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 14 deletions(-) diff --git a/web/zebra-spaces.html b/web/zebra-spaces.html index 15b9795..d32fe3f 100644 --- a/web/zebra-spaces.html +++ b/web/zebra-spaces.html @@ -3192,6 +3192,7 @@ async function handleSignal(raw){ handraise.delete(m.uuid); spotlights.delete(m.uuid); tearPeer(m.uuid); + clearMeshRetryState(m.uuid); if (hostUUID === m.uuid) hostUUID = ''; renderRoom(); reorderTiles(); @@ -3362,7 +3363,7 @@ async function handleSignal(raw){ removeVideoTile('gameshare', victPub); } catch(_){} } - members.delete(m.uuid); tearPeer(m.uuid); handraise.delete(m.uuid); + members.delete(m.uuid); tearPeer(m.uuid); clearMeshRetryState(m.uuid); handraise.delete(m.uuid); renderRoom(); } break; @@ -3518,8 +3519,30 @@ function updateRoleUI(){ * by tear + reconnect with the same rule, so the same side always * drives recovery. * ================================================================== */ +/* per-peer retry tracking — repeated mesh failures (NAT/firewall the + * page can't traverse, even with TURN) used to trigger a tight 1.5s + * reconnect loop forever. Each cycle ate CPU/network and added to + * the robot-voice/lost-mic noise we keep chasing. Now we exponential- + * backoff and after PEER_MESH_MAX_RETRIES we give up the mesh entirely + * for that peer and let SFU carry the audio. peer-left clears the + * tracking, so a fresh join from the same uuid starts a new budget. */ +const PEER_MESH_MAX_RETRIES = 4; +const peerMeshGiveUp = new Set(); // uuids we've stopped trying to mesh +const peerMeshRetries = new Map(); // uuid -> attempt count +const peerMeshTimers = new Map(); // uuid -> pending setTimeout id +function clearMeshRetryState(uuid){ + peerMeshGiveUp.delete(uuid); + peerMeshRetries.delete(uuid); + const t = peerMeshTimers.get(uuid); + if (t){ clearTimeout(t); peerMeshTimers.delete(uuid); } +} + async function connectToPeer(uuid, weOffer){ if (peers.has(uuid)) return; + if (peerMeshGiveUp.has(uuid)){ + logLine('', 'peer '+uuid+' mesh disabled (max retries) — staying on SFU'); + return; + } if (!micStream){ try { await getMic(); } catch(e){ logLine('err','mic for '+uuid+': '+e.message); return; } } const pc = new RTCPeerConnection(rtcConfig); peers.set(uuid, pc); @@ -3537,18 +3560,39 @@ async function connectToPeer(uuid, weOffer){ }; pc.onicecandidate = (ev) => { /* using waitForIceGathering pattern, candidates ignored */ }; pc.onconnectionstatechange = () => { - if (pc.connectionState === 'failed' && peers.get(uuid) === pc){ - logLine('', 'peer '+uuid+' failed — reconnecting'); - tearPeer(uuid); - /* Mesh PC just died; the audio element for this peer was bound to - * the dying mesh stream and won't recover on its own. Switch back - * to the cached SFU stream so the user keeps hearing them while - * mesh reconnect attempts run in the background. */ - try { attachCachedSfuStreamFor(uuid); } catch(_){} - /* let the offerer drive recovery */ - setTimeout(()=>{ if (members.has(uuid) && canSpeak(members.get(uuid).role) && canSpeak(myRole)) - connectToPeer(uuid, myUUID < uuid); }, 1500); + if (pc.connectionState === 'connected'){ + /* successful connect — reset the retry budget so a much-later + * transient failure gets a fresh round of attempts. */ + peerMeshRetries.delete(uuid); + return; } + if (pc.connectionState !== 'failed' || peers.get(uuid) !== pc) return; + + const attempts = (peerMeshRetries.get(uuid) || 0) + 1; + peerMeshRetries.set(uuid, attempts); + tearPeer(uuid); + /* Mesh PC just died; the audio element for this peer was bound to + * the dying mesh stream and won't recover on its own. Switch back + * to the cached SFU stream so the user keeps hearing them while + * mesh reconnect attempts run in the background. */ + try { attachCachedSfuStreamFor(uuid); } catch(_){} + + if (attempts >= PEER_MESH_MAX_RETRIES){ + peerMeshGiveUp.add(uuid); + logLine('err', 'peer '+uuid+' mesh failed '+attempts+'x — giving up, audio stays on SFU'); + return; + } + /* exponential backoff: 2s, 4s, 8s, 16s (capped). Keeps the room + * from melting when a peer's NAT genuinely can't mesh through. */ + const delayMs = Math.min(16000, 2000 * Math.pow(2, attempts - 1)); + logLine('', 'peer '+uuid+' failed — reconnecting in '+(delayMs/1000)+'s (attempt '+attempts+'/'+PEER_MESH_MAX_RETRIES+')'); + const t = setTimeout(() => { + peerMeshTimers.delete(uuid); + if (members.has(uuid) && canSpeak(members.get(uuid).role) && canSpeak(myRole)){ + connectToPeer(uuid, myUUID < uuid); + } + }, delayMs); + peerMeshTimers.set(uuid, t); }; if (weOffer){ const offer = await pc.createOffer(); @@ -4127,8 +4171,8 @@ logLine('', 'ready — pick a handle, type a rendezvous code, enter the space');