From eff2eee69b39fe90820e6c5150aaf203de8269db Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Sat, 12 Jul 2025 13:50:01 -0400 Subject: [PATCH] create centralized Groq TPM rate limit handling for all requests - new groq-fetch.js provides groqAwareStreamingFetch for all API calls - automatically detects Groq endpoints and handles TPM rate limits - works for chat, translation, TTS - any request to Groq - waits 60 seconds for TPM window reset, retries up to 3 times - other platforms (Hermes, OpenAI, etc.) unaffected - cleaner abstraction replaces inline retry logic in chat.js --- src/chat.js | 67 +++++++--------------------- src/groq-fetch.js | 108 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 51 deletions(-) create mode 100644 src/groq-fetch.js diff --git a/src/chat.js b/src/chat.js index 973d7ec..0300f50 100644 --- a/src/chat.js +++ b/src/chat.js @@ -3,6 +3,7 @@ import { marked } from "https://cdn.jsdelivr.net/npm/marked/lib/marked.esm.js"; import hljs from "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/es/highlight.min.js"; import { API_KEY, getAPIConfig } from "./config.js"; import { getSelectedModel, getSelectedModelEndpoint, getSelectedModelMaxTokens, getSelectedModelMaxCompletionTokens } from "./models.js"; +import { groqAwareStreamingFetch } from "./groq-fetch.js"; // Rough token estimation (4 chars = 1 token average for English) function estimateTokens(text) { @@ -115,60 +116,24 @@ export async function* sendMessage(message) { console.log("Chat using tokens:", finalAvailable, "calculated:", calculatedAvailable, "max completion:", maxCompletionTokens); - let response; - let retryCount = 0; - const maxRetries = 3; - - while (retryCount < maxRetries) { - try { - response = await fetch(apiUrl, { - method: "POST", - headers: headers, - body: JSON.stringify({ - model: model, - messages: chatHistory, - temperature: 0.5, - max_tokens: finalAvailable, - stream: true, - }), - }); - - // Check for rate limit error (Groq-specific handling) - if ((response.status === 429 || response.status === 400) && apiUrl.includes('groq.com')) { - const errorText = await response.text(); - if (errorText.includes('rate_limit_exceeded')) { - console.log(`Groq rate limit hit, waiting 60 seconds before retry ${retryCount + 1}/${maxRetries}...`); - yield "⏳ Groq rate limit reached, waiting 60 seconds for TPM window to reset..."; - await new Promise(resolve => setTimeout(resolve, 60000)); // Wait 60 seconds for TPM reset - retryCount++; - continue; - } else { - // Non-rate-limit error, throw it - throw new Error(`API Error ${response.status}: ${errorText}`); - } - } - - // If we get here, either success or non-rate-limit error - break; - } catch (error) { - console.error("Fetch error:", error); - throw error; - } - } - - if (retryCount >= maxRetries) { - throw new Error("Rate limit exceeded after multiple retries. Please try again later."); - } + const requestBody = JSON.stringify({ + model: model, + messages: chatHistory, + temperature: 0.5, + max_tokens: finalAvailable, + stream: true, + }); + + const streamGenerator = groqAwareStreamingFetch(apiUrl, { + method: "POST", + headers: headers, + body: requestBody, + }); - const reader = response.body.getReader(); - const decoder = new TextDecoder(); let buffer = ""; - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - buffer += decoder.decode(value, { stream: true }); + for await (const chunk of streamGenerator) { + buffer += chunk; const lines = buffer.split("\n"); for (let i = 0; i < lines.length - 1; i++) { diff --git a/src/groq-fetch.js b/src/groq-fetch.js new file mode 100644 index 0000000..09a8a5a --- /dev/null +++ b/src/groq-fetch.js @@ -0,0 +1,108 @@ +// Groq-aware fetch wrapper that handles TPM rate limits +// Use this for all API requests to Groq endpoints + +export async function groqAwareFetch(url, options = {}) { + // If not a Groq URL, use regular fetch + if (!url.includes('groq.com')) { + return fetch(url, options); + } + + // Groq-specific retry logic for TPM rate limits + let retryCount = 0; + const maxRetries = 3; + + while (retryCount < maxRetries) { + try { + const response = await fetch(url, options); + + // Check for Groq rate limit error + if (response.status === 429 || response.status === 400) { + const responseClone = response.clone(); // Clone so we can read body twice + const errorText = await responseClone.text(); + + if (errorText.includes('rate_limit_exceeded') && errorText.includes('TPM')) { + console.log(`Groq TPM rate limit hit, waiting 60 seconds before retry ${retryCount + 1}/${maxRetries}...`); + + // Wait 60 seconds for TPM window to reset + await new Promise(resolve => setTimeout(resolve, 60000)); + retryCount++; + continue; + } + } + + // Return response (success or non-rate-limit error) + return response; + + } catch (error) { + // Network errors etc - don't retry these + throw error; + } + } + + // If we get here, we've exhausted retries + throw new Error("Groq TPM rate limit exceeded after multiple retries. Please try again later."); +} + +// Generator version for streaming responses +export async function* groqAwareStreamingFetch(url, options = {}) { + // If not a Groq URL, use regular fetch and pass through + if (!url.includes('groq.com')) { + const response = await fetch(url, options); + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + yield decoder.decode(value, { stream: true }); + } + return; + } + + // Groq-specific retry logic for TPM rate limits + let retryCount = 0; + const maxRetries = 3; + + while (retryCount < maxRetries) { + try { + const response = await fetch(url, options); + + // Check for Groq rate limit error + if (response.status === 429 || response.status === 400) { + const responseClone = response.clone(); + const errorText = await responseClone.text(); + + if (errorText.includes('rate_limit_exceeded') && errorText.includes('TPM')) { + console.log(`Groq TPM rate limit hit, waiting 60 seconds before retry ${retryCount + 1}/${maxRetries}...`); + yield "⏳ Groq rate limit reached, waiting 60 seconds for TPM window to reset..."; + + await new Promise(resolve => setTimeout(resolve, 60000)); + retryCount++; + continue; + } + } + + // Success or non-rate-limit error - process streaming response + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`HTTP error! status: ${response.status}: ${errorText}`); + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + yield decoder.decode(value, { stream: true }); + } + return; + + } catch (error) { + // Network errors etc - don't retry these + throw error; + } + } + + throw new Error("Groq TPM rate limit exceeded after multiple retries. Please try again later."); +} \ No newline at end of file