create centralized Groq TPM rate limit handling for all requests
- new groq-fetch.js provides groqAwareStreamingFetch for all API calls - automatically detects Groq endpoints and handles TPM rate limits - works for chat, translation, TTS - any request to Groq - waits 60 seconds for TPM window reset, retries up to 3 times - other platforms (Hermes, OpenAI, etc.) unaffected - cleaner abstraction replaces inline retry logic in chat.js
This commit is contained in:
parent
bd10045c4a
commit
eff2eee69b
2 changed files with 124 additions and 51 deletions
67
src/chat.js
67
src/chat.js
|
|
@ -3,6 +3,7 @@ import { marked } from "https://cdn.jsdelivr.net/npm/marked/lib/marked.esm.js";
|
|||
import hljs from "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/es/highlight.min.js";
|
||||
import { API_KEY, getAPIConfig } from "./config.js";
|
||||
import { getSelectedModel, getSelectedModelEndpoint, getSelectedModelMaxTokens, getSelectedModelMaxCompletionTokens } from "./models.js";
|
||||
import { groqAwareStreamingFetch } from "./groq-fetch.js";
|
||||
|
||||
// Rough token estimation (4 chars = 1 token average for English)
|
||||
function estimateTokens(text) {
|
||||
|
|
@ -115,60 +116,24 @@ export async function* sendMessage(message) {
|
|||
|
||||
console.log("Chat using tokens:", finalAvailable, "calculated:", calculatedAvailable, "max completion:", maxCompletionTokens);
|
||||
|
||||
let response;
|
||||
let retryCount = 0;
|
||||
const maxRetries = 3;
|
||||
|
||||
while (retryCount < maxRetries) {
|
||||
try {
|
||||
response = await fetch(apiUrl, {
|
||||
method: "POST",
|
||||
headers: headers,
|
||||
body: JSON.stringify({
|
||||
model: model,
|
||||
messages: chatHistory,
|
||||
temperature: 0.5,
|
||||
max_tokens: finalAvailable,
|
||||
stream: true,
|
||||
}),
|
||||
});
|
||||
|
||||
// Check for rate limit error (Groq-specific handling)
|
||||
if ((response.status === 429 || response.status === 400) && apiUrl.includes('groq.com')) {
|
||||
const errorText = await response.text();
|
||||
if (errorText.includes('rate_limit_exceeded')) {
|
||||
console.log(`Groq rate limit hit, waiting 60 seconds before retry ${retryCount + 1}/${maxRetries}...`);
|
||||
yield "⏳ Groq rate limit reached, waiting 60 seconds for TPM window to reset...";
|
||||
await new Promise(resolve => setTimeout(resolve, 60000)); // Wait 60 seconds for TPM reset
|
||||
retryCount++;
|
||||
continue;
|
||||
} else {
|
||||
// Non-rate-limit error, throw it
|
||||
throw new Error(`API Error ${response.status}: ${errorText}`);
|
||||
}
|
||||
}
|
||||
|
||||
// If we get here, either success or non-rate-limit error
|
||||
break;
|
||||
} catch (error) {
|
||||
console.error("Fetch error:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
if (retryCount >= maxRetries) {
|
||||
throw new Error("Rate limit exceeded after multiple retries. Please try again later.");
|
||||
}
|
||||
const requestBody = JSON.stringify({
|
||||
model: model,
|
||||
messages: chatHistory,
|
||||
temperature: 0.5,
|
||||
max_tokens: finalAvailable,
|
||||
stream: true,
|
||||
});
|
||||
|
||||
const streamGenerator = groqAwareStreamingFetch(apiUrl, {
|
||||
method: "POST",
|
||||
headers: headers,
|
||||
body: requestBody,
|
||||
});
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
for await (const chunk of streamGenerator) {
|
||||
buffer += chunk;
|
||||
const lines = buffer.split("\n");
|
||||
|
||||
for (let i = 0; i < lines.length - 1; i++) {
|
||||
|
|
|
|||
108
src/groq-fetch.js
Normal file
108
src/groq-fetch.js
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
// Groq-aware fetch wrapper that handles TPM rate limits
|
||||
// Use this for all API requests to Groq endpoints
|
||||
|
||||
export async function groqAwareFetch(url, options = {}) {
|
||||
// If not a Groq URL, use regular fetch
|
||||
if (!url.includes('groq.com')) {
|
||||
return fetch(url, options);
|
||||
}
|
||||
|
||||
// Groq-specific retry logic for TPM rate limits
|
||||
let retryCount = 0;
|
||||
const maxRetries = 3;
|
||||
|
||||
while (retryCount < maxRetries) {
|
||||
try {
|
||||
const response = await fetch(url, options);
|
||||
|
||||
// Check for Groq rate limit error
|
||||
if (response.status === 429 || response.status === 400) {
|
||||
const responseClone = response.clone(); // Clone so we can read body twice
|
||||
const errorText = await responseClone.text();
|
||||
|
||||
if (errorText.includes('rate_limit_exceeded') && errorText.includes('TPM')) {
|
||||
console.log(`Groq TPM rate limit hit, waiting 60 seconds before retry ${retryCount + 1}/${maxRetries}...`);
|
||||
|
||||
// Wait 60 seconds for TPM window to reset
|
||||
await new Promise(resolve => setTimeout(resolve, 60000));
|
||||
retryCount++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Return response (success or non-rate-limit error)
|
||||
return response;
|
||||
|
||||
} catch (error) {
|
||||
// Network errors etc - don't retry these
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// If we get here, we've exhausted retries
|
||||
throw new Error("Groq TPM rate limit exceeded after multiple retries. Please try again later.");
|
||||
}
|
||||
|
||||
// Generator version for streaming responses
|
||||
export async function* groqAwareStreamingFetch(url, options = {}) {
|
||||
// If not a Groq URL, use regular fetch and pass through
|
||||
if (!url.includes('groq.com')) {
|
||||
const response = await fetch(url, options);
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
yield decoder.decode(value, { stream: true });
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Groq-specific retry logic for TPM rate limits
|
||||
let retryCount = 0;
|
||||
const maxRetries = 3;
|
||||
|
||||
while (retryCount < maxRetries) {
|
||||
try {
|
||||
const response = await fetch(url, options);
|
||||
|
||||
// Check for Groq rate limit error
|
||||
if (response.status === 429 || response.status === 400) {
|
||||
const responseClone = response.clone();
|
||||
const errorText = await responseClone.text();
|
||||
|
||||
if (errorText.includes('rate_limit_exceeded') && errorText.includes('TPM')) {
|
||||
console.log(`Groq TPM rate limit hit, waiting 60 seconds before retry ${retryCount + 1}/${maxRetries}...`);
|
||||
yield "⏳ Groq rate limit reached, waiting 60 seconds for TPM window to reset...";
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 60000));
|
||||
retryCount++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Success or non-rate-limit error - process streaming response
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`HTTP error! status: ${response.status}: ${errorText}`);
|
||||
}
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
yield decoder.decode(value, { stream: true });
|
||||
}
|
||||
return;
|
||||
|
||||
} catch (error) {
|
||||
// Network errors etc - don't retry these
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error("Groq TPM rate limit exceeded after multiple retries. Please try again later.");
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue