engineering/uncloseai.com
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

use API-provided max completion tokens instead of hardcoding

Browse source 
Fetch and store maxCompletionTokens from models endpoint to respect
each model's specific output limits. This fixes 400 errors from
Groq models that have different completion limits than context windows.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Russell Ballestrini 2025-07-12 07:36:31 -04:00
parent a864366c80
commit 41ba59db1b
2 changed files with 29 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

22
src/models.js
View file

@ -116,6 +116,7 @@ export async function fetchModelsFromEndpoints() {
endpointId: endpoint.id,
uniqueId: `${endpoint.id}-${model.id}`,
maxTokens: maxTokens || 8192,
maxCompletionTokens: model.max_completion_tokens || model.max_output_tokens || maxTokens || 8192,
};
});
} catch (error) {
@ -134,6 +135,7 @@ export async function fetchModelsFromEndpoints() {
url: endpoint.url,
endpointId: model.endpointId,
maxTokens: model.maxTokens,
maxCompletionTokens: model.maxCompletionTokens,
modelName: model.modelName,
};
}
@ -325,6 +327,26 @@ export function getSelectedModelMaxTokens() {
return DEFAULT_MAX_TOKENS;
}
// Get max completion tokens for the currently selected model
export function getSelectedModelMaxCompletionTokens() {
// Check modal dropdown first
const modalDropdown = document.getElementById("modal-model-selection");
if (modalDropdown?.value && modelRegistry[modalDropdown.value]) {
const modelData = modelRegistry[modalDropdown.value];
return modelData.maxCompletionTokens || DEFAULT_MAX_TOKENS;
}
// Fallback to main dropdown
const dropdown = document.getElementById("model-selection");
if (dropdown?.value && modelRegistry[dropdown.value]) {
const modelData = modelRegistry[dropdown.value];
return modelData.maxCompletionTokens || DEFAULT_MAX_TOKENS;
}
// Default fallback
return DEFAULT_MAX_TOKENS;
}
// Helper function to get the API endpoint for the selected model.
export function getSelectedModelEndpoint() {
// Check modal dropdown first

9
src/translation.js
View file

@ -1,5 +1,5 @@
import { API_KEY, getAPIConfig } from "./config.js";
import { getSelectedModel, getSelectedModelEndpoint, getSelectedModelMaxTokens, modelRegistry } from "./models.js";
import { getSelectedModel, getSelectedModelEndpoint, getSelectedModelMaxTokens, getSelectedModelMaxCompletionTokens, modelRegistry } from "./models.js";
import { countTokens, countTokensWithUpstream } from "./token_estimator.js";
// Languages supported by Hermes 3 model (in English for reference)
@ -77,7 +77,12 @@ async function* sendMessageWithHistory(messageHistory) {
const inputText = messageHistory.map(msg => msg.content).join('');
const inputTokens = Math.ceil(inputText.length / 4); // Simple token estimation
const outputReserve = Math.floor(modelMaxTokens * 0.4); // Reserve 40% for output
const availableTokens = Math.max(512, modelMaxTokens - inputTokens - outputReserve);
let availableTokens = Math.max(512, modelMaxTokens - inputTokens - outputReserve);
// Cap output tokens based on model's max completion tokens limit
const maxCompletionTokens = getSelectedModelMaxCompletionTokens();
availableTokens = Math.min(availableTokens, maxCompletionTokens);
console.log(`Capped available tokens to model's max completion limit: ${maxCompletionTokens}`);
console.log("Translation using endpoint:", apiUrl);
console.log("Translation using model:", model);