diff --git a/src/models.js b/src/models.js index 8190c0d..30f6dd9 100644 --- a/src/models.js +++ b/src/models.js @@ -116,6 +116,7 @@ export async function fetchModelsFromEndpoints() { endpointId: endpoint.id, uniqueId: `${endpoint.id}-${model.id}`, maxTokens: maxTokens || 8192, + maxCompletionTokens: model.max_completion_tokens || model.max_output_tokens || maxTokens || 8192, }; }); } catch (error) { @@ -134,6 +135,7 @@ export async function fetchModelsFromEndpoints() { url: endpoint.url, endpointId: model.endpointId, maxTokens: model.maxTokens, + maxCompletionTokens: model.maxCompletionTokens, modelName: model.modelName, }; } @@ -325,6 +327,26 @@ export function getSelectedModelMaxTokens() { return DEFAULT_MAX_TOKENS; } +// Get max completion tokens for the currently selected model +export function getSelectedModelMaxCompletionTokens() { + // Check modal dropdown first + const modalDropdown = document.getElementById("modal-model-selection"); + if (modalDropdown?.value && modelRegistry[modalDropdown.value]) { + const modelData = modelRegistry[modalDropdown.value]; + return modelData.maxCompletionTokens || DEFAULT_MAX_TOKENS; + } + + // Fallback to main dropdown + const dropdown = document.getElementById("model-selection"); + if (dropdown?.value && modelRegistry[dropdown.value]) { + const modelData = modelRegistry[dropdown.value]; + return modelData.maxCompletionTokens || DEFAULT_MAX_TOKENS; + } + + // Default fallback + return DEFAULT_MAX_TOKENS; +} + // Helper function to get the API endpoint for the selected model. export function getSelectedModelEndpoint() { // Check modal dropdown first diff --git a/src/translation.js b/src/translation.js index ed1585c..e6b52a0 100644 --- a/src/translation.js +++ b/src/translation.js @@ -1,5 +1,5 @@ import { API_KEY, getAPIConfig } from "./config.js"; -import { getSelectedModel, getSelectedModelEndpoint, getSelectedModelMaxTokens, modelRegistry } from "./models.js"; +import { getSelectedModel, getSelectedModelEndpoint, getSelectedModelMaxTokens, getSelectedModelMaxCompletionTokens, modelRegistry } from "./models.js"; import { countTokens, countTokensWithUpstream } from "./token_estimator.js"; // Languages supported by Hermes 3 model (in English for reference) @@ -77,7 +77,12 @@ async function* sendMessageWithHistory(messageHistory) { const inputText = messageHistory.map(msg => msg.content).join(''); const inputTokens = Math.ceil(inputText.length / 4); // Simple token estimation const outputReserve = Math.floor(modelMaxTokens * 0.4); // Reserve 40% for output - const availableTokens = Math.max(512, modelMaxTokens - inputTokens - outputReserve); + let availableTokens = Math.max(512, modelMaxTokens - inputTokens - outputReserve); + + // Cap output tokens based on model's max completion tokens limit + const maxCompletionTokens = getSelectedModelMaxCompletionTokens(); + availableTokens = Math.min(availableTokens, maxCompletionTokens); + console.log(`Capped available tokens to model's max completion limit: ${maxCompletionTokens}`); console.log("Translation using endpoint:", apiUrl); console.log("Translation using model:", model);