engineering/uncloseai.com
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add dynamic max_tokens support: capture from /v1/models API and use in translation

Browse source
This commit is contained in:
Russell Ballestrini 2025-07-11 00:24:33 -04:00
parent 461194ee83
commit 1b571b4ed6
2 changed files with 27 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

23
src/models.js
View file

@ -66,6 +66,7 @@ export async function fetchModelsFromEndpoints() {
modelName: model.id, // Explicitly store model name
endpointId: endpoint.id,
uniqueId: `${endpoint.id}-${model.id}`, // Unique ID with endpoint ID first
maxTokens: model.max_tokens || model.context_length || model.max_context_length || model.max_model_len || 8192, // Capture max tokens
}));
} catch (error) {
console.error(`Error fetching models from ${endpoint.url}:`, error);
@ -82,6 +83,8 @@ export async function fetchModelsFromEndpoints() {
modelRegistry[model.uniqueId] = {
url: endpoint.url,
endpointId: model.endpointId,
maxTokens: model.maxTokens,
modelName: model.modelName,
};
}
});
@ -241,6 +244,26 @@ export function getSelectedModel() {
return "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic";
}
// Get max tokens for the currently selected model
export function getSelectedModelMaxTokens() {
// Check modal dropdown first
const modalDropdown = document.getElementById("hermes-model-selection");
if (modalDropdown?.value && modelRegistry[modalDropdown.value]) {
const modelData = modelRegistry[modalDropdown.value];
return modelData.maxTokens || 8192;
}
// Fallback to main dropdown
const dropdown = document.getElementById("model-selection");
if (dropdown?.value && modelRegistry[dropdown.value]) {
const modelData = modelRegistry[dropdown.value];
return modelData.maxTokens || 8192;
}
// Default fallback
return 8192;
}
// Helper function to get the API endpoint for the selected model.
export function getSelectedModelEndpoint() {
// Check modal dropdown first

6
src/translation.js
View file

@ -1,5 +1,5 @@
import { API_KEY, getAPIConfig } from "./config.js";
import { getSelectedModel, getSelectedModelEndpoint } from "./models.js";
import { getSelectedModel, getSelectedModelEndpoint, getSelectedModelMaxTokens } from "./models.js";
import { countTokens, countTokensWithUpstream } from "./token_estimator.js";
// Languages supported by Hermes 3 model (in English for reference)
@ -70,8 +70,10 @@ async function* sendMessageWithHistory(messageHistory) {
model = getSelectedModel();
}
const maxTokens = getSelectedModelMaxTokens();
console.log("Translation using endpoint:", apiUrl);
console.log("Translation using model:", model);
console.log("Translation using max_tokens:", maxTokens);
const response = await fetch(apiUrl, {
method: "POST",
@ -80,7 +82,7 @@ async function* sendMessageWithHistory(messageHistory) {
model: model,
messages: messageHistory,
temperature: 0.3, // Lower temperature for more consistent translation
max_tokens: 70000,
max_tokens: maxTokens,
stream: true,
}),
});