Russell Ballestrini
41ba59db1b
Fetch and store maxCompletionTokens from models endpoint to respect each model's specific output limits. This fixes 400 errors from Groq models that have different completion limits than context windows. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
377 lines
13 KiB
JavaScript
377 lines
13 KiB
JavaScript
// Model registry and selection functionality
|
|
import { VLLM_ENDPOINTS, getAPIConfig, DEFAULT_MAX_TOKENS } from "./config.js";
|
|
|
|
// This registry maps a model's ID to the endpoint where it resides.
|
|
export const modelRegistry = {};
|
|
|
|
// Fetch models from each endpoint with caching.
|
|
// Cache is busted if the endpoint array changes or TTL.
|
|
export async function fetchModelsFromEndpoints() {
|
|
const cacheKey = "modelRegistryCache";
|
|
const endpointsKey = "vllmEndpointsHash";
|
|
|
|
// Include custom API config in cache key to bust cache when it changes
|
|
const customAPIConfig = {
|
|
useCustomAPI: localStorage.getItem("useCustomAPI"),
|
|
customBaseURL: localStorage.getItem("customBaseURL"),
|
|
customAPIKey: localStorage.getItem("customAPIKey")
|
|
};
|
|
const endpointsString = JSON.stringify({
|
|
vllm: VLLM_ENDPOINTS,
|
|
custom: customAPIConfig
|
|
});
|
|
|
|
const cachedEndpoints = localStorage.getItem(endpointsKey);
|
|
const cacheItem = localStorage.getItem(cacheKey);
|
|
const now = Date.now();
|
|
const TTL = 300000; // 5 minutes in milliseconds
|
|
|
|
if (cacheItem && cachedEndpoints === endpointsString) {
|
|
try {
|
|
const cachedData = JSON.parse(cacheItem);
|
|
if (now - cachedData.timestamp < TTL) {
|
|
// Restore cached modelRegistry
|
|
Object.assign(modelRegistry, cachedData.modelRegistry);
|
|
return cachedData.models;
|
|
}
|
|
} catch (e) {
|
|
console.error("Error reading model registry from cache", e);
|
|
}
|
|
}
|
|
|
|
// Collect endpoints to fetch from
|
|
const endpointsToFetch = [...VLLM_ENDPOINTS];
|
|
|
|
// Add custom API endpoint if the checkbox is checked (regardless of current model routing)
|
|
const useCustomAPI = localStorage.getItem("useCustomAPI") === "true";
|
|
const customBaseURL = localStorage.getItem("customBaseURL");
|
|
|
|
|
|
if (useCustomAPI && customBaseURL) {
|
|
console.log("🔧 Adding custom API endpoint to model fetch:", customBaseURL);
|
|
endpointsToFetch.push({
|
|
id: customBaseURL,
|
|
url: customBaseURL
|
|
});
|
|
} else {
|
|
console.log("🏠 No custom API configured, using default endpoints only");
|
|
}
|
|
|
|
console.log("📡 Fetching models from endpoints:", endpointsToFetch.map(e => e.url));
|
|
|
|
// If no valid cache, fetch models from all endpoints
|
|
const fetchPromises = endpointsToFetch.map(async (endpoint) => {
|
|
try {
|
|
const headers = {
|
|
"Content-Type": "application/json"
|
|
};
|
|
|
|
// Add authorization for custom API
|
|
if (endpoint.id === customBaseURL && useCustomAPI) {
|
|
const customAPIKey = localStorage.getItem("customAPIKey");
|
|
if (customAPIKey) {
|
|
headers["Authorization"] = `Bearer ${customAPIKey}`;
|
|
}
|
|
}
|
|
|
|
const res = await fetch(`${endpoint.url}/models`, { headers });
|
|
if (!res.ok)
|
|
throw new Error(
|
|
`HTTP error! status: ${res.status} from ${endpoint.url}`,
|
|
);
|
|
const jsonResponse = await res.json();
|
|
// Expected JSON structure: { data: [ { id, ... }, ... ], object: "list" }
|
|
const models = jsonResponse.data || [];
|
|
console.log(`✅ Fetched ${models.length} models from ${endpoint.url}:`, models.map(m => m.id));
|
|
// Map each model to include its endpoint ID, unique ID, and model name
|
|
return models.map((model) => {
|
|
// Get context window size from API or use known values for Groq models
|
|
let maxTokens = model.max_tokens || model.context_length || model.max_context_length || model.max_model_len;
|
|
|
|
// Hardcoded context windows for known Groq models (since API doesn't return them)
|
|
if (endpoint.id.includes('groq.com')) {
|
|
const groqContextWindows = {
|
|
'llama-3.1-8b-instant': 131072,
|
|
'llama-3.3-70b-versatile': 131072,
|
|
'llama-3.1-70b-versatile': 131072,
|
|
'llama3-8b-8192': 8192,
|
|
'llama3-70b-8192': 8192,
|
|
'mixtral-8x7b-32768': 32768,
|
|
'gemma2-9b-it': 8192,
|
|
'meta-llama/llama-4-scout-17b-16e-instruct': 131072,
|
|
'meta-llama/llama-4-maverick-17b-128e-instruct': 131072,
|
|
'deepseek-r1-distill-llama-70b': 131072,
|
|
'qwen/qwen3-32b': 131072,
|
|
'mistral-saba-24b': 32768,
|
|
'compound-beta': 131072,
|
|
'compound-beta-mini': 131072,
|
|
};
|
|
maxTokens = groqContextWindows[model.id] || 8192;
|
|
console.log(`Using hardcoded context window for Groq model ${model.id}: ${maxTokens} tokens`);
|
|
}
|
|
|
|
return {
|
|
...model,
|
|
modelName: model.id,
|
|
endpointId: endpoint.id,
|
|
uniqueId: `${endpoint.id}-${model.id}`,
|
|
maxTokens: maxTokens || 8192,
|
|
maxCompletionTokens: model.max_completion_tokens || model.max_output_tokens || maxTokens || 8192,
|
|
};
|
|
});
|
|
} catch (error) {
|
|
console.error(`Error fetching models from ${endpoint.url}:`, error);
|
|
return [];
|
|
}
|
|
});
|
|
const allModelsArrays = await Promise.all(fetchPromises);
|
|
const models = allModelsArrays.flat();
|
|
|
|
// Update modelRegistry with unique model instances
|
|
models.forEach((model) => {
|
|
const endpoint = endpointsToFetch.find((e) => e.id === model.endpointId);
|
|
if (endpoint) {
|
|
modelRegistry[model.uniqueId] = {
|
|
url: endpoint.url,
|
|
endpointId: model.endpointId,
|
|
maxTokens: model.maxTokens,
|
|
maxCompletionTokens: model.maxCompletionTokens,
|
|
modelName: model.modelName,
|
|
};
|
|
}
|
|
});
|
|
|
|
// Cache the results
|
|
const cacheData = {
|
|
timestamp: now,
|
|
modelRegistry: modelRegistry,
|
|
models: models,
|
|
};
|
|
localStorage.setItem(cacheKey, JSON.stringify(cacheData));
|
|
localStorage.setItem(endpointsKey, endpointsString);
|
|
|
|
return models;
|
|
}
|
|
|
|
// Create a dynamic drop-down for model selection
|
|
// This function creates a <select> element and fills it with unique model instances.
|
|
export async function createModelSelectionDropdown() {
|
|
// Only create dropdown if there's a chat interface or user input
|
|
const userInput = document.getElementById("user-input");
|
|
const chatBox = document.getElementById("chat-box");
|
|
const uncloseaiElements = document.querySelectorAll(".uncloseai");
|
|
|
|
if (!userInput && !chatBox && uncloseaiElements.length === 0) {
|
|
console.log("No chat interface found, skipping model dropdown creation");
|
|
return;
|
|
}
|
|
|
|
const models = await fetchModelsFromEndpoints();
|
|
if (!models.length) {
|
|
console.warn("No models returned from any endpoint.");
|
|
return;
|
|
}
|
|
|
|
const dropdown = document.createElement("select");
|
|
dropdown.id = "model-selection";
|
|
dropdown.className = "uncloseai-ui-button-margin";
|
|
|
|
models.forEach((model) => {
|
|
const option = document.createElement("option");
|
|
option.value = model.uniqueId; // Use unique ID for selection
|
|
option.textContent = `${model.endpointId} | ${model.modelName}`;
|
|
dropdown.appendChild(option);
|
|
});
|
|
|
|
// Insert the dropdown directly above the chat input box.
|
|
if (userInput?.parentNode) {
|
|
userInput.parentNode.parentNode.insertBefore(
|
|
dropdown,
|
|
userInput.parentNode,
|
|
);
|
|
} else {
|
|
// Only insert at top of body if there are uncloseai elements
|
|
if (uncloseaiElements.length > 0) {
|
|
document.body.insertBefore(dropdown, document.body.firstChild);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Function to add the Refresh Models button
|
|
export function addRefreshModelsButton() {
|
|
// Only create button if there's a chat interface
|
|
const userInput = document.getElementById("user-input");
|
|
const chatBox = document.getElementById("chat-box");
|
|
const uncloseaiElements = document.querySelectorAll(".uncloseai");
|
|
|
|
if (!userInput && !chatBox && uncloseaiElements.length === 0) {
|
|
console.log("No chat interface found, skipping refresh button creation");
|
|
return;
|
|
}
|
|
|
|
// Create a container for the button
|
|
const container = document.createElement("div");
|
|
container.id = "refresh-models-container";
|
|
|
|
// Position it near the model selection dropdown
|
|
if (userInput?.parentNode) {
|
|
userInput.parentNode.parentNode.insertBefore(
|
|
container,
|
|
userInput.parentNode,
|
|
);
|
|
}
|
|
|
|
// Create a button element
|
|
const refreshButton = document.createElement("button");
|
|
refreshButton.textContent = "Refresh Models";
|
|
refreshButton.className = "uncloseai-ui-button-margin";
|
|
refreshButton.onclick = async () => {
|
|
refreshButton.textContent = "Refreshing...";
|
|
refreshButton.disabled = true;
|
|
|
|
try {
|
|
// Clear the model registry cache
|
|
localStorage.removeItem("modelRegistryCache");
|
|
localStorage.removeItem("vllmEndpointsHash");
|
|
|
|
// Fetch models again
|
|
const models = await fetchModelsFromEndpoints();
|
|
|
|
// Update the dropdown
|
|
const dropdown = document.getElementById("model-selection");
|
|
if (dropdown) {
|
|
// Clear existing options
|
|
dropdown.innerHTML = "";
|
|
|
|
// Add options using models array
|
|
models.forEach((model) => {
|
|
const option = document.createElement("option");
|
|
option.value = model.uniqueId;
|
|
option.textContent = `${model.endpointId} | ${model.modelName}`;
|
|
dropdown.appendChild(option);
|
|
});
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error("Error refreshing models:", error);
|
|
} finally {
|
|
refreshButton.textContent = "Refresh Models";
|
|
refreshButton.disabled = false;
|
|
}
|
|
};
|
|
container.appendChild(refreshButton);
|
|
}
|
|
|
|
// Helper function to get the selected model from the dropdown.
|
|
export function getSelectedModel() {
|
|
// Check modal dropdown first
|
|
const modalDropdown = document.getElementById("modal-model-selection");
|
|
if (modalDropdown?.value) {
|
|
// Extract just the model name, not the unique ID
|
|
const selectedId = modalDropdown.value;
|
|
console.log("🔍 DEBUG: Modal dropdown value:", selectedId);
|
|
if (modelRegistry[selectedId]) {
|
|
const modelName = selectedId.split("-").slice(1).join("-"); // Remove endpoint prefix
|
|
console.log("🔍 DEBUG: Extracted model name:", modelName);
|
|
return modelName;
|
|
}
|
|
console.log("🔍 DEBUG: Model not in registry, returning raw value:", modalDropdown.value);
|
|
return modalDropdown.value;
|
|
}
|
|
|
|
// Fallback to main dropdown
|
|
const dropdown = document.getElementById("model-selection");
|
|
if (dropdown?.value) {
|
|
// Extract just the model name, not the unique ID
|
|
const selectedId = dropdown.value;
|
|
console.log("🔍 DEBUG: Main dropdown value:", selectedId);
|
|
if (modelRegistry[selectedId]) {
|
|
const modelName = selectedId.split("-").slice(1).join("-"); // Remove endpoint prefix
|
|
console.log("🔍 DEBUG: Extracted model name:", modelName);
|
|
return modelName;
|
|
}
|
|
console.log("🔍 DEBUG: Model not in registry, returning raw value:", dropdown.value);
|
|
return dropdown.value;
|
|
}
|
|
|
|
// Return default model if nothing selected
|
|
const defaultModel = Object.keys(modelRegistry)[0];
|
|
if (defaultModel) {
|
|
const modelName = defaultModel.split("-").slice(1).join("-"); // Remove endpoint prefix
|
|
console.log("🔍 DEBUG: Using first model from registry:", modelName);
|
|
return modelName;
|
|
}
|
|
|
|
// Fallback to hardcoded model
|
|
console.log("🔍 DEBUG: Using hardcoded fallback model");
|
|
return "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic";
|
|
}
|
|
|
|
// Get max tokens for the currently selected model
|
|
export function getSelectedModelMaxTokens() {
|
|
// Check modal dropdown first
|
|
const modalDropdown = document.getElementById("modal-model-selection");
|
|
if (modalDropdown?.value && modelRegistry[modalDropdown.value]) {
|
|
const modelData = modelRegistry[modalDropdown.value];
|
|
return modelData.maxTokens || DEFAULT_MAX_TOKENS;
|
|
}
|
|
|
|
// Fallback to main dropdown
|
|
const dropdown = document.getElementById("model-selection");
|
|
if (dropdown?.value && modelRegistry[dropdown.value]) {
|
|
const modelData = modelRegistry[dropdown.value];
|
|
return modelData.maxTokens || DEFAULT_MAX_TOKENS;
|
|
}
|
|
|
|
// Default fallback
|
|
return DEFAULT_MAX_TOKENS;
|
|
}
|
|
|
|
// Get max completion tokens for the currently selected model
|
|
export function getSelectedModelMaxCompletionTokens() {
|
|
// Check modal dropdown first
|
|
const modalDropdown = document.getElementById("modal-model-selection");
|
|
if (modalDropdown?.value && modelRegistry[modalDropdown.value]) {
|
|
const modelData = modelRegistry[modalDropdown.value];
|
|
return modelData.maxCompletionTokens || DEFAULT_MAX_TOKENS;
|
|
}
|
|
|
|
// Fallback to main dropdown
|
|
const dropdown = document.getElementById("model-selection");
|
|
if (dropdown?.value && modelRegistry[dropdown.value]) {
|
|
const modelData = modelRegistry[dropdown.value];
|
|
return modelData.maxCompletionTokens || DEFAULT_MAX_TOKENS;
|
|
}
|
|
|
|
// Default fallback
|
|
return DEFAULT_MAX_TOKENS;
|
|
}
|
|
|
|
// Helper function to get the API endpoint for the selected model.
|
|
export function getSelectedModelEndpoint() {
|
|
// Check modal dropdown first
|
|
const modalDropdown = document.getElementById("modal-model-selection");
|
|
if (modalDropdown?.value && modelRegistry[modalDropdown.value]) {
|
|
return modelRegistry[modalDropdown.value].url;
|
|
}
|
|
|
|
// Fallback to main dropdown
|
|
const dropdown = document.getElementById("model-selection");
|
|
if (dropdown?.value && modelRegistry[dropdown.value]) {
|
|
return modelRegistry[dropdown.value].url;
|
|
}
|
|
|
|
// Find the first available model in the registry
|
|
const availableModels = Object.keys(modelRegistry);
|
|
if (availableModels.length > 0) {
|
|
return modelRegistry[availableModels[0]].url;
|
|
}
|
|
|
|
// Fallback to the first endpoint if model registry is empty
|
|
if (VLLM_ENDPOINTS.length > 0) {
|
|
return VLLM_ENDPOINTS[0].url;
|
|
}
|
|
|
|
// Final fallback
|
|
return "https://hermes.ai.unturf.com/v1";
|
|
}
|