322 lines
11 KiB
JavaScript
322 lines
11 KiB
JavaScript
// Model registry and selection functionality
|
|
import { VLLM_ENDPOINTS, getAPIConfig } from "./config.js";
|
|
|
|
// This registry maps a model's ID to the endpoint where it resides.
|
|
export const modelRegistry = {};
|
|
|
|
// Fetch models from each endpoint with caching.
|
|
// Cache is busted if the endpoint array changes or TTL.
|
|
export async function fetchModelsFromEndpoints() {
|
|
const cacheKey = "modelRegistryCache";
|
|
const endpointsKey = "vllmEndpointsHash";
|
|
|
|
// Include custom API config in cache key to bust cache when it changes
|
|
const customAPIConfig = {
|
|
useCustomAPI: localStorage.getItem("useCustomAPI"),
|
|
customBaseURL: localStorage.getItem("customBaseURL"),
|
|
customAPIKey: localStorage.getItem("customAPIKey")
|
|
};
|
|
const endpointsString = JSON.stringify({
|
|
vllm: VLLM_ENDPOINTS,
|
|
custom: customAPIConfig
|
|
});
|
|
|
|
const cachedEndpoints = localStorage.getItem(endpointsKey);
|
|
const cacheItem = localStorage.getItem(cacheKey);
|
|
const now = Date.now();
|
|
const TTL = 300000; // 5 minutes in milliseconds
|
|
|
|
if (cacheItem && cachedEndpoints === endpointsString) {
|
|
try {
|
|
const cachedData = JSON.parse(cacheItem);
|
|
if (now - cachedData.timestamp < TTL) {
|
|
// Restore cached modelRegistry
|
|
Object.assign(modelRegistry, cachedData.modelRegistry);
|
|
return cachedData.models;
|
|
}
|
|
} catch (e) {
|
|
console.error("Error reading model registry from cache", e);
|
|
}
|
|
}
|
|
|
|
// Collect endpoints to fetch from
|
|
const endpointsToFetch = [...VLLM_ENDPOINTS];
|
|
|
|
// Add custom API endpoint if configured
|
|
const apiConfig = await getAPIConfig();
|
|
if (apiConfig.isCustom) {
|
|
console.log("🔧 Adding custom API endpoint to model fetch:", apiConfig.endpoint);
|
|
endpointsToFetch.push({
|
|
id: apiConfig.endpoint,
|
|
url: apiConfig.endpoint
|
|
});
|
|
} else {
|
|
console.log("🏠 No custom API configured, using default endpoints only");
|
|
}
|
|
|
|
console.log("📡 Fetching models from endpoints:", endpointsToFetch.map(e => e.url));
|
|
|
|
// If no valid cache, fetch models from all endpoints
|
|
const fetchPromises = endpointsToFetch.map(async (endpoint) => {
|
|
try {
|
|
const headers = {
|
|
"Content-Type": "application/json"
|
|
};
|
|
|
|
// Add authorization for custom API
|
|
if (endpoint.id === apiConfig.endpoint && apiConfig.isCustom) {
|
|
headers["Authorization"] = `Bearer ${apiConfig.apiKey}`;
|
|
}
|
|
|
|
const res = await fetch(`${endpoint.url}/models`, { headers });
|
|
if (!res.ok)
|
|
throw new Error(
|
|
`HTTP error! status: ${res.status} from ${endpoint.url}`,
|
|
);
|
|
const jsonResponse = await res.json();
|
|
// Expected JSON structure: { data: [ { id, ... }, ... ], object: "list" }
|
|
const models = jsonResponse.data || [];
|
|
console.log(`✅ Fetched ${models.length} models from ${endpoint.url}:`, models.map(m => m.id));
|
|
// Map each model to include its endpoint ID, unique ID, and model name
|
|
return models.map((model) => ({
|
|
...model,
|
|
modelName: model.id, // Explicitly store model name
|
|
endpointId: endpoint.id,
|
|
uniqueId: `${endpoint.id}-${model.id}`, // Unique ID with endpoint ID first
|
|
maxTokens: model.max_tokens || model.context_length || model.max_context_length || model.max_model_len || 8192, // Capture max tokens
|
|
}));
|
|
} catch (error) {
|
|
console.error(`Error fetching models from ${endpoint.url}:`, error);
|
|
return [];
|
|
}
|
|
});
|
|
const allModelsArrays = await Promise.all(fetchPromises);
|
|
const models = allModelsArrays.flat();
|
|
|
|
// Update modelRegistry with unique model instances
|
|
models.forEach((model) => {
|
|
const endpoint = endpointsToFetch.find((e) => e.id === model.endpointId);
|
|
if (endpoint) {
|
|
modelRegistry[model.uniqueId] = {
|
|
url: endpoint.url,
|
|
endpointId: model.endpointId,
|
|
maxTokens: model.maxTokens,
|
|
modelName: model.modelName,
|
|
};
|
|
}
|
|
});
|
|
|
|
// Cache the results
|
|
const cacheData = {
|
|
timestamp: now,
|
|
modelRegistry: modelRegistry,
|
|
models: models,
|
|
};
|
|
localStorage.setItem(cacheKey, JSON.stringify(cacheData));
|
|
localStorage.setItem(endpointsKey, endpointsString);
|
|
|
|
return models;
|
|
}
|
|
|
|
// Create a dynamic drop-down for model selection
|
|
// This function creates a <select> element and fills it with unique model instances.
|
|
export async function createModelSelectionDropdown() {
|
|
// Only create dropdown if there's a chat interface or user input
|
|
const userInput = document.getElementById("user-input");
|
|
const chatBox = document.getElementById("chat-box");
|
|
const uncloseaiElements = document.querySelectorAll(".uncloseai");
|
|
|
|
if (!userInput && !chatBox && uncloseaiElements.length === 0) {
|
|
console.log("No chat interface found, skipping model dropdown creation");
|
|
return;
|
|
}
|
|
|
|
const models = await fetchModelsFromEndpoints();
|
|
if (!models.length) {
|
|
console.warn("No models returned from any endpoint.");
|
|
return;
|
|
}
|
|
|
|
const dropdown = document.createElement("select");
|
|
dropdown.id = "model-selection";
|
|
dropdown.className = "uncloseai-ui-button-margin";
|
|
|
|
models.forEach((model) => {
|
|
const option = document.createElement("option");
|
|
option.value = model.uniqueId; // Use unique ID for selection
|
|
option.textContent = `${model.endpointId} | ${model.modelName}`;
|
|
dropdown.appendChild(option);
|
|
});
|
|
|
|
// Insert the dropdown directly above the chat input box.
|
|
if (userInput?.parentNode) {
|
|
userInput.parentNode.parentNode.insertBefore(
|
|
dropdown,
|
|
userInput.parentNode,
|
|
);
|
|
} else {
|
|
// Only insert at top of body if there are uncloseai elements
|
|
if (uncloseaiElements.length > 0) {
|
|
document.body.insertBefore(dropdown, document.body.firstChild);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Function to add the Refresh Models button
|
|
export function addRefreshModelsButton() {
|
|
// Only create button if there's a chat interface
|
|
const userInput = document.getElementById("user-input");
|
|
const chatBox = document.getElementById("chat-box");
|
|
const uncloseaiElements = document.querySelectorAll(".uncloseai");
|
|
|
|
if (!userInput && !chatBox && uncloseaiElements.length === 0) {
|
|
console.log("No chat interface found, skipping refresh button creation");
|
|
return;
|
|
}
|
|
|
|
// Create a container for the button
|
|
const container = document.createElement("div");
|
|
container.id = "refresh-models-container";
|
|
|
|
// Position it near the model selection dropdown
|
|
if (userInput?.parentNode) {
|
|
userInput.parentNode.parentNode.insertBefore(
|
|
container,
|
|
userInput.parentNode,
|
|
);
|
|
}
|
|
|
|
// Create a button element
|
|
const refreshButton = document.createElement("button");
|
|
refreshButton.textContent = "Refresh Models";
|
|
refreshButton.className = "uncloseai-ui-button-margin";
|
|
refreshButton.onclick = async () => {
|
|
refreshButton.textContent = "Refreshing...";
|
|
refreshButton.disabled = true;
|
|
|
|
try {
|
|
// Clear the model registry cache
|
|
localStorage.removeItem("modelRegistryCache");
|
|
localStorage.removeItem("vllmEndpointsHash");
|
|
|
|
// Fetch models again
|
|
const models = await fetchModelsFromEndpoints();
|
|
|
|
// Update the dropdown
|
|
const dropdown = document.getElementById("model-selection");
|
|
if (dropdown) {
|
|
// Clear existing options
|
|
dropdown.innerHTML = "";
|
|
|
|
// Add options using models array
|
|
models.forEach((model) => {
|
|
const option = document.createElement("option");
|
|
option.value = model.uniqueId;
|
|
option.textContent = `${model.endpointId} | ${model.modelName}`;
|
|
dropdown.appendChild(option);
|
|
});
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error("Error refreshing models:", error);
|
|
} finally {
|
|
refreshButton.textContent = "Refresh Models";
|
|
refreshButton.disabled = false;
|
|
}
|
|
};
|
|
container.appendChild(refreshButton);
|
|
}
|
|
|
|
// Helper function to get the selected model from the dropdown.
|
|
export function getSelectedModel() {
|
|
// Check modal dropdown first
|
|
const modalDropdown = document.getElementById("hermes-model-selection");
|
|
if (modalDropdown?.value) {
|
|
// Extract just the model name, not the unique ID
|
|
const selectedId = modalDropdown.value;
|
|
console.log("🔍 DEBUG: Modal dropdown value:", selectedId);
|
|
if (modelRegistry[selectedId]) {
|
|
const modelName = selectedId.split("-").slice(1).join("-"); // Remove endpoint prefix
|
|
console.log("🔍 DEBUG: Extracted model name:", modelName);
|
|
return modelName;
|
|
}
|
|
console.log("🔍 DEBUG: Model not in registry, returning raw value:", modalDropdown.value);
|
|
return modalDropdown.value;
|
|
}
|
|
|
|
// Fallback to main dropdown
|
|
const dropdown = document.getElementById("model-selection");
|
|
if (dropdown?.value) {
|
|
// Extract just the model name, not the unique ID
|
|
const selectedId = dropdown.value;
|
|
console.log("🔍 DEBUG: Main dropdown value:", selectedId);
|
|
if (modelRegistry[selectedId]) {
|
|
const modelName = selectedId.split("-").slice(1).join("-"); // Remove endpoint prefix
|
|
console.log("🔍 DEBUG: Extracted model name:", modelName);
|
|
return modelName;
|
|
}
|
|
console.log("🔍 DEBUG: Model not in registry, returning raw value:", dropdown.value);
|
|
return dropdown.value;
|
|
}
|
|
|
|
// Return default model if nothing selected
|
|
const defaultModel = Object.keys(modelRegistry)[0];
|
|
if (defaultModel) {
|
|
const modelName = defaultModel.split("-").slice(1).join("-"); // Remove endpoint prefix
|
|
console.log("🔍 DEBUG: Using first model from registry:", modelName);
|
|
return modelName;
|
|
}
|
|
|
|
// Fallback to hardcoded model
|
|
console.log("🔍 DEBUG: Using hardcoded fallback model");
|
|
return "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic";
|
|
}
|
|
|
|
// Get max tokens for the currently selected model
|
|
export function getSelectedModelMaxTokens() {
|
|
// Check modal dropdown first
|
|
const modalDropdown = document.getElementById("hermes-model-selection");
|
|
if (modalDropdown?.value && modelRegistry[modalDropdown.value]) {
|
|
const modelData = modelRegistry[modalDropdown.value];
|
|
return modelData.maxTokens || 8192;
|
|
}
|
|
|
|
// Fallback to main dropdown
|
|
const dropdown = document.getElementById("model-selection");
|
|
if (dropdown?.value && modelRegistry[dropdown.value]) {
|
|
const modelData = modelRegistry[dropdown.value];
|
|
return modelData.maxTokens || 8192;
|
|
}
|
|
|
|
// Default fallback
|
|
return 8192;
|
|
}
|
|
|
|
// Helper function to get the API endpoint for the selected model.
|
|
export function getSelectedModelEndpoint() {
|
|
// Check modal dropdown first
|
|
const modalDropdown = document.getElementById("modal-model-selection");
|
|
if (modalDropdown?.value && modelRegistry[modalDropdown.value]) {
|
|
return modelRegistry[modalDropdown.value].url;
|
|
}
|
|
|
|
// Fallback to main dropdown
|
|
const dropdown = document.getElementById("model-selection");
|
|
if (dropdown?.value && modelRegistry[dropdown.value]) {
|
|
return modelRegistry[dropdown.value].url;
|
|
}
|
|
|
|
// Find the first available model in the registry
|
|
const availableModels = Object.keys(modelRegistry);
|
|
if (availableModels.length > 0) {
|
|
return modelRegistry[availableModels[0]].url;
|
|
}
|
|
|
|
// Fallback to the first endpoint if model registry is empty
|
|
if (VLLM_ENDPOINTS.length > 0) {
|
|
return VLLM_ENDPOINTS[0].url;
|
|
}
|
|
|
|
// Final fallback
|
|
return "https://hermes.ai.unturf.com/v1";
|
|
}
|