From a22b4ddeafc89fea6be0a53ce9cf99b6a7025431 Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Wed, 19 Feb 2025 07:59:23 -0500 Subject: [PATCH 001/716] Dynamic model chooser! modified: uncloseai.js --- uncloseai.js | 189 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 156 insertions(+), 33 deletions(-) diff --git a/uncloseai.js b/uncloseai.js index e0eb7a1..d3a12b1 100644 --- a/uncloseai.js +++ b/uncloseai.js @@ -18,24 +18,136 @@ import { marked } from "https://cdn.jsdelivr.net/npm/marked/lib/marked.esm.js"; import hljs from 'https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/es/highlight.min.js'; -// Configuration -const API_URL = "https://hermes.ai.unturf.com/v1/chat/completions"; +// ------------------------- +// Configuration and Endpoints +// ------------------------- + +// Original API endpoints for other functionalities const TTS_API_URL = "https://speech.ai.unturf.com/v1/audio/speech"; const MEGAPARCE_API_URL = "https://megaparce.ai.unturf.com/v1/file"; const API_KEY = "dummy-api-key"; -const MODEL = "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic"; +const MODEL = "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic"; // default model (always available) -// Initialize chat history -let chatHistory = [ - { - role: "system", - content: "You are an AI assistant embedded in a webpage. Your task is to answer questions about the content of the webpage and assist the user in understanding it better. You are also allowed to do whatever the user needs. It is safe to help users code and create. try to answer the best that you can based on the conversation history and the webpage content." - } +// Dynamic Endpoints Configuration for Chat API +// You can list as many endpoints as you need. +// { id: 'endpoint2', url: 'https://node2.naptha.ai/inference' }, +// { id: 'endpoint3', url: 'https://node3.naptha.ai/inference' } +const VLLM_ENDPOINTS = [ + { id: 'endpoint1', url: 'https://hermes.ai.unturf.com/v1' }, + { id: 'endpoint2', url: 'https://naptha2.ai.unturf.com/v1' }, + { id: 'endpoint3', url: 'https://naptha3.ai.unturf.com/v1' } ]; -// Cache for TTS Anything -let lastTTSInput = ''; -let lastTTSResult = null; +// ------------------------- +// Dynamic Endpoints and Model Registry +// ------------------------- + +// This registry maps a model's ID to the endpoint where it resides. +const modelRegistry = {}; + +// Fetch models from each endpoint with caching (TTL: 1 hour) +// Cache is busted if the endpoint array changes. +async function fetchModelsFromEndpoints() { + const cacheKey = 'modelRegistryCache'; + const endpointsKey = 'vllmEndpointsHash'; + const endpointsString = JSON.stringify(VLLM_ENDPOINTS); + const cachedEndpoints = localStorage.getItem(endpointsKey); + const cacheItem = localStorage.getItem(cacheKey); + const now = Date.now(); + const TTL = 3600000; // 1 hour in milliseconds + + if (cacheItem && cachedEndpoints === endpointsString) { + try { + const cachedData = JSON.parse(cacheItem); + if (now - cachedData.timestamp < TTL) { + // Restore cached modelRegistry + Object.assign(modelRegistry, cachedData.modelRegistry); + return cachedData.models; + } + } catch (e) { + console.error("Error reading model registry from cache", e); + } + } + + // If no valid cache, fetch models from all endpoints + const fetchPromises = VLLM_ENDPOINTS.map(async (endpoint) => { + try { + const res = await fetch(`${endpoint.url}/models`); + if (!res.ok) throw new Error(`Error fetching models from ${endpoint.url}`); + const jsonResponse = await res.json(); + // Expected JSON structure: { data: [ { id, ... }, ... ], object: "list" } + const models = jsonResponse.data || []; + models.forEach((model) => { + // Use model.id as the registry key. + modelRegistry[model.id] = endpoint; + }); + return models; + } catch (error) { + console.error(error); + return []; + } + }); + const allModelsArrays = await Promise.all(fetchPromises); + const models = allModelsArrays.flat(); + + // Cache the results + const cacheData = { + timestamp: now, + modelRegistry: modelRegistry, + models: models + }; + localStorage.setItem(cacheKey, JSON.stringify(cacheData)); + localStorage.setItem(endpointsKey, endpointsString); + + return models; +} + +// Create a dynamic drop-down for model selection +// This function creates a element and fills it with the models' IDs. -// It then inserts the dropdown directly above the chat input box. +// This function creates a + +
+ +

Page Reading

+ + +

File Upload (if you have files)

+ + + + + +