diff --git a/src/config.js b/src/config.js index b59fead..1ba4f8e 100644 --- a/src/config.js +++ b/src/config.js @@ -68,7 +68,14 @@ export async function getSystemMessage() { export const VLLM_ENDPOINTS = [ { id: "hermes.ai.unturf.com", url: "https://hermes.ai.unturf.com/v1" }, { id: "hermes2.ai.unturf.com", url: "https://hermes2.ai.unturf.com/v1" }, - { id: "qwen.ai.unturf.com", url: "https://qwen.ai.unturf.com/v1" }, + { + id: "qwen.ai.unturf.com", + url: "https://qwen.ai.unturf.com/v1", + // Ollama endpoints don't return max_model_len, so we specify it here + modelContextWindows: { + 'hf.co/unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF:Q4_K_M': 32768, + } + }, ]; // Function to get API configuration (custom or default) diff --git a/src/models.js b/src/models.js index 30f6dd9..94635cc 100644 --- a/src/models.js +++ b/src/models.js @@ -87,7 +87,13 @@ export async function fetchModelsFromEndpoints() { return models.map((model) => { // Get context window size from API or use known values for Groq models let maxTokens = model.max_tokens || model.context_length || model.max_context_length || model.max_model_len; - + + // Check if endpoint has modelContextWindows config (for Ollama/other endpoints without API max_model_len) + if (endpoint.modelContextWindows && endpoint.modelContextWindows[model.id]) { + maxTokens = endpoint.modelContextWindows[model.id]; + console.log(`Using configured context window for model ${model.id}: ${maxTokens} tokens`); + } + // Hardcoded context windows for known Groq models (since API doesn't return them) if (endpoint.id.includes('groq.com')) { const groqContextWindows = {