Add vision model support with auto alt-text on image hover

Backend:
- Track VISION_MODELS list at startup from available endpoints
- Add is_vision_model() to detect vision-capable models (*-vl*, *vision*, gpt-4o)
- Add extract_base64_from_img_tag() and build_message_content() helpers
- Modify chat_gpt() to include base64 images for vision models
- Add GET /vision endpoint for vision availability status
- Add POST /vision/describe endpoint for image alt-text generation

Frontend:
- Check vision availability on page load via /vision
- Add hover event delegation on chat images
- On hover: call vision model, cache result, set img.title and img.alt
- Shows cursor:wait while loading description
This commit is contained in:
Russell Ballestrini 2025-12-07 08:36:47 -05:00
parent 030b1cc447
commit bd2dbf7ed5
2 changed files with 206 additions and 10 deletions

View file

@ -133,6 +133,87 @@ let autoPlayTTS = localStorage.getItem('autoPlayTTS') === 'true' || false;
let ttsQueue = [];
let isPlayingTTS = false;
// Vision model state for auto alt-text
let visionAvailable = false;
let visionModel = null;
const imageDescriptionCache = new Map(); // Cache descriptions by image src hash
// Check vision availability on load
async function initVisionCapability() {
try {
const response = await fetch('/vision');
const data = await response.json();
visionAvailable = data.available;
visionModel = data.default;
if (visionAvailable) {
console.log(`Vision available: ${visionModel}`);
setupImageHoverDescriptions();
}
} catch (e) {
console.warn('Vision check failed:', e);
}
}
// Generate a simple hash for caching
function hashString(str) {
let hash = 0;
for (let i = 0; i < Math.min(str.length, 1000); i++) {
hash = ((hash << 5) - hash) + str.charCodeAt(i);
hash |= 0;
}
return hash.toString();
}
// Fetch description for an image
async function getImageDescription(imgSrc) {
const cacheKey = hashString(imgSrc);
if (imageDescriptionCache.has(cacheKey)) {
return imageDescriptionCache.get(cacheKey);
}
try {
const response = await fetch('/vision/describe', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ image: imgSrc })
});
const data = await response.json();
if (data.description) {
imageDescriptionCache.set(cacheKey, data.description);
return data.description;
}
} catch (e) {
console.warn('Failed to get image description:', e);
}
return null;
}
// Setup hover handlers for images in chat
function setupImageHoverDescriptions() {
const messagesContainer = document.getElementById('messages');
if (!messagesContainer) return;
// Use event delegation for efficiency
messagesContainer.addEventListener('mouseenter', async (e) => {
if (!visionAvailable) return;
const img = e.target.closest('img[src^="data:image"]');
if (!img || img.dataset.visionProcessing || img.title) return;
// Mark as processing to avoid duplicate requests
img.dataset.visionProcessing = 'true';
img.style.cursor = 'wait';
const description = await getImageDescription(img.src);
if (description) {
img.title = description;
img.alt = description;
}
img.style.cursor = '';
delete img.dataset.visionProcessing;
}, true);
}
// Function to sanitize the username
function sanitizeUsername(username) {
// Split the username on commas and take the first part.
@ -187,6 +268,9 @@ document.addEventListener('DOMContentLoaded', (event) => {
// Initialize auto-play TTS button state from localStorage
updateAutoPlayTTSDisplay();
// Check for vision model availability (enables image hover descriptions)
initVisionCapability();
// Function to populate the model dropdown
function populateModelDropdown(models) {
// Clear options starting from index 1 (preserve "None" at index 0)