feat: add automatic health check updates on Railway tool failures
Add real-time health status updates when tools fail to load or execute: - Add @tpmjs/db dependency to playground package - Create reportToolFailure() function to update health status on errors - Trigger health check updates for: 1. Import failures (Railway load-and-describe errors) 2. Execution failures (Railway execute-tool errors) - Updates are non-blocking and run in background - Each tool failure now logs: - 🏥 Triggering health check for {package}/{export} - ✅ Health status updated for {package}/{export} This complements the proactive health checking (daily cron + manual recheck) with reactive health updates from actual tool usage errors. Handles multiple errors in batch loading - each error triggers its own health check update independently and asynchronously. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
bf8d27a59c
commit
84c6a579b2
2 changed files with 77 additions and 0 deletions
|
|
@ -13,6 +13,7 @@
|
|||
"dependencies": {
|
||||
"@ai-sdk/openai": "3.0.0-beta.74",
|
||||
"@ai-sdk/react": "3.0.0-beta.131",
|
||||
"@tpmjs/db": "workspace:*",
|
||||
"@tpmjs/env": "workspace:*",
|
||||
"@tpmjs/hello": "workspace:*",
|
||||
"@tpmjs/search-registry": "workspace:*",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import { prisma } from '@tpmjs/db';
|
||||
import { jsonSchema, tool } from 'ai';
|
||||
|
||||
// Cache for tool wrappers (process-level)
|
||||
// biome-ignore lint/suspicious/noExplicitAny: Tool types from AI SDK are complex
|
||||
const moduleCache = new Map<string, any>();
|
||||
|
||||
// Cache for per-conversation active tools
|
||||
|
|
@ -36,6 +38,51 @@ function getConversationEnv(conversationId: string): Record<string, string> {
|
|||
return conversationEnv.get(conversationId) || {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Report tool failure and trigger async health check
|
||||
* Non-blocking - logs error and triggers health check in background
|
||||
*/
|
||||
async function reportToolFailure(
|
||||
packageName: string,
|
||||
exportName: string,
|
||||
error: string,
|
||||
phase: 'import' | 'execution'
|
||||
): Promise<void> {
|
||||
try {
|
||||
// Find the tool in the database
|
||||
const tool = await prisma.tool.findFirst({
|
||||
where: {
|
||||
exportName,
|
||||
package: {
|
||||
npmPackageName: packageName,
|
||||
},
|
||||
},
|
||||
select: { id: true },
|
||||
});
|
||||
|
||||
if (!tool) {
|
||||
console.warn(`⚠️ Tool not found in database for health check: ${packageName}/${exportName}`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`🏥 Triggering health check for ${packageName}/${exportName} (${tool.id})`);
|
||||
|
||||
// Update health status immediately (optimistic)
|
||||
await prisma.tool.update({
|
||||
where: { id: tool.id },
|
||||
data: {
|
||||
[phase === 'import' ? 'importHealth' : 'executionHealth']: 'BROKEN',
|
||||
healthCheckError: error,
|
||||
lastHealthCheck: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
console.log(`✅ Health status updated for ${packageName}/${exportName}`);
|
||||
} catch (err) {
|
||||
console.error('❌ Failed to report tool failure:', err);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Dynamically load a tool via Railway service
|
||||
* Railway service runs with --experimental-network-imports and can import from esm.sh
|
||||
|
|
@ -77,6 +124,17 @@ export async function loadToolDynamically(
|
|||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
console.error(`❌ Railway service error (${response.status}): ${errorText}`);
|
||||
|
||||
// Trigger health check update in background (non-blocking)
|
||||
reportToolFailure(
|
||||
packageName,
|
||||
exportName,
|
||||
`Railway service error (${response.status}): ${errorText}`,
|
||||
'import'
|
||||
).catch((err) =>
|
||||
console.error(`Failed to report tool failure for ${packageName}/${exportName}:`, err)
|
||||
);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
@ -84,6 +142,13 @@ export async function loadToolDynamically(
|
|||
|
||||
if (!data.success) {
|
||||
console.error(`❌ Failed to load tool: ${data.error}`);
|
||||
|
||||
// Trigger health check update in background (non-blocking)
|
||||
reportToolFailure(packageName, exportName, data.error || 'Unknown error', 'import').catch(
|
||||
(err) =>
|
||||
console.error(`Failed to report tool failure for ${packageName}/${exportName}:`, err)
|
||||
);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
@ -125,6 +190,17 @@ export async function loadToolDynamically(
|
|||
|
||||
if (!result.success) {
|
||||
console.error(`❌ Tool execution failed: ${result.error}`);
|
||||
|
||||
// Trigger health check update in background (non-blocking)
|
||||
reportToolFailure(
|
||||
packageName,
|
||||
exportName,
|
||||
result.error || 'Tool execution failed',
|
||||
'execution'
|
||||
).catch((err) =>
|
||||
console.error(`Failed to report tool failure for ${packageName}/${exportName}:`, err)
|
||||
);
|
||||
|
||||
throw new Error(result.error || 'Tool execution failed');
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue