feat: add automatic health check updates on Railway tool failures

Add real-time health status updates when tools fail to load or execute:

- Add @tpmjs/db dependency to playground package
- Create reportToolFailure() function to update health status on errors
- Trigger health check updates for:
  1. Import failures (Railway load-and-describe errors)
  2. Execution failures (Railway execute-tool errors)
- Updates are non-blocking and run in background
- Each tool failure now logs:
  - 🏥 Triggering health check for {package}/{export}
  -  Health status updated for {package}/{export}

This complements the proactive health checking (daily cron + manual recheck)
with reactive health updates from actual tool usage errors.

Handles multiple errors in batch loading - each error triggers its own
health check update independently and asynchronously.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Ajax Davis 2025-12-04 16:51:27 +10:00
parent bf8d27a59c
commit 84c6a579b2
2 changed files with 77 additions and 0 deletions

View file

@ -13,6 +13,7 @@
"dependencies": {
"@ai-sdk/openai": "3.0.0-beta.74",
"@ai-sdk/react": "3.0.0-beta.131",
"@tpmjs/db": "workspace:*",
"@tpmjs/env": "workspace:*",
"@tpmjs/hello": "workspace:*",
"@tpmjs/search-registry": "workspace:*",

View file

@ -1,6 +1,8 @@
import { prisma } from '@tpmjs/db';
import { jsonSchema, tool } from 'ai';
// Cache for tool wrappers (process-level)
// biome-ignore lint/suspicious/noExplicitAny: Tool types from AI SDK are complex
const moduleCache = new Map<string, any>();
// Cache for per-conversation active tools
@ -36,6 +38,51 @@ function getConversationEnv(conversationId: string): Record<string, string> {
return conversationEnv.get(conversationId) || {};
}
/**
* Report tool failure and trigger async health check
* Non-blocking - logs error and triggers health check in background
*/
async function reportToolFailure(
packageName: string,
exportName: string,
error: string,
phase: 'import' | 'execution'
): Promise<void> {
try {
// Find the tool in the database
const tool = await prisma.tool.findFirst({
where: {
exportName,
package: {
npmPackageName: packageName,
},
},
select: { id: true },
});
if (!tool) {
console.warn(`⚠️ Tool not found in database for health check: ${packageName}/${exportName}`);
return;
}
console.log(`🏥 Triggering health check for ${packageName}/${exportName} (${tool.id})`);
// Update health status immediately (optimistic)
await prisma.tool.update({
where: { id: tool.id },
data: {
[phase === 'import' ? 'importHealth' : 'executionHealth']: 'BROKEN',
healthCheckError: error,
lastHealthCheck: new Date(),
},
});
console.log(`✅ Health status updated for ${packageName}/${exportName}`);
} catch (err) {
console.error('❌ Failed to report tool failure:', err);
}
}
/**
* Dynamically load a tool via Railway service
* Railway service runs with --experimental-network-imports and can import from esm.sh
@ -77,6 +124,17 @@ export async function loadToolDynamically(
if (!response.ok) {
const errorText = await response.text();
console.error(`❌ Railway service error (${response.status}): ${errorText}`);
// Trigger health check update in background (non-blocking)
reportToolFailure(
packageName,
exportName,
`Railway service error (${response.status}): ${errorText}`,
'import'
).catch((err) =>
console.error(`Failed to report tool failure for ${packageName}/${exportName}:`, err)
);
return null;
}
@ -84,6 +142,13 @@ export async function loadToolDynamically(
if (!data.success) {
console.error(`❌ Failed to load tool: ${data.error}`);
// Trigger health check update in background (non-blocking)
reportToolFailure(packageName, exportName, data.error || 'Unknown error', 'import').catch(
(err) =>
console.error(`Failed to report tool failure for ${packageName}/${exportName}:`, err)
);
return null;
}
@ -125,6 +190,17 @@ export async function loadToolDynamically(
if (!result.success) {
console.error(`❌ Tool execution failed: ${result.error}`);
// Trigger health check update in background (non-blocking)
reportToolFailure(
packageName,
exportName,
result.error || 'Tool execution failed',
'execution'
).catch((err) =>
console.error(`Failed to report tool failure for ${packageName}/${exportName}:`, err)
);
throw new Error(result.error || 'Tool execution failed');
}