From e2af4cfd6a657e4671c4377e0da8788a472eb18c Mon Sep 17 00:00:00 2001 From: Ajax Davis Date: Thu, 4 Dec 2025 16:03:21 +1000 Subject: [PATCH] feat: implement health check system (Phase 1 & 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add comprehensive health monitoring for TPMJS tools that tracks both import and execution health via Railway executor service. ## Database Schema - Add HealthStatus enum (UNKNOWN, HEALTHY, BROKEN) - Add HealthCheckType enum (IMPORT, EXECUTION, FULL) - Add health fields to Tool model: - importHealth: tracks if tool can be loaded - executionHealth: tracks if tool can execute - lastHealthCheck: timestamp of last check - healthCheckError: stores error message - Add HealthCheck audit table for full history ## Core Service Create health-check-service.ts with 5 functions: 1. checkImportHealth() - Tests tool loading via /load-and-describe 2. checkExecutionHealth() - Tests execution via /execute-tool 3. generateTestParameters() - Creates minimal test params by type 4. performHealthCheck() - Full check with database updates 5. performBatchHealthCheck() - Processes tools in batches Features: - 30-second timeout per check - Skips execution if import fails - Batch processing (5 concurrent, 1s delays) - Full audit trail in HealthCheck table ## API Endpoints /api/sync/health-check (POST): - Daily cron job at 2am UTC - Checks all tools in database - Requires CRON_SECRET auth - Logs results to SyncLog table - Max duration: 5 minutes /api/tools/broken (GET): - Lists all tools with broken health status - Filters by importHealth='BROKEN' OR executionHealth='BROKEN' - Includes package metadata - Orders by lastHealthCheck DESC ## Configuration - Add RAILWAY_EXECUTOR_URL to env.ts - Add daily cron job to vercel.json - Use db:push for schema changes (existing production data) Next: Manual trigger endpoint + health filtering + UI components 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../src/app/api/sync/health-check/route.ts | 89 +++++ apps/web/src/app/api/tools/broken/route.ts | 48 +++ apps/web/src/env.ts | 4 + .../lib/health-check/health-check-service.ts | 314 ++++++++++++++++++ packages/db/prisma/schema.prisma | 62 +++- vercel.json | 4 + 6 files changed, 520 insertions(+), 1 deletion(-) create mode 100644 apps/web/src/app/api/sync/health-check/route.ts create mode 100644 apps/web/src/app/api/tools/broken/route.ts create mode 100644 apps/web/src/lib/health-check/health-check-service.ts diff --git a/apps/web/src/app/api/sync/health-check/route.ts b/apps/web/src/app/api/sync/health-check/route.ts new file mode 100644 index 0000000..7bc86f6 --- /dev/null +++ b/apps/web/src/app/api/sync/health-check/route.ts @@ -0,0 +1,89 @@ +import { prisma } from '@tpmjs/db'; +import { type NextRequest, NextResponse } from 'next/server'; +import { env } from '~/env'; +import { performBatchHealthCheck } from '~/lib/health-check/health-check-service'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; +export const maxDuration = 300; // 5 minutes + +/** + * POST /api/sync/health-check + * Daily health check for all tools + * + * This endpoint is called by Vercel Cron (daily at 2am UTC) + * Requires Authorization: Bearer + */ +export async function POST(request: NextRequest) { + // Verify cron secret + const authHeader = request.headers.get('authorization'); + const token = authHeader?.replace('Bearer ', ''); + + if (env.CRON_SECRET && token !== env.CRON_SECRET) { + return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }); + } + + const startTime = Date.now(); + + try { + console.log('🏥 Daily health check cron job starting...'); + + // Get all tools + const tools = await prisma.tool.findMany({ + select: { id: true }, + }); + + console.log(`📊 Found ${tools.length} tools to check`); + + const toolIds = tools.map((t) => t.id); + + // Perform batch health checks + const result = await performBatchHealthCheck(toolIds, 'daily-cron', 5); + + const durationMs = Date.now() - startTime; + + // Log sync operation + await prisma.syncLog.create({ + data: { + source: 'health-check', + status: result.errors > 0 ? 'partial' : 'success', + processed: result.healthy + result.broken + result.unknown, + skipped: 0, + errors: result.errors, + message: `Checked ${result.total} tools: ${result.healthy} healthy, ${result.broken} broken, ${result.unknown} unknown`, + metadata: { + durationMs, + ...result, + }, + }, + }); + + console.log(`✅ Daily health check complete in ${durationMs}ms`); + + return NextResponse.json({ + success: true, + data: { + ...result, + durationMs, + }, + }); + } catch (error) { + console.error('❌ Health check cron failed:', error); + + const durationMs = Date.now() - startTime; + + await prisma.syncLog.create({ + data: { + source: 'health-check', + status: 'error', + processed: 0, + skipped: 0, + errors: 1, + message: error instanceof Error ? error.message : 'Unknown error', + metadata: { durationMs }, + }, + }); + + return NextResponse.json({ success: false, error: 'Health check failed' }, { status: 500 }); + } +} diff --git a/apps/web/src/app/api/tools/broken/route.ts b/apps/web/src/app/api/tools/broken/route.ts new file mode 100644 index 0000000..1a537d2 --- /dev/null +++ b/apps/web/src/app/api/tools/broken/route.ts @@ -0,0 +1,48 @@ +import { prisma } from '@tpmjs/db'; +import { NextResponse } from 'next/server'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; +export const maxDuration = 60; + +/** + * GET /api/tools/broken + * List all tools with broken health status + * + * Returns tools where importHealth='BROKEN' OR executionHealth='BROKEN' + * Includes package relation with npmPackageName and npmVersion + */ +export async function GET() { + try { + const brokenTools = await prisma.tool.findMany({ + where: { + OR: [{ importHealth: 'BROKEN' }, { executionHealth: 'BROKEN' }], + }, + include: { + package: { + select: { + npmPackageName: true, + npmVersion: true, + category: true, + isOfficial: true, + }, + }, + }, + orderBy: { + lastHealthCheck: 'desc', + }, + }); + + return NextResponse.json({ + success: true, + data: brokenTools, + count: brokenTools.length, + }); + } catch (error) { + console.error('Failed to fetch broken tools:', error); + return NextResponse.json( + { success: false, error: 'Failed to fetch broken tools' }, + { status: 500 } + ); + } +} diff --git a/apps/web/src/env.ts b/apps/web/src/env.ts index 13dda6c..ab751c8 100644 --- a/apps/web/src/env.ts +++ b/apps/web/src/env.ts @@ -5,4 +5,8 @@ export const env = createEnv({ NODE_ENV: z.enum(['development', 'production', 'test']).default('development'), NEXT_PUBLIC_API_URL: z.string().url().optional(), CRON_SECRET: z.string().min(32).optional(), // Required for Vercel Cron security + RAILWAY_EXECUTOR_URL: z + .string() + .url() + .default('https://endearing-commitment-production.up.railway.app'), // Railway service for health checks }); diff --git a/apps/web/src/lib/health-check/health-check-service.ts b/apps/web/src/lib/health-check/health-check-service.ts new file mode 100644 index 0000000..776ecad --- /dev/null +++ b/apps/web/src/lib/health-check/health-check-service.ts @@ -0,0 +1,314 @@ +/** + * Health Check Service + * Checks tool import and execution health via Railway executor + */ + +import { type HealthStatus, type Package, type Prisma, type Tool, prisma } from '@tpmjs/db'; +import { env } from '~/env'; + +const RAILWAY_EXECUTOR_URL = env.RAILWAY_EXECUTOR_URL; + +interface HealthCheckResult { + toolId: string; + importStatus: HealthStatus; + importError: string | null; + importTimeMs: number | null; + executionStatus: HealthStatus; + executionError: string | null; + executionTimeMs: number | null; + overallStatus: HealthStatus; +} + +/** + * Check if a tool can be imported (load-and-describe) + */ +async function checkImportHealth(tool: Tool & { package: Package }): Promise<{ + status: HealthStatus; + error: string | null; + timeMs: number; +}> { + const startTime = Date.now(); + + try { + const response = await fetch(`${RAILWAY_EXECUTOR_URL}/load-and-describe`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + packageName: tool.package.npmPackageName, + exportName: tool.exportName, + version: tool.package.npmVersion, + env: tool.package.env || {}, + }), + signal: AbortSignal.timeout(30000), // 30 second timeout + }); + + const timeMs = Date.now() - startTime; + const data = await response.json(); + + if (!response.ok || !data.success) { + return { + status: 'BROKEN', + error: data.error || `HTTP ${response.status}`, + timeMs, + }; + } + + // Verify tool has required fields + if (!data.tool?.description || !data.tool?.inputSchema) { + return { + status: 'BROKEN', + error: 'Missing required tool fields (description or inputSchema)', + timeMs, + }; + } + + return { status: 'HEALTHY', error: null, timeMs }; + } catch (error) { + return { + status: 'BROKEN', + error: error instanceof Error ? error.message : 'Unknown error', + timeMs: Date.now() - startTime, + }; + } +} + +/** + * Check if a tool can execute with test parameters + */ +async function checkExecutionHealth(tool: Tool & { package: Package }): Promise<{ + status: HealthStatus; + error: string | null; + timeMs: number; + testParams: Record; +}> { + const startTime = Date.now(); + + // Generate test parameters based on tool schema + const testParams = generateTestParameters(tool); + + try { + const response = await fetch(`${RAILWAY_EXECUTOR_URL}/execute-tool`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + packageName: tool.package.npmPackageName, + exportName: tool.exportName, + version: tool.package.npmVersion, + params: testParams, + env: tool.package.env || {}, + }), + signal: AbortSignal.timeout(30000), // 30 second timeout + }); + + const timeMs = Date.now() - startTime; + const data = await response.json(); + + if (!response.ok || !data.success) { + return { + status: 'BROKEN', + error: data.error || `HTTP ${response.status}`, + timeMs, + testParams, + }; + } + + return { status: 'HEALTHY', error: null, timeMs, testParams }; + } catch (error) { + return { + status: 'BROKEN', + error: error instanceof Error ? error.message : 'Unknown error', + timeMs: Date.now() - startTime, + testParams, + }; + } +} + +/** + * Generate minimal test parameters for a tool + * Uses required parameters with sensible defaults + */ +function generateTestParameters(tool: Tool & { package: Package }): Record { + const parameters = Array.isArray(tool.parameters) + ? (tool.parameters as Array<{ name: string; type: string; required: boolean }>) + : []; + + const testParams: Record = {}; + + for (const param of parameters) { + if (param.required) { + // Generate minimal test value based on type + switch (param.type) { + case 'string': + testParams[param.name] = 'test'; + break; + case 'number': + testParams[param.name] = 1; + break; + case 'boolean': + testParams[param.name] = true; + break; + case 'object': + testParams[param.name] = {}; + break; + case 'array': + testParams[param.name] = []; + break; + default: + testParams[param.name] = 'test'; + } + } + } + + return testParams; +} + +/** + * Perform full health check on a tool (import + execution) + */ +export async function performHealthCheck( + toolId: string, + triggerSource = 'manual' +): Promise { + // Fetch tool with package relation + const tool = await prisma.tool.findUnique({ + where: { id: toolId }, + include: { package: true }, + }); + + if (!tool) { + throw new Error(`Tool not found: ${toolId}`); + } + + console.log(`🏥 Health check starting for ${tool.package.npmPackageName}/${tool.exportName}`); + + // Check import health + const importResult = await checkImportHealth(tool); + console.log( + ` Import: ${importResult.status} ${importResult.error ? `(${importResult.error})` : ''}` + ); + + // Only check execution if import succeeded + let executionResult: Awaited>; + if (importResult.status === 'HEALTHY') { + executionResult = await checkExecutionHealth(tool); + console.log( + ` Execution: ${executionResult.status} ${executionResult.error ? `(${executionResult.error})` : ''}` + ); + } else { + // Skip execution check if import failed + executionResult = { + status: 'UNKNOWN', + error: 'Skipped due to import failure', + timeMs: 0, + testParams: {}, + }; + console.log(' Execution: UNKNOWN (skipped due to import failure)'); + } + + // Determine overall status + const overallStatus: HealthStatus = + importResult.status === 'BROKEN' || executionResult.status === 'BROKEN' + ? 'BROKEN' + : importResult.status === 'HEALTHY' && executionResult.status === 'HEALTHY' + ? 'HEALTHY' + : 'UNKNOWN'; + + console.log(` Overall: ${overallStatus}`); + + // Create HealthCheck record + await prisma.healthCheck.create({ + data: { + toolId: tool.id, + checkType: 'FULL', + triggerSource, + importStatus: importResult.status, + importError: importResult.error, + importTimeMs: importResult.timeMs, + executionStatus: executionResult.status, + executionError: executionResult.error, + executionTimeMs: executionResult.timeMs, + testParameters: executionResult.testParams as Prisma.InputJsonValue, + overallStatus, + }, + }); + + // Update Tool record with latest health status + await prisma.tool.update({ + where: { id: tool.id }, + data: { + importHealth: importResult.status, + executionHealth: executionResult.status, + lastHealthCheck: new Date(), + healthCheckError: importResult.error || executionResult.error, + }, + }); + + return { + toolId: tool.id, + importStatus: importResult.status, + importError: importResult.error, + importTimeMs: importResult.timeMs, + executionStatus: executionResult.status, + executionError: executionResult.error, + executionTimeMs: executionResult.timeMs, + overallStatus, + }; +} + +/** + * Batch health check for multiple tools + * Processes in batches to avoid overwhelming Railway + */ +export async function performBatchHealthCheck( + toolIds: string[], + triggerSource = 'daily-cron', + batchSize = 5 +): Promise<{ + total: number; + healthy: number; + broken: number; + unknown: number; + errors: number; +}> { + let healthy = 0; + let broken = 0; + let unknown = 0; + let errors = 0; + + console.log( + `🏥 Batch health check starting for ${toolIds.length} tools (batch size: ${batchSize})` + ); + + // Process in batches + for (let i = 0; i < toolIds.length; i += batchSize) { + const batch = toolIds.slice(i, i + batchSize); + console.log( + ` Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(toolIds.length / batchSize)}` + ); + + await Promise.all( + batch.map(async (toolId) => { + try { + const result = await performHealthCheck(toolId, triggerSource); + if (result.overallStatus === 'HEALTHY') healthy++; + else if (result.overallStatus === 'BROKEN') broken++; + else unknown++; + } catch (error) { + errors++; + console.error(` ❌ Health check failed for tool ${toolId}:`, error); + } + }) + ); + + // Brief delay between batches to avoid rate limiting + if (i + batchSize < toolIds.length) { + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + } + + console.log( + `✅ Batch health check complete: ${healthy} healthy, ${broken} broken, ${unknown} unknown, ${errors} errors` + ); + + return { total: toolIds.length, healthy, broken, unknown, errors }; +} diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index da53165..a98ddf6 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -73,15 +73,25 @@ model Tool { // Tool Metrics qualityScore Decimal? @map("quality_score") @db.Decimal(3, 2) // 0.00 to 1.00 + // Health Status Fields + importHealth HealthStatus? @default(UNKNOWN) @map("import_health") + executionHealth HealthStatus? @default(UNKNOWN) @map("execution_health") + lastHealthCheck DateTime? @map("last_health_check") + healthCheckError String? @map("health_check_error") @db.Text + // Timestamps createdAt DateTime @default(now()) @map("created_at") updatedAt DateTime @updatedAt @map("updated_at") // Relations - simulations Simulation[] + simulations Simulation[] + healthChecks HealthCheck[] @@unique([packageId, exportName]) @@index([qualityScore]) + @@index([importHealth]) + @@index([executionHealth]) + @@index([lastHealthCheck]) @@map("tools") } @@ -186,3 +196,53 @@ model ExecutionLog { @@index([simulationId]) @@map("execution_logs") } + +/// HealthCheck table - tracks full audit history of health checks +model HealthCheck { + id String @id @default(cuid()) + + // Relations + toolId String @map("tool_id") + tool Tool @relation(fields: [toolId], references: [id], onDelete: Cascade) + + // Check metadata + checkType HealthCheckType @map("check_type") + triggerSource String @map("trigger_source") @db.VarChar(50) // 'sync' | 'manual' | 'daily-cron' + + // Import check results + importStatus HealthStatus @map("import_status") + importError String? @map("import_error") @db.Text + importTimeMs Int? @map("import_time_ms") + + // Execution check results + executionStatus HealthStatus @map("execution_status") + executionError String? @map("execution_error") @db.Text + executionTimeMs Int? @map("execution_time_ms") + testParameters Json? @map("test_parameters") @db.JsonB + + // Overall status + overallStatus HealthStatus @map("overall_status") + + // Timestamps + createdAt DateTime @default(now()) @map("created_at") + + @@index([toolId]) + @@index([checkType]) + @@index([overallStatus]) + @@index([createdAt]) + @@map("health_checks") +} + +/// Health status enum - tracks health check results +enum HealthStatus { + UNKNOWN // Not yet checked + HEALTHY // Passed health check + BROKEN // Failed health check +} + +/// Health check type enum - types of health checks performed +enum HealthCheckType { + IMPORT // Only import check + EXECUTION // Only execution check + FULL // Both import and execution +} diff --git a/vercel.json b/vercel.json index 2eb65e2..6543ce1 100644 --- a/vercel.json +++ b/vercel.json @@ -21,6 +21,10 @@ { "path": "/api/sync/metrics", "schedule": "0 * * * *" + }, + { + "path": "/api/sync/health-check", + "schedule": "0 2 * * *" } ] }