feat: implement health check system (Phase 1 & 2)
Add comprehensive health monitoring for TPMJS tools that tracks both import and execution health via Railway executor service. ## Database Schema - Add HealthStatus enum (UNKNOWN, HEALTHY, BROKEN) - Add HealthCheckType enum (IMPORT, EXECUTION, FULL) - Add health fields to Tool model: - importHealth: tracks if tool can be loaded - executionHealth: tracks if tool can execute - lastHealthCheck: timestamp of last check - healthCheckError: stores error message - Add HealthCheck audit table for full history ## Core Service Create health-check-service.ts with 5 functions: 1. checkImportHealth() - Tests tool loading via /load-and-describe 2. checkExecutionHealth() - Tests execution via /execute-tool 3. generateTestParameters() - Creates minimal test params by type 4. performHealthCheck() - Full check with database updates 5. performBatchHealthCheck() - Processes tools in batches Features: - 30-second timeout per check - Skips execution if import fails - Batch processing (5 concurrent, 1s delays) - Full audit trail in HealthCheck table ## API Endpoints /api/sync/health-check (POST): - Daily cron job at 2am UTC - Checks all tools in database - Requires CRON_SECRET auth - Logs results to SyncLog table - Max duration: 5 minutes /api/tools/broken (GET): - Lists all tools with broken health status - Filters by importHealth='BROKEN' OR executionHealth='BROKEN' - Includes package metadata - Orders by lastHealthCheck DESC ## Configuration - Add RAILWAY_EXECUTOR_URL to env.ts - Add daily cron job to vercel.json - Use db:push for schema changes (existing production data) Next: Manual trigger endpoint + health filtering + UI components 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Ajax Davis
parent
8562eb5b38
commit
148207bcaa
6 changed files with 520 additions and 1 deletions
89
apps/web/src/app/api/sync/health-check/route.ts
Normal file
89
apps/web/src/app/api/sync/health-check/route.ts
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
import { prisma } from '@tpmjs/db';
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
import { env } from '~/env';
|
||||
import { performBatchHealthCheck } from '~/lib/health-check/health-check-service';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 300; // 5 minutes
|
||||
|
||||
/**
|
||||
* POST /api/sync/health-check
|
||||
* Daily health check for all tools
|
||||
*
|
||||
* This endpoint is called by Vercel Cron (daily at 2am UTC)
|
||||
* Requires Authorization: Bearer <CRON_SECRET>
|
||||
*/
|
||||
export async function POST(request: NextRequest) {
|
||||
// Verify cron secret
|
||||
const authHeader = request.headers.get('authorization');
|
||||
const token = authHeader?.replace('Bearer ', '');
|
||||
|
||||
if (env.CRON_SECRET && token !== env.CRON_SECRET) {
|
||||
return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 });
|
||||
}
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
console.log('🏥 Daily health check cron job starting...');
|
||||
|
||||
// Get all tools
|
||||
const tools = await prisma.tool.findMany({
|
||||
select: { id: true },
|
||||
});
|
||||
|
||||
console.log(`📊 Found ${tools.length} tools to check`);
|
||||
|
||||
const toolIds = tools.map((t) => t.id);
|
||||
|
||||
// Perform batch health checks
|
||||
const result = await performBatchHealthCheck(toolIds, 'daily-cron', 5);
|
||||
|
||||
const durationMs = Date.now() - startTime;
|
||||
|
||||
// Log sync operation
|
||||
await prisma.syncLog.create({
|
||||
data: {
|
||||
source: 'health-check',
|
||||
status: result.errors > 0 ? 'partial' : 'success',
|
||||
processed: result.healthy + result.broken + result.unknown,
|
||||
skipped: 0,
|
||||
errors: result.errors,
|
||||
message: `Checked ${result.total} tools: ${result.healthy} healthy, ${result.broken} broken, ${result.unknown} unknown`,
|
||||
metadata: {
|
||||
durationMs,
|
||||
...result,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
console.log(`✅ Daily health check complete in ${durationMs}ms`);
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
...result,
|
||||
durationMs,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('❌ Health check cron failed:', error);
|
||||
|
||||
const durationMs = Date.now() - startTime;
|
||||
|
||||
await prisma.syncLog.create({
|
||||
data: {
|
||||
source: 'health-check',
|
||||
status: 'error',
|
||||
processed: 0,
|
||||
skipped: 0,
|
||||
errors: 1,
|
||||
message: error instanceof Error ? error.message : 'Unknown error',
|
||||
metadata: { durationMs },
|
||||
},
|
||||
});
|
||||
|
||||
return NextResponse.json({ success: false, error: 'Health check failed' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
48
apps/web/src/app/api/tools/broken/route.ts
Normal file
48
apps/web/src/app/api/tools/broken/route.ts
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
import { prisma } from '@tpmjs/db';
|
||||
import { NextResponse } from 'next/server';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 60;
|
||||
|
||||
/**
|
||||
* GET /api/tools/broken
|
||||
* List all tools with broken health status
|
||||
*
|
||||
* Returns tools where importHealth='BROKEN' OR executionHealth='BROKEN'
|
||||
* Includes package relation with npmPackageName and npmVersion
|
||||
*/
|
||||
export async function GET() {
|
||||
try {
|
||||
const brokenTools = await prisma.tool.findMany({
|
||||
where: {
|
||||
OR: [{ importHealth: 'BROKEN' }, { executionHealth: 'BROKEN' }],
|
||||
},
|
||||
include: {
|
||||
package: {
|
||||
select: {
|
||||
npmPackageName: true,
|
||||
npmVersion: true,
|
||||
category: true,
|
||||
isOfficial: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
orderBy: {
|
||||
lastHealthCheck: 'desc',
|
||||
},
|
||||
});
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: brokenTools,
|
||||
count: brokenTools.length,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch broken tools:', error);
|
||||
return NextResponse.json(
|
||||
{ success: false, error: 'Failed to fetch broken tools' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -5,4 +5,8 @@ export const env = createEnv({
|
|||
NODE_ENV: z.enum(['development', 'production', 'test']).default('development'),
|
||||
NEXT_PUBLIC_API_URL: z.string().url().optional(),
|
||||
CRON_SECRET: z.string().min(32).optional(), // Required for Vercel Cron security
|
||||
RAILWAY_EXECUTOR_URL: z
|
||||
.string()
|
||||
.url()
|
||||
.default('https://endearing-commitment-production.up.railway.app'), // Railway service for health checks
|
||||
});
|
||||
|
|
|
|||
314
apps/web/src/lib/health-check/health-check-service.ts
Normal file
314
apps/web/src/lib/health-check/health-check-service.ts
Normal file
|
|
@ -0,0 +1,314 @@
|
|||
/**
|
||||
* Health Check Service
|
||||
* Checks tool import and execution health via Railway executor
|
||||
*/
|
||||
|
||||
import { type HealthStatus, type Package, type Prisma, type Tool, prisma } from '@tpmjs/db';
|
||||
import { env } from '~/env';
|
||||
|
||||
const RAILWAY_EXECUTOR_URL = env.RAILWAY_EXECUTOR_URL;
|
||||
|
||||
interface HealthCheckResult {
|
||||
toolId: string;
|
||||
importStatus: HealthStatus;
|
||||
importError: string | null;
|
||||
importTimeMs: number | null;
|
||||
executionStatus: HealthStatus;
|
||||
executionError: string | null;
|
||||
executionTimeMs: number | null;
|
||||
overallStatus: HealthStatus;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a tool can be imported (load-and-describe)
|
||||
*/
|
||||
async function checkImportHealth(tool: Tool & { package: Package }): Promise<{
|
||||
status: HealthStatus;
|
||||
error: string | null;
|
||||
timeMs: number;
|
||||
}> {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const response = await fetch(`${RAILWAY_EXECUTOR_URL}/load-and-describe`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
packageName: tool.package.npmPackageName,
|
||||
exportName: tool.exportName,
|
||||
version: tool.package.npmVersion,
|
||||
env: tool.package.env || {},
|
||||
}),
|
||||
signal: AbortSignal.timeout(30000), // 30 second timeout
|
||||
});
|
||||
|
||||
const timeMs = Date.now() - startTime;
|
||||
const data = await response.json();
|
||||
|
||||
if (!response.ok || !data.success) {
|
||||
return {
|
||||
status: 'BROKEN',
|
||||
error: data.error || `HTTP ${response.status}`,
|
||||
timeMs,
|
||||
};
|
||||
}
|
||||
|
||||
// Verify tool has required fields
|
||||
if (!data.tool?.description || !data.tool?.inputSchema) {
|
||||
return {
|
||||
status: 'BROKEN',
|
||||
error: 'Missing required tool fields (description or inputSchema)',
|
||||
timeMs,
|
||||
};
|
||||
}
|
||||
|
||||
return { status: 'HEALTHY', error: null, timeMs };
|
||||
} catch (error) {
|
||||
return {
|
||||
status: 'BROKEN',
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
timeMs: Date.now() - startTime,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a tool can execute with test parameters
|
||||
*/
|
||||
async function checkExecutionHealth(tool: Tool & { package: Package }): Promise<{
|
||||
status: HealthStatus;
|
||||
error: string | null;
|
||||
timeMs: number;
|
||||
testParams: Record<string, unknown>;
|
||||
}> {
|
||||
const startTime = Date.now();
|
||||
|
||||
// Generate test parameters based on tool schema
|
||||
const testParams = generateTestParameters(tool);
|
||||
|
||||
try {
|
||||
const response = await fetch(`${RAILWAY_EXECUTOR_URL}/execute-tool`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
packageName: tool.package.npmPackageName,
|
||||
exportName: tool.exportName,
|
||||
version: tool.package.npmVersion,
|
||||
params: testParams,
|
||||
env: tool.package.env || {},
|
||||
}),
|
||||
signal: AbortSignal.timeout(30000), // 30 second timeout
|
||||
});
|
||||
|
||||
const timeMs = Date.now() - startTime;
|
||||
const data = await response.json();
|
||||
|
||||
if (!response.ok || !data.success) {
|
||||
return {
|
||||
status: 'BROKEN',
|
||||
error: data.error || `HTTP ${response.status}`,
|
||||
timeMs,
|
||||
testParams,
|
||||
};
|
||||
}
|
||||
|
||||
return { status: 'HEALTHY', error: null, timeMs, testParams };
|
||||
} catch (error) {
|
||||
return {
|
||||
status: 'BROKEN',
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
timeMs: Date.now() - startTime,
|
||||
testParams,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate minimal test parameters for a tool
|
||||
* Uses required parameters with sensible defaults
|
||||
*/
|
||||
function generateTestParameters(tool: Tool & { package: Package }): Record<string, unknown> {
|
||||
const parameters = Array.isArray(tool.parameters)
|
||||
? (tool.parameters as Array<{ name: string; type: string; required: boolean }>)
|
||||
: [];
|
||||
|
||||
const testParams: Record<string, unknown> = {};
|
||||
|
||||
for (const param of parameters) {
|
||||
if (param.required) {
|
||||
// Generate minimal test value based on type
|
||||
switch (param.type) {
|
||||
case 'string':
|
||||
testParams[param.name] = 'test';
|
||||
break;
|
||||
case 'number':
|
||||
testParams[param.name] = 1;
|
||||
break;
|
||||
case 'boolean':
|
||||
testParams[param.name] = true;
|
||||
break;
|
||||
case 'object':
|
||||
testParams[param.name] = {};
|
||||
break;
|
||||
case 'array':
|
||||
testParams[param.name] = [];
|
||||
break;
|
||||
default:
|
||||
testParams[param.name] = 'test';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return testParams;
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform full health check on a tool (import + execution)
|
||||
*/
|
||||
export async function performHealthCheck(
|
||||
toolId: string,
|
||||
triggerSource = 'manual'
|
||||
): Promise<HealthCheckResult> {
|
||||
// Fetch tool with package relation
|
||||
const tool = await prisma.tool.findUnique({
|
||||
where: { id: toolId },
|
||||
include: { package: true },
|
||||
});
|
||||
|
||||
if (!tool) {
|
||||
throw new Error(`Tool not found: ${toolId}`);
|
||||
}
|
||||
|
||||
console.log(`🏥 Health check starting for ${tool.package.npmPackageName}/${tool.exportName}`);
|
||||
|
||||
// Check import health
|
||||
const importResult = await checkImportHealth(tool);
|
||||
console.log(
|
||||
` Import: ${importResult.status} ${importResult.error ? `(${importResult.error})` : ''}`
|
||||
);
|
||||
|
||||
// Only check execution if import succeeded
|
||||
let executionResult: Awaited<ReturnType<typeof checkExecutionHealth>>;
|
||||
if (importResult.status === 'HEALTHY') {
|
||||
executionResult = await checkExecutionHealth(tool);
|
||||
console.log(
|
||||
` Execution: ${executionResult.status} ${executionResult.error ? `(${executionResult.error})` : ''}`
|
||||
);
|
||||
} else {
|
||||
// Skip execution check if import failed
|
||||
executionResult = {
|
||||
status: 'UNKNOWN',
|
||||
error: 'Skipped due to import failure',
|
||||
timeMs: 0,
|
||||
testParams: {},
|
||||
};
|
||||
console.log(' Execution: UNKNOWN (skipped due to import failure)');
|
||||
}
|
||||
|
||||
// Determine overall status
|
||||
const overallStatus: HealthStatus =
|
||||
importResult.status === 'BROKEN' || executionResult.status === 'BROKEN'
|
||||
? 'BROKEN'
|
||||
: importResult.status === 'HEALTHY' && executionResult.status === 'HEALTHY'
|
||||
? 'HEALTHY'
|
||||
: 'UNKNOWN';
|
||||
|
||||
console.log(` Overall: ${overallStatus}`);
|
||||
|
||||
// Create HealthCheck record
|
||||
await prisma.healthCheck.create({
|
||||
data: {
|
||||
toolId: tool.id,
|
||||
checkType: 'FULL',
|
||||
triggerSource,
|
||||
importStatus: importResult.status,
|
||||
importError: importResult.error,
|
||||
importTimeMs: importResult.timeMs,
|
||||
executionStatus: executionResult.status,
|
||||
executionError: executionResult.error,
|
||||
executionTimeMs: executionResult.timeMs,
|
||||
testParameters: executionResult.testParams as Prisma.InputJsonValue,
|
||||
overallStatus,
|
||||
},
|
||||
});
|
||||
|
||||
// Update Tool record with latest health status
|
||||
await prisma.tool.update({
|
||||
where: { id: tool.id },
|
||||
data: {
|
||||
importHealth: importResult.status,
|
||||
executionHealth: executionResult.status,
|
||||
lastHealthCheck: new Date(),
|
||||
healthCheckError: importResult.error || executionResult.error,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
toolId: tool.id,
|
||||
importStatus: importResult.status,
|
||||
importError: importResult.error,
|
||||
importTimeMs: importResult.timeMs,
|
||||
executionStatus: executionResult.status,
|
||||
executionError: executionResult.error,
|
||||
executionTimeMs: executionResult.timeMs,
|
||||
overallStatus,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch health check for multiple tools
|
||||
* Processes in batches to avoid overwhelming Railway
|
||||
*/
|
||||
export async function performBatchHealthCheck(
|
||||
toolIds: string[],
|
||||
triggerSource = 'daily-cron',
|
||||
batchSize = 5
|
||||
): Promise<{
|
||||
total: number;
|
||||
healthy: number;
|
||||
broken: number;
|
||||
unknown: number;
|
||||
errors: number;
|
||||
}> {
|
||||
let healthy = 0;
|
||||
let broken = 0;
|
||||
let unknown = 0;
|
||||
let errors = 0;
|
||||
|
||||
console.log(
|
||||
`🏥 Batch health check starting for ${toolIds.length} tools (batch size: ${batchSize})`
|
||||
);
|
||||
|
||||
// Process in batches
|
||||
for (let i = 0; i < toolIds.length; i += batchSize) {
|
||||
const batch = toolIds.slice(i, i + batchSize);
|
||||
console.log(
|
||||
` Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(toolIds.length / batchSize)}`
|
||||
);
|
||||
|
||||
await Promise.all(
|
||||
batch.map(async (toolId) => {
|
||||
try {
|
||||
const result = await performHealthCheck(toolId, triggerSource);
|
||||
if (result.overallStatus === 'HEALTHY') healthy++;
|
||||
else if (result.overallStatus === 'BROKEN') broken++;
|
||||
else unknown++;
|
||||
} catch (error) {
|
||||
errors++;
|
||||
console.error(` ❌ Health check failed for tool ${toolId}:`, error);
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
// Brief delay between batches to avoid rate limiting
|
||||
if (i + batchSize < toolIds.length) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`✅ Batch health check complete: ${healthy} healthy, ${broken} broken, ${unknown} unknown, ${errors} errors`
|
||||
);
|
||||
|
||||
return { total: toolIds.length, healthy, broken, unknown, errors };
|
||||
}
|
||||
|
|
@ -73,15 +73,25 @@ model Tool {
|
|||
// Tool Metrics
|
||||
qualityScore Decimal? @map("quality_score") @db.Decimal(3, 2) // 0.00 to 1.00
|
||||
|
||||
// Health Status Fields
|
||||
importHealth HealthStatus? @default(UNKNOWN) @map("import_health")
|
||||
executionHealth HealthStatus? @default(UNKNOWN) @map("execution_health")
|
||||
lastHealthCheck DateTime? @map("last_health_check")
|
||||
healthCheckError String? @map("health_check_error") @db.Text
|
||||
|
||||
// Timestamps
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
|
||||
// Relations
|
||||
simulations Simulation[]
|
||||
simulations Simulation[]
|
||||
healthChecks HealthCheck[]
|
||||
|
||||
@@unique([packageId, exportName])
|
||||
@@index([qualityScore])
|
||||
@@index([importHealth])
|
||||
@@index([executionHealth])
|
||||
@@index([lastHealthCheck])
|
||||
@@map("tools")
|
||||
}
|
||||
|
||||
|
|
@ -186,3 +196,53 @@ model ExecutionLog {
|
|||
@@index([simulationId])
|
||||
@@map("execution_logs")
|
||||
}
|
||||
|
||||
/// HealthCheck table - tracks full audit history of health checks
|
||||
model HealthCheck {
|
||||
id String @id @default(cuid())
|
||||
|
||||
// Relations
|
||||
toolId String @map("tool_id")
|
||||
tool Tool @relation(fields: [toolId], references: [id], onDelete: Cascade)
|
||||
|
||||
// Check metadata
|
||||
checkType HealthCheckType @map("check_type")
|
||||
triggerSource String @map("trigger_source") @db.VarChar(50) // 'sync' | 'manual' | 'daily-cron'
|
||||
|
||||
// Import check results
|
||||
importStatus HealthStatus @map("import_status")
|
||||
importError String? @map("import_error") @db.Text
|
||||
importTimeMs Int? @map("import_time_ms")
|
||||
|
||||
// Execution check results
|
||||
executionStatus HealthStatus @map("execution_status")
|
||||
executionError String? @map("execution_error") @db.Text
|
||||
executionTimeMs Int? @map("execution_time_ms")
|
||||
testParameters Json? @map("test_parameters") @db.JsonB
|
||||
|
||||
// Overall status
|
||||
overallStatus HealthStatus @map("overall_status")
|
||||
|
||||
// Timestamps
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
|
||||
@@index([toolId])
|
||||
@@index([checkType])
|
||||
@@index([overallStatus])
|
||||
@@index([createdAt])
|
||||
@@map("health_checks")
|
||||
}
|
||||
|
||||
/// Health status enum - tracks health check results
|
||||
enum HealthStatus {
|
||||
UNKNOWN // Not yet checked
|
||||
HEALTHY // Passed health check
|
||||
BROKEN // Failed health check
|
||||
}
|
||||
|
||||
/// Health check type enum - types of health checks performed
|
||||
enum HealthCheckType {
|
||||
IMPORT // Only import check
|
||||
EXECUTION // Only execution check
|
||||
FULL // Both import and execution
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,10 @@
|
|||
{
|
||||
"path": "/api/sync/metrics",
|
||||
"schedule": "0 * * * *"
|
||||
},
|
||||
{
|
||||
"path": "/api/sync/health-check",
|
||||
"schedule": "0 2 * * *"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue