diff --git a/apps/web/package.json b/apps/web/package.json index 49693d9..7fb84c6 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -45,6 +45,8 @@ "@vercel/blob": "^2.0.0", "@vercel/kv": "^3.0.0", "ai": "6.0.49", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", "better-auth": "^1.4.10", "bm25": "^0.1.1", "d3": "^7.9.0", diff --git a/apps/web/src/app/(profile)/[username]/collections/[slug]/scenarios/[id]/ExpandedRunDetails.tsx b/apps/web/src/app/(profile)/[username]/collections/[slug]/scenarios/[id]/ExpandedRunDetails.tsx index f24ca8f..baa2c86 100644 --- a/apps/web/src/app/(profile)/[username]/collections/[slug]/scenarios/[id]/ExpandedRunDetails.tsx +++ b/apps/web/src/app/(profile)/[username]/collections/[slug]/scenarios/[id]/ExpandedRunDetails.tsx @@ -1,10 +1,130 @@ +import { Badge } from '@tpmjs/ui/Badge/Badge'; +import { Icon } from '@tpmjs/ui/Icon/Icon'; import type { ScenarioRun } from './page'; interface ExpandedRunDetailsProps { run: ScenarioRun; } +/** Displays passed and failed assertions */ +function AssertionsSection({ assertions }: { assertions: { passed: string[]; failed: string[] } }) { + return ( +
+

+ Assertions +

+
+ {assertions.passed.length > 0 && ( +
+
+ + Passed ({assertions.passed.length}) +
+
+ {assertions.passed.map((assertion) => ( +
+ {assertion} +
+ ))} +
+
+ )} + {assertions.failed.length > 0 && ( +
+
+ + Failed ({assertions.failed.length}) +
+
+ {assertions.failed.map((assertion) => ( +
+ {assertion} +
+ ))} +
+
+ )} +
+
+ ); +} + +/** Displays conversation messages */ +function ConversationSection({ + conversation, +}: { + conversation: NonNullable; +}) { + return ( +
+
+

+ Conversation History +

+
+
+ {conversation.map((msg) => ( + + ))} +
+
+ ); +} + +/** Single conversation message */ +function ConversationMessage({ msg }: { msg: NonNullable[number] }) { + if (msg.role === 'USER') { + return ( +
+
+
{msg.content}
+
+
+ ); + } + + if (msg.role === 'ASSISTANT') { + return msg.content ? ( +
+
+
+
{msg.content}
+
+
+
+ ) : null; + } + + if (msg.role === 'TOOL') { + return ( +
+
+
+
+ {msg.toolName || 'Unknown Tool'} +
+ {msg.toolResult != null && ( +
+ 
+                  {typeof msg.toolResult === 'string'
+                    ? msg.toolResult
+                    : JSON.stringify(msg.toolResult, null, 2)}
+
+
+ )} +
+
+
+ ); + } + + return null; +} + export function ExpandedRunDetails({ run }: ExpandedRunDetailsProps) { + const hasAssertions = + run.assertions && (run.assertions.passed.length > 0 || run.assertions.failed.length > 0); + return (
@@ -15,7 +135,24 @@ export function ExpandedRunDetails({ run }: ExpandedRunDetailsProps) { LLM Evaluation
- {run.evaluator.model && {run.evaluator.model}} + + + {run.evaluator.verdict === 'pass' ? 'Pass' : 'Fail'} + + {run.evaluator.model && ( + + {run.evaluator.model} + + )}
{run.evaluator?.reason && (

{run.evaluator.reason}

@@ -55,6 +192,9 @@ export function ExpandedRunDetails({ run }: ExpandedRunDetailsProps) {
+ {/* Assertions Results */} + {hasAssertions && run.assertions && } + {/* Output (if owner) */} {run.output && (
@@ -80,61 +220,7 @@ export function ExpandedRunDetails({ run }: ExpandedRunDetailsProps) { )} {/* Conversation History */} - {run.conversation && ( -
-
-

- Conversation History -

-
-
- {run.conversation.map((msg) => ( -
- {msg.role === 'USER' && ( -
-
-
{msg.content}
-
-
- )} - {msg.role === 'ASSISTANT' && ( -
- {msg.content && ( -
-
-
- {msg.content} -
-
-
- )} -
- )} - {msg.role === 'TOOL' && ( -
-
-
-
- {msg.toolName || 'Unknown Tool'} -
- {msg.toolResult != null && ( -
- 
-                              {typeof msg.toolResult === 'string'
-                                ? msg.toolResult
-                                : JSON.stringify(msg.toolResult, null, 2)}
-
-
- )} -
-
-
- )} -
- ))} -
-
- )} + {run.conversation && }
); } diff --git a/apps/web/src/lib/scenarios/evaluate.test.ts b/apps/web/src/lib/scenarios/evaluate.test.ts index 0d998d1..39c7dbd 100644 --- a/apps/web/src/lib/scenarios/evaluate.test.ts +++ b/apps/web/src/lib/scenarios/evaluate.test.ts @@ -3,13 +3,20 @@ * * Tests for the evaluation logic including: * - Regex assertion matching + * - JSON Schema validation + * - JSON extraction from various formats * - Final verdict determination * - Assertion result handling */ import { describe, expect, it } from 'vitest'; -import { determineFinalVerdict, runAssertions } from './evaluate'; +import { + determineFinalVerdict, + extractJsonFromOutput, + runAssertions, + validateJsonSchema, +} from './evaluate'; describe('runAssertions', () => { describe('regex assertions', () => { @@ -93,29 +100,262 @@ describe('runAssertions', () => { }); describe('schema assertions', () => { - it('should note when schema is provided', () => { - const output = '{"name": "test"}'; - const assertions = { schema: { type: 'object' } }; + it('should pass when JSON validates against schema', () => { + const output = '{"name": "test", "age": 25}'; + const assertions = { + schema: { + type: 'object', + properties: { + name: { type: 'string' }, + age: { type: 'number' }, + }, + required: ['name'], + }, + }; const result = runAssertions(output, assertions); - expect(result.passed).toContain('schema:provided (validation pending)'); + expect(result.passed).toContain('schema: JSON validates against schema'); + expect(result.failed).toHaveLength(0); }); - }); - describe('combined assertions', () => { - it('should handle both regex and schema assertions', () => { - const output = '{"status": "success"}'; + it('should fail when JSON does not validate against schema', () => { + const output = '{"name": 123}'; + const assertions = { + schema: { + type: 'object', + properties: { + name: { type: 'string' }, + }, + }, + }; + + const result = runAssertions(output, assertions); + + expect(result.failed.length).toBeGreaterThan(0); + expect(result.failed[0]).toContain('schema:'); + expect(result.passed).toHaveLength(0); + }); + + it('should fail when output is not valid JSON', () => { + const output = 'This is not JSON at all'; const assertions = { - regex: ['success'], schema: { type: 'object' }, }; const result = runAssertions(output, assertions); - expect(result.passed).toContain('regex:success'); - expect(result.passed).toContain('schema:provided (validation pending)'); + expect(result.failed).toContain('schema: Output does not contain valid JSON'); }); + + it('should handle schema with required fields', () => { + const output = '{"name": "test"}'; + const assertions = { + schema: { + type: 'object', + required: ['name', 'email'], + }, + }; + + const result = runAssertions(output, assertions); + + expect(result.failed.length).toBeGreaterThan(0); + expect(result.failed[0]).toContain('schema:'); + }); + + it('should handle empty schema object', () => { + const output = '{"anything": "goes"}'; + const assertions = { schema: {} }; + + const result = runAssertions(output, assertions); + + // Empty schema should not run validation + expect(result.passed).toHaveLength(0); + expect(result.failed).toHaveLength(0); + }); + }); + + describe('combined assertions', () => { + it('should handle both regex and schema assertions passing', () => { + const output = '{"status": "success", "count": 42}'; + const assertions = { + regex: ['success', '42'], + schema: { + type: 'object', + properties: { + status: { type: 'string' }, + count: { type: 'number' }, + }, + }, + }; + + const result = runAssertions(output, assertions); + + expect(result.passed).toContain('regex:success'); + expect(result.passed).toContain('regex:42'); + expect(result.passed).toContain('schema: JSON validates against schema'); + expect(result.failed).toHaveLength(0); + }); + + it('should handle regex passing but schema failing', () => { + const output = '{"status": "success", "count": "not a number"}'; + const assertions = { + regex: ['success'], + schema: { + type: 'object', + properties: { + count: { type: 'number' }, + }, + }, + }; + + const result = runAssertions(output, assertions); + + expect(result.passed).toContain('regex:success'); + expect(result.failed.length).toBeGreaterThan(0); + expect(result.failed[0]).toContain('schema:'); + }); + }); +}); + +describe('extractJsonFromOutput', () => { + it('should extract direct JSON object', () => { + const output = '{"name": "test"}'; + const result = extractJsonFromOutput(output); + expect(result).toEqual({ name: 'test' }); + }); + + it('should extract direct JSON array', () => { + const output = '[1, 2, 3]'; + const result = extractJsonFromOutput(output); + expect(result).toEqual([1, 2, 3]); + }); + + it('should extract JSON from markdown code block with json tag', () => { + const output = 'Here is the result:\n```json\n{"status": "ok"}\n```'; + const result = extractJsonFromOutput(output); + expect(result).toEqual({ status: 'ok' }); + }); + + it('should extract JSON from markdown code block without tag', () => { + const output = 'Result:\n```\n{"value": 42}\n```\nDone.'; + const result = extractJsonFromOutput(output); + expect(result).toEqual({ value: 42 }); + }); + + it('should extract JSON embedded in text', () => { + const output = 'The response is {"data": "found"} as expected.'; + const result = extractJsonFromOutput(output); + expect(result).toEqual({ data: 'found' }); + }); + + it('should return null for non-JSON output', () => { + const output = 'This is just plain text with no JSON.'; + const result = extractJsonFromOutput(output); + expect(result).toBeNull(); + }); + + it('should handle whitespace around JSON', () => { + const output = ' \n {"trimmed": true} \n '; + const result = extractJsonFromOutput(output); + expect(result).toEqual({ trimmed: true }); + }); + + it('should handle nested JSON objects', () => { + const output = '{"outer": {"inner": {"deep": "value"}}}'; + const result = extractJsonFromOutput(output); + expect(result).toEqual({ outer: { inner: { deep: 'value' } } }); + }); +}); + +describe('validateJsonSchema', () => { + it('should validate simple object schema', () => { + const data = { name: 'test', age: 25 }; + const schema = { + type: 'object', + properties: { + name: { type: 'string' }, + age: { type: 'number' }, + }, + }; + + const result = validateJsonSchema(data, schema); + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + }); + + it('should return errors for invalid data', () => { + const data = { name: 123 }; + const schema = { + type: 'object', + properties: { + name: { type: 'string' }, + }, + }; + + const result = validateJsonSchema(data, schema); + expect(result.valid).toBe(false); + expect(result.errors.length).toBeGreaterThan(0); + }); + + it('should validate required fields', () => { + const data = { name: 'test' }; + const schema = { + type: 'object', + required: ['name', 'email'], + }; + + const result = validateJsonSchema(data, schema); + expect(result.valid).toBe(false); + expect(result.errors.some((e) => e.includes('email'))).toBe(true); + }); + + it('should validate array schemas', () => { + const data = [1, 2, 3]; + const schema = { + type: 'array', + items: { type: 'number' }, + }; + + const result = validateJsonSchema(data, schema); + expect(result.valid).toBe(true); + }); + + it('should handle format validation (email)', () => { + const data = { email: 'invalid-email' }; + const schema = { + type: 'object', + properties: { + email: { type: 'string', format: 'email' }, + }, + }; + + const result = validateJsonSchema(data, schema); + expect(result.valid).toBe(false); + }); + + it('should pass format validation for valid email', () => { + const data = { email: 'test@example.com' }; + const schema = { + type: 'object', + properties: { + email: { type: 'string', format: 'email' }, + }, + }; + + const result = validateJsonSchema(data, schema); + expect(result.valid).toBe(true); + }); + + it('should handle invalid schema gracefully', () => { + const data = { test: true }; + const schema = { + type: 'invalid-type-that-does-not-exist', + }; + + const result = validateJsonSchema(data, schema as Record); + // Ajv with strict: false will still try to validate + expect(result.valid).toBe(false); }); }); diff --git a/apps/web/src/lib/scenarios/evaluate.ts b/apps/web/src/lib/scenarios/evaluate.ts index e86b77e..b0c17fd 100644 --- a/apps/web/src/lib/scenarios/evaluate.ts +++ b/apps/web/src/lib/scenarios/evaluate.ts @@ -2,13 +2,20 @@ * Scenario Evaluation Service * * Uses LLM judgment to evaluate if a scenario execution was successful. + * Supports both regex pattern matching and JSON Schema validation for assertions. */ import { anthropic } from '@ai-sdk/anthropic'; import { openai } from '@ai-sdk/openai'; import { generateObject } from 'ai'; +import Ajv from 'ajv'; +import addFormats from 'ajv-formats'; import { z } from 'zod'; +// Initialize Ajv with common formats (email, uri, date-time, etc.) +const ajv = new Ajv({ allErrors: true, strict: false }); +addFormats(ajv); + const EvaluationSchema = z.object({ verdict: z.enum(['pass', 'fail']).describe('Whether the scenario was completed successfully'), reason: z.string().describe('Brief explanation of why the scenario passed or failed'), @@ -94,9 +101,90 @@ Provide your verdict, a brief reason, and your confidence level.`, return object; } +/** + * Extract JSON from text output + * + * Attempts to find and parse JSON from various formats: + * - Direct JSON + * - JSON wrapped in markdown code blocks + * - JSON embedded in text + * + * @param output The text to extract JSON from + * @returns Parsed JSON or null if not found + */ +export function extractJsonFromOutput(output: string): unknown | null { + // Try direct parse first + try { + return JSON.parse(output.trim()); + } catch { + // Continue to other strategies + } + + // Try extracting from markdown code blocks (```json ... ``` or ``` ... ```) + const codeBlockMatch = output.match(/```(?:json)?\s*([\s\S]*?)```/); + if (codeBlockMatch?.[1]) { + try { + return JSON.parse(codeBlockMatch[1].trim()); + } catch { + // Continue to other strategies + } + } + + // Try finding JSON object or array in the text + const jsonMatch = output.match(/(\{[\s\S]*\}|\[[\s\S]*\])/); + if (jsonMatch?.[1]) { + try { + return JSON.parse(jsonMatch[1]); + } catch { + // Could not parse as JSON + } + } + + return null; +} + +/** + * Validate data against a JSON Schema + * + * @param data The data to validate + * @param schema The JSON Schema to validate against + * @returns Validation result with errors if any + */ +export function validateJsonSchema( + data: unknown, + schema: Record +): { valid: boolean; errors: string[] } { + try { + const validate = ajv.compile(schema); + const valid = validate(data); + + if (valid) { + return { valid: true, errors: [] }; + } + + // Format errors into readable strings + const errors = (validate.errors || []).map((err) => { + const path = err.instancePath || 'root'; + const message = err.message || 'Unknown error'; + return `${path}: ${message}`; + }); + + return { valid: false, errors }; + } catch (err) { + return { + valid: false, + errors: [`Schema compilation error: ${err instanceof Error ? err.message : 'Unknown error'}`], + }; + } +} + /** * Run assertions against the output * + * Supports two types of assertions: + * - regex: Array of regex patterns that must match the output + * - schema: JSON Schema that the output (parsed as JSON) must validate against + * * @param output The agent output to check * @param assertions The assertions to run */ @@ -123,11 +211,25 @@ export function runAssertions( } } - // Schema assertions would require more complex validation - // For now, we'll just note if schema was provided - if (assertions.schema) { - // TODO: Implement JSON schema validation against parsed output - passed.push('schema:provided (validation pending)'); + // Validate against JSON Schema if provided + if (assertions.schema && Object.keys(assertions.schema).length > 0) { + const extractedJson = extractJsonFromOutput(output); + + if (extractedJson === null) { + failed.push('schema: Output does not contain valid JSON'); + } else { + const validation = validateJsonSchema(extractedJson, assertions.schema); + + if (validation.valid) { + passed.push('schema: JSON validates against schema'); + } else { + // Include first 3 errors for clarity + const errorSummary = validation.errors.slice(0, 3).join('; '); + const moreErrors = + validation.errors.length > 3 ? ` (+${validation.errors.length - 3} more)` : ''; + failed.push(`schema: ${errorSummary}${moreErrors}`); + } + } } return { passed, failed }; diff --git a/apps/web/src/lib/scenarios/execute.test.ts b/apps/web/src/lib/scenarios/execute.test.ts new file mode 100644 index 0000000..9beb132 --- /dev/null +++ b/apps/web/src/lib/scenarios/execute.test.ts @@ -0,0 +1,447 @@ +/** + * Scenario Execution Unit Tests + * + * Tests for the execution logic including: + * - Quota management + * - Quality score metrics + * - Error handling + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { ScenarioExecutionError } from './execute'; + +// Mock prisma +vi.mock('@tpmjs/db', () => ({ + prisma: { + scenarioQuota: { + findUnique: vi.fn(), + create: vi.fn(), + update: vi.fn(), + }, + scenario: { + findUnique: vi.fn(), + update: vi.fn(), + }, + scenarioRun: { + create: vi.fn(), + update: vi.fn(), + }, + collection: { + findUnique: vi.fn(), + }, + }, +})); + +describe('ScenarioExecutionError', () => { + it('should create error with category', () => { + const error = new ScenarioExecutionError('Test error message', 'COLLECTION_NOT_FOUND', { + scenarioId: 'test-123', + }); + + expect(error.message).toBe('Test error message'); + expect(error.category).toBe('COLLECTION_NOT_FOUND'); + expect(error.context).toEqual({ scenarioId: 'test-123' }); + expect(error.name).toBe('ScenarioExecutionError'); + }); + + it('should work without context', () => { + const error = new ScenarioExecutionError('No context error', 'NO_TOOLS'); + + expect(error.message).toBe('No context error'); + expect(error.category).toBe('NO_TOOLS'); + expect(error.context).toBeUndefined(); + }); + + it('should be an instance of Error', () => { + const error = new ScenarioExecutionError('Test', 'UNKNOWN_ERROR'); + expect(error instanceof Error).toBe(true); + }); + + it('should support all error categories', () => { + const categories = [ + 'COLLECTION_NOT_FOUND', + 'NO_COLLECTION', + 'NO_TOOLS', + 'TOOL_BUILD_ERROR', + 'EXECUTION_ERROR', + 'EVALUATION_ERROR', + 'QUOTA_EXCEEDED', + 'UNKNOWN_ERROR', + ] as const; + + for (const category of categories) { + const error = new ScenarioExecutionError(`Error: ${category}`, category); + expect(error.category).toBe(category); + } + }); +}); + +describe('Quota Management', () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2025-01-15T12:00:00Z')); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.clearAllMocks(); + }); + + describe('checkAndDecrementQuota', () => { + it('should create quota for new user', async () => { + const { prisma } = await import('@tpmjs/db'); + const { checkAndDecrementQuota } = await import('./execute'); + + vi.mocked(prisma.scenarioQuota.findUnique).mockResolvedValue(null); + vi.mocked(prisma.scenarioQuota.create).mockResolvedValue({ + id: 'quota-1', + userId: 'user-1', + dailyLimit: 50, + dailyUsed: 0, + lastResetAt: new Date('2025-01-15T00:00:00Z'), + createdAt: new Date(), + updatedAt: new Date(), + }); + vi.mocked(prisma.scenarioQuota.update).mockResolvedValue({ + id: 'quota-1', + userId: 'user-1', + dailyLimit: 50, + dailyUsed: 1, + lastResetAt: new Date('2025-01-15T00:00:00Z'), + createdAt: new Date(), + updatedAt: new Date(), + }); + + const result = await checkAndDecrementQuota('user-1'); + + expect(result.allowed).toBe(true); + expect(result.remaining).toBe(49); + expect(prisma.scenarioQuota.create).toHaveBeenCalled(); + }); + + it('should deny when quota exceeded', async () => { + const { prisma } = await import('@tpmjs/db'); + const { checkAndDecrementQuota } = await import('./execute'); + + vi.mocked(prisma.scenarioQuota.findUnique).mockResolvedValue({ + id: 'quota-1', + userId: 'user-1', + dailyLimit: 50, + dailyUsed: 50, + lastResetAt: new Date('2025-01-15T00:00:00Z'), + createdAt: new Date(), + updatedAt: new Date(), + }); + + const result = await checkAndDecrementQuota('user-1'); + + expect(result.allowed).toBe(false); + expect(result.remaining).toBe(0); + }); + + it('should reset quota on new day', async () => { + const { prisma } = await import('@tpmjs/db'); + const { checkAndDecrementQuota } = await import('./execute'); + + // Quota from yesterday + vi.mocked(prisma.scenarioQuota.findUnique).mockResolvedValue({ + id: 'quota-1', + userId: 'user-1', + dailyLimit: 50, + dailyUsed: 45, + lastResetAt: new Date('2025-01-14T00:00:00Z'), // Yesterday + createdAt: new Date(), + updatedAt: new Date(), + }); + + // After reset + vi.mocked(prisma.scenarioQuota.update).mockResolvedValue({ + id: 'quota-1', + userId: 'user-1', + dailyLimit: 50, + dailyUsed: 0, + lastResetAt: new Date('2025-01-15T00:00:00Z'), + createdAt: new Date(), + updatedAt: new Date(), + }); + + const result = await checkAndDecrementQuota('user-1'); + + expect(result.allowed).toBe(true); + // After reset: 50 - 0 - 1 = 49 + expect(result.remaining).toBe(49); + }); + }); + + describe('getQuotaStatus', () => { + it('should return default quota for new user', async () => { + const { prisma } = await import('@tpmjs/db'); + const { getQuotaStatus } = await import('./execute'); + + vi.mocked(prisma.scenarioQuota.findUnique).mockResolvedValue(null); + + const result = await getQuotaStatus('user-1'); + + expect(result.used).toBe(0); + expect(result.limit).toBe(50); + expect(result.remaining).toBe(50); + }); + + it('should return current quota for existing user', async () => { + const { prisma } = await import('@tpmjs/db'); + const { getQuotaStatus } = await import('./execute'); + + vi.mocked(prisma.scenarioQuota.findUnique).mockResolvedValue({ + id: 'quota-1', + userId: 'user-1', + dailyLimit: 50, + dailyUsed: 10, + lastResetAt: new Date('2025-01-15T00:00:00Z'), + createdAt: new Date(), + updatedAt: new Date(), + }); + + const result = await getQuotaStatus('user-1'); + + expect(result.used).toBe(10); + expect(result.limit).toBe(50); + expect(result.remaining).toBe(40); + }); + }); +}); + +describe('Scenario Metrics', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe('updateScenarioMetrics', () => { + it('should increase quality score on pass', async () => { + const { prisma } = await import('@tpmjs/db'); + const { updateScenarioMetrics } = await import('./execute'); + + vi.mocked(prisma.scenario.findUnique).mockResolvedValue({ + id: 'scenario-1', + collectionId: 'col-1', + prompt: 'Test', + name: 'Test', + description: null, + tags: [], + assertions: null, + qualityScore: 0.5, + consecutivePasses: 0, + consecutiveFails: 2, + totalRuns: 5, + lastRunAt: null, + lastRunStatus: 'fail', + createdAt: new Date(), + updatedAt: new Date(), + }); + + vi.mocked(prisma.scenario.update).mockResolvedValue({} as any); + + await updateScenarioMetrics('scenario-1', 'pass'); + + expect(prisma.scenario.update).toHaveBeenCalledWith({ + where: { id: 'scenario-1' }, + data: expect.objectContaining({ + consecutivePasses: 1, + consecutiveFails: 0, + totalRuns: 6, + lastRunStatus: 'pass', + }), + }); + + // Check quality score increased + const updateCall = vi.mocked(prisma.scenario.update).mock.calls[0]?.[0]; + expect(updateCall?.data.qualityScore).toBeGreaterThan(0.5); + }); + + it('should decrease quality score on fail', async () => { + const { prisma } = await import('@tpmjs/db'); + const { updateScenarioMetrics } = await import('./execute'); + + vi.mocked(prisma.scenario.findUnique).mockResolvedValue({ + id: 'scenario-1', + collectionId: 'col-1', + prompt: 'Test', + name: 'Test', + description: null, + tags: [], + assertions: null, + qualityScore: 0.5, + consecutivePasses: 3, + consecutiveFails: 0, + totalRuns: 5, + lastRunAt: null, + lastRunStatus: 'pass', + createdAt: new Date(), + updatedAt: new Date(), + }); + + vi.mocked(prisma.scenario.update).mockResolvedValue({} as any); + + await updateScenarioMetrics('scenario-1', 'fail'); + + expect(prisma.scenario.update).toHaveBeenCalledWith({ + where: { id: 'scenario-1' }, + data: expect.objectContaining({ + consecutivePasses: 0, + consecutiveFails: 1, + totalRuns: 6, + lastRunStatus: 'fail', + }), + }); + + // Check quality score decreased + const updateCall = vi.mocked(prisma.scenario.update).mock.calls[0]?.[0]; + expect(updateCall?.data.qualityScore).toBeLessThan(0.5); + }); + + it('should not exceed quality score bounds', async () => { + const { prisma } = await import('@tpmjs/db'); + const { updateScenarioMetrics } = await import('./execute'); + + // Test max bound + vi.mocked(prisma.scenario.findUnique).mockResolvedValue({ + id: 'scenario-1', + collectionId: 'col-1', + prompt: 'Test', + name: 'Test', + description: null, + tags: [], + assertions: null, + qualityScore: 0.99, + consecutivePasses: 10, + consecutiveFails: 0, + totalRuns: 20, + lastRunAt: null, + lastRunStatus: 'pass', + createdAt: new Date(), + updatedAt: new Date(), + }); + + vi.mocked(prisma.scenario.update).mockResolvedValue({} as any); + + await updateScenarioMetrics('scenario-1', 'pass'); + + const updateCall = vi.mocked(prisma.scenario.update).mock.calls[0]?.[0]; + expect(updateCall?.data.qualityScore).toBeLessThanOrEqual(1.0); + }); + + it('should not go below zero', async () => { + const { prisma } = await import('@tpmjs/db'); + const { updateScenarioMetrics } = await import('./execute'); + + // Test min bound + vi.mocked(prisma.scenario.findUnique).mockResolvedValue({ + id: 'scenario-1', + collectionId: 'col-1', + prompt: 'Test', + name: 'Test', + description: null, + tags: [], + assertions: null, + qualityScore: 0.05, + consecutivePasses: 0, + consecutiveFails: 5, + totalRuns: 10, + lastRunAt: null, + lastRunStatus: 'fail', + createdAt: new Date(), + updatedAt: new Date(), + }); + + vi.mocked(prisma.scenario.update).mockResolvedValue({} as any); + + await updateScenarioMetrics('scenario-1', 'fail'); + + const updateCall = vi.mocked(prisma.scenario.update).mock.calls[0]?.[0]; + expect(updateCall?.data.qualityScore).toBeGreaterThanOrEqual(0); + }); + + it('should handle non-existent scenario gracefully', async () => { + const { prisma } = await import('@tpmjs/db'); + const { updateScenarioMetrics } = await import('./execute'); + + vi.mocked(prisma.scenario.findUnique).mockResolvedValue(null); + + // Should not throw + await expect(updateScenarioMetrics('non-existent', 'pass')).resolves.toBeUndefined(); + expect(prisma.scenario.update).not.toHaveBeenCalled(); + }); + }); +}); + +describe('Quality Score Algorithm', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('should reward consecutive passes with streak bonus', async () => { + const { prisma } = await import('@tpmjs/db'); + const { updateScenarioMetrics } = await import('./execute'); + + // Starting with 3 consecutive passes + vi.mocked(prisma.scenario.findUnique).mockResolvedValue({ + id: 'scenario-1', + collectionId: 'col-1', + prompt: 'Test', + name: 'Test', + description: null, + tags: [], + assertions: null, + qualityScore: 0.3, + consecutivePasses: 3, + consecutiveFails: 0, + totalRuns: 5, + lastRunAt: null, + lastRunStatus: 'pass', + createdAt: new Date(), + updatedAt: new Date(), + }); + + vi.mocked(prisma.scenario.update).mockResolvedValue({} as any); + + await updateScenarioMetrics('scenario-1', 'pass'); + + const updateCall = vi.mocked(prisma.scenario.update).mock.calls[0]?.[0]; + // Base: 0.3, +0.05 for pass, +0.01*4 for streak bonus = 0.3 + 0.05 + 0.04 = 0.39 + expect(updateCall?.data.qualityScore).toBeCloseTo(0.39, 2); + expect(updateCall?.data.consecutivePasses).toBe(4); + }); + + it('should penalize consecutive fails with streak penalty', async () => { + const { prisma } = await import('@tpmjs/db'); + const { updateScenarioMetrics } = await import('./execute'); + + // Starting with 2 consecutive fails + vi.mocked(prisma.scenario.findUnique).mockResolvedValue({ + id: 'scenario-1', + collectionId: 'col-1', + prompt: 'Test', + name: 'Test', + description: null, + tags: [], + assertions: null, + qualityScore: 0.5, + consecutivePasses: 0, + consecutiveFails: 2, + totalRuns: 5, + lastRunAt: null, + lastRunStatus: 'fail', + createdAt: new Date(), + updatedAt: new Date(), + }); + + vi.mocked(prisma.scenario.update).mockResolvedValue({} as any); + + await updateScenarioMetrics('scenario-1', 'fail'); + + const updateCall = vi.mocked(prisma.scenario.update).mock.calls[0]?.[0]; + // Base: 0.5, -0.1 for fail, -0.02*3 for streak penalty = 0.5 - 0.1 - 0.06 = 0.34 + expect(updateCall?.data.qualityScore).toBeCloseTo(0.34, 2); + expect(updateCall?.data.consecutiveFails).toBe(3); + }); +}); diff --git a/apps/web/src/lib/scenarios/execute.ts b/apps/web/src/lib/scenarios/execute.ts index 7ffb240..d5c481d 100644 --- a/apps/web/src/lib/scenarios/execute.ts +++ b/apps/web/src/lib/scenarios/execute.ts @@ -15,6 +15,7 @@ import { createToolDefinition } from '../ai-agent/tool-executor-agent'; import { parseExecutorConfig, resolveExecutorConfig } from '../executors'; import { determineFinalVerdict, + type EvaluationResult, type EvaluatorModelId, evaluateScenarioRun, runAssertions, @@ -25,6 +26,55 @@ const DEFAULT_MODEL = 'gpt-4.1-mini'; const MAX_RETRIES = 1; const MAX_TOOL_STEPS = 10; +/** + * Error categories for scenario execution + */ +export type ScenarioErrorCategory = + | 'COLLECTION_NOT_FOUND' + | 'NO_COLLECTION' + | 'NO_TOOLS' + | 'TOOL_BUILD_ERROR' + | 'EXECUTION_ERROR' + | 'EVALUATION_ERROR' + | 'QUOTA_EXCEEDED' + | 'UNKNOWN_ERROR'; + +/** + * Structured error for scenario execution + */ +export class ScenarioExecutionError extends Error { + constructor( + message: string, + public readonly category: ScenarioErrorCategory, + public readonly context?: Record + ) { + super(message); + this.name = 'ScenarioExecutionError'; + } +} + +/** + * Structured logging for scenario execution + */ +function logScenarioEvent( + event: 'start' | 'tools_built' | 'execution_complete' | 'evaluation_complete' | 'error' | 'retry', + scenarioId: string, + data?: Record +): void { + const logEntry = { + timestamp: new Date().toISOString(), + event: `scenario.${event}`, + scenarioId, + ...data, + }; + + // In production, this could be sent to a logging service + // For now, we use structured console logging that can be parsed + if (process.env.NODE_ENV === 'development') { + console.log('[Scenario]', JSON.stringify(logEntry, null, 2)); + } +} + /** * Collection type with full tool relations */ @@ -156,6 +206,12 @@ export async function executeScenario( ): Promise { const { evaluatorModel = DEFAULT_EVALUATOR } = options; + logScenarioEvent('start', scenario.id, { + userId, + collectionId: scenario.collectionId, + evaluatorModel, + }); + // Create run record const run = await prisma.scenarioRun.create({ data: { @@ -172,6 +228,13 @@ export async function executeScenario( for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { retryCount = attempt; + if (attempt > 0) { + logScenarioEvent('retry', scenario.id, { + attempt, + previousError: lastError?.message, + }); + } + try { // Update status to running await prisma.scenarioRun.update({ @@ -187,12 +250,21 @@ export async function executeScenario( const executionResult = await executeWithAgent(scenario); // Evaluate with LLM - const evaluation = await evaluateScenarioRun( - scenario.prompt, - executionResult.output, - executionResult.conversation, - evaluatorModel - ); + let evaluation: EvaluationResult; + try { + evaluation = await evaluateScenarioRun( + scenario.prompt, + executionResult.output, + executionResult.conversation, + evaluatorModel + ); + } catch (evalError) { + throw new ScenarioExecutionError( + `Evaluation failed: ${evalError instanceof Error ? evalError.message : 'Unknown error'}`, + 'EVALUATION_ERROR', + { scenarioId: scenario.id, evaluatorModel, error: String(evalError) } + ); + } // Run assertions if defined const assertions = scenario.assertions as { @@ -206,6 +278,14 @@ export async function executeScenario( // Determine final verdict const finalStatus = determineFinalVerdict(evaluation, assertionResults); + logScenarioEvent('evaluation_complete', scenario.id, { + verdict: finalStatus, + evaluatorVerdict: evaluation.verdict, + evaluatorConfidence: evaluation.confidence, + assertionsPassed: assertionResults?.passed.length ?? 0, + assertionsFailed: assertionResults?.failed.length ?? 0, + }); + // Update run record const updatedRun = await prisma.scenarioRun.update({ where: { id: run.id }, @@ -234,12 +314,29 @@ export async function executeScenario( // If this is the last attempt, mark as error if (attempt === MAX_RETRIES) { + // Extract error category if it's a ScenarioExecutionError + const errorCategory = + error instanceof ScenarioExecutionError ? error.category : 'UNKNOWN_ERROR'; + const errorContext = error instanceof ScenarioExecutionError ? error.context : undefined; + + logScenarioEvent('error', scenario.id, { + category: errorCategory, + message: lastError.message, + context: errorContext, + retryCount, + }); + const errorRun = await prisma.scenarioRun.update({ where: { id: run.id }, data: { status: 'error', retryCount, - errorLog: (error as Error).stack || (error as Error).message, + errorLog: JSON.stringify({ + message: lastError.message, + category: errorCategory, + context: errorContext, + stack: lastError.stack, + }), completedAt: new Date(), }, }); @@ -353,23 +450,45 @@ async function executeWithAgent(scenario: Scenario): Promise<{ // Verify scenario has a collection if (!scenario.collectionId) { - throw new Error('Scenario has no associated collection'); + throw new ScenarioExecutionError('Scenario has no associated collection', 'NO_COLLECTION', { + scenarioId: scenario.id, + }); } // Fetch collection with tools const collection = await fetchCollectionWithTools(scenario.collectionId); if (!collection) { - throw new Error(`Collection not found: ${scenario.collectionId}`); + throw new ScenarioExecutionError( + `Collection not found: ${scenario.collectionId}`, + 'COLLECTION_NOT_FOUND', + { scenarioId: scenario.id, collectionId: scenario.collectionId } + ); } if (collection.tools.length === 0) { - throw new Error('Collection has no tools configured'); + throw new ScenarioExecutionError('Collection has no tools configured', 'NO_TOOLS', { + scenarioId: scenario.id, + collectionId: collection.id, + collectionName: collection.name, + }); } // Build tools from collection - const tools = buildCollectionTools(collection); + let tools: Record>; + try { + tools = buildCollectionTools(collection); + } catch (err) { + throw new ScenarioExecutionError( + `Failed to build tools: ${err instanceof Error ? err.message : 'Unknown error'}`, + 'TOOL_BUILD_ERROR', + { scenarioId: scenario.id, collectionId: collection.id, error: String(err) } + ); + } - console.log('[executeWithAgent] Executing scenario with tools:', Object.keys(tools)); + logScenarioEvent('tools_built', scenario.id, { + toolCount: Object.keys(tools).length, + toolNames: Object.keys(tools), + }); // Build system prompt for scenario execution const systemPrompt = `You are an AI assistant tasked with completing the following scenario using the available tools. @@ -381,15 +500,30 @@ Available tools: ${Object.keys(tools).join(', ')} Complete the user's task to the best of your ability using the tools provided.`; // Execute with generateText and multi-step tool loop - const result = await generateText({ - model: openai(DEFAULT_MODEL), - messages: [ - { role: 'system', content: systemPrompt }, - { role: 'user', content: scenario.prompt }, - ], - tools, - stopWhen: stepCountIs(MAX_TOOL_STEPS), - }); + const result = await (async () => { + try { + return await generateText({ + model: openai(DEFAULT_MODEL), + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: scenario.prompt }, + ], + tools, + stopWhen: stepCountIs(MAX_TOOL_STEPS), + }); + } catch (err) { + throw new ScenarioExecutionError( + `AI execution failed: ${err instanceof Error ? err.message : 'Unknown error'}`, + 'EXECUTION_ERROR', + { + scenarioId: scenario.id, + error: String(err), + model: DEFAULT_MODEL, + maxSteps: MAX_TOOL_STEPS, + } + ); + } + })(); // Build conversation history from response const conversation: unknown[] = [{ role: 'user', content: scenario.prompt }]; @@ -435,13 +569,13 @@ Complete the user's task to the best of your ability using the tools provided.`; // Get token usage const usage = result.usage || { inputTokens: 0, outputTokens: 0, totalTokens: 0 }; - console.log( - '[executeWithAgent] Completed in', + logScenarioEvent('execution_complete', scenario.id, { durationMs, - 'ms with', - result.steps?.length || 0, - 'steps' - ); + stepCount: result.steps?.length || 0, + inputTokens: usage.inputTokens ?? 0, + outputTokens: usage.outputTokens ?? 0, + hasOutput: !!result.text, + }); return { output: result.text || '[No output generated]', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 60eceb2..108faf7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -303,6 +303,12 @@ importers: ai: specifier: 6.0.49 version: 6.0.49(zod@4.3.5) + ajv: + specifier: ^8.17.1 + version: 8.17.1 + ajv-formats: + specifier: ^3.0.1 + version: 3.0.1(ajv@8.17.1) better-auth: specifier: ^1.4.10 version: 1.4.10(@prisma/client@6.19.1(prisma@6.19.1(typescript@5.9.3))(typescript@5.9.3))(better-sqlite3@12.5.0)(drizzle-kit@0.31.8)(drizzle-orm@0.45.1(@electric-sql/pglite@0.3.2)(@opentelemetry/api@1.9.0)(@prisma/client@6.19.1(prisma@6.19.1(typescript@5.9.3))(typescript@5.9.3))(@types/better-sqlite3@7.6.13)(@upstash/redis@1.36.1)(better-sqlite3@12.5.0)(gel@2.2.0)(kysely@0.28.9)(mysql2@3.15.3)(pg@8.16.3)(postgres@3.4.7)(prisma@6.19.1(typescript@5.9.3)))(mongodb@6.20.0)(mysql2@3.15.3)(next@16.1.1(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(pg@8.16.3)(prisma@6.19.1(typescript@5.9.3))(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(vitest@4.0.16(@opentelemetry/api@1.9.0)(@types/node@25.0.3)(happy-dom@20.1.0)(jiti@2.6.1)(jsdom@27.4.0)(lightningcss@1.30.2)(msw@2.12.7(@types/node@25.0.3)(typescript@5.9.3))(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)) @@ -7413,6 +7419,7 @@ packages: '@vercel/kv@3.0.0': resolution: {integrity: sha512-pKT8fRnfyYk2MgvyB6fn6ipJPCdfZwiKDdw7vB+HL50rjboEBHDVBEcnwfkEpVSp2AjNtoaOUH7zG+bVC/rvSg==} engines: {node: '>=14.6'} + deprecated: 'Vercel KV is deprecated. If you had an existing KV store, it should have moved to Upstash Redis which you will see under Vercel Integrations. For new projects, install a Redis integration from Vercel Marketplace: https://vercel.com/marketplace?category=storage&search=redis' '@vercel/oidc@3.1.0': resolution: {integrity: sha512-Fw28YZpRnA3cAHHDlkt7xQHiJ0fcL+NRcIqsocZQUSmbzeIKRpwttJjik5ZGanXP+vlA4SbTg+AbA3bP363l+w==} @@ -15583,14 +15590,14 @@ snapshots: '@remotion/media-parser': 4.0.409 '@remotion/studio': 4.0.409(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@remotion/studio-shared': 4.0.409(react-dom@19.2.3(react@19.2.3))(react@19.2.3) - css-loader: 5.2.7(webpack@5.96.1(esbuild@0.25.0)) + css-loader: 5.2.7(webpack@5.96.1) esbuild: 0.25.0 react: 19.2.3 react-dom: 19.2.3(react@19.2.3) react-refresh: 0.9.0 remotion: 4.0.409(react-dom@19.2.3(react@19.2.3))(react@19.2.3) source-map: 0.7.3 - style-loader: 4.0.0(webpack@5.96.1(esbuild@0.25.0)) + style-loader: 4.0.0(webpack@5.96.1) webpack: 5.96.1(esbuild@0.25.0) transitivePeerDependencies: - '@swc/core' @@ -17968,7 +17975,7 @@ snapshots: shebang-command: 2.0.0 which: 2.0.2 - css-loader@5.2.7(webpack@5.96.1(esbuild@0.25.0)): + css-loader@5.2.7(webpack@5.96.1): dependencies: icss-utils: 5.1.0(postcss@8.5.6) loader-utils: 2.0.4 @@ -22802,7 +22809,7 @@ snapshots: stubborn-utils@1.0.2: {} - style-loader@4.0.0(webpack@5.96.1(esbuild@0.25.0)): + style-loader@4.0.0(webpack@5.96.1): dependencies: webpack: 5.96.1(esbuild@0.25.0)