feat(scenarios): complete scenario evaluation system with JSON Schema validation
- Add JSON Schema validation using Ajv for structured output assertions - Add extractJsonFromOutput() to parse JSON from various formats (direct, markdown, embedded) - Add validateJsonSchema() for proper schema validation with error messages - Implement structured error handling with ScenarioExecutionError class - Add 8 error categories: COLLECTION_NOT_FOUND, NO_COLLECTION, NO_TOOLS, etc. - Add comprehensive unit tests for execute.ts (16 tests) - Expand evaluate.test.ts with JSON Schema validation tests (36 tests total) - Refactor ExpandedRunDetails.tsx into smaller components for assertions and conversation display
This commit is contained in:
Ajax Davis
parent
13e4fd954d
commit
2df4b53354
7 changed files with 1122 additions and 104 deletions
|
|
@ -45,6 +45,8 @@
|
|||
"@vercel/blob": "^2.0.0",
|
||||
"@vercel/kv": "^3.0.0",
|
||||
"ai": "6.0.49",
|
||||
"ajv": "^8.17.1",
|
||||
"ajv-formats": "^3.0.1",
|
||||
"better-auth": "^1.4.10",
|
||||
"bm25": "^0.1.1",
|
||||
"d3": "^7.9.0",
|
||||
|
|
|
|||
|
|
@ -1,10 +1,130 @@
|
|||
import { Badge } from '@tpmjs/ui/Badge/Badge';
|
||||
import { Icon } from '@tpmjs/ui/Icon/Icon';
|
||||
import type { ScenarioRun } from './page';
|
||||
|
||||
interface ExpandedRunDetailsProps {
|
||||
run: ScenarioRun;
|
||||
}
|
||||
|
||||
/** Displays passed and failed assertions */
|
||||
function AssertionsSection({ assertions }: { assertions: { passed: string[]; failed: string[] } }) {
|
||||
return (
|
||||
<div className="mt-4">
|
||||
<h4 className="text-xs font-semibold text-foreground-secondary uppercase tracking-wide mb-2">
|
||||
Assertions
|
||||
</h4>
|
||||
<div className="p-3 bg-surface-secondary rounded-lg space-y-3">
|
||||
{assertions.passed.length > 0 && (
|
||||
<div>
|
||||
<div className="flex items-center gap-1.5 text-success text-sm font-medium mb-1.5">
|
||||
<Icon icon="check" className="w-4 h-4" />
|
||||
Passed ({assertions.passed.length})
|
||||
</div>
|
||||
<div className="space-y-1 ml-5">
|
||||
{assertions.passed.map((assertion) => (
|
||||
<div key={assertion} className="text-sm text-foreground-secondary font-mono">
|
||||
{assertion}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{assertions.failed.length > 0 && (
|
||||
<div>
|
||||
<div className="flex items-center gap-1.5 text-error text-sm font-medium mb-1.5">
|
||||
<Icon icon="x" className="w-4 h-4" />
|
||||
Failed ({assertions.failed.length})
|
||||
</div>
|
||||
<div className="space-y-1 ml-5">
|
||||
{assertions.failed.map((assertion) => (
|
||||
<div key={assertion} className="text-sm text-error/80 font-mono">
|
||||
{assertion}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/** Displays conversation messages */
|
||||
function ConversationSection({
|
||||
conversation,
|
||||
}: {
|
||||
conversation: NonNullable<ScenarioRun['conversation']>;
|
||||
}) {
|
||||
return (
|
||||
<div className="mt-4">
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<h4 className="text-xs font-semibold text-foreground-secondary uppercase tracking-wide">
|
||||
Conversation History
|
||||
</h4>
|
||||
</div>
|
||||
<div className="space-y-4">
|
||||
{conversation.map((msg) => (
|
||||
<ConversationMessage key={msg.id} msg={msg} />
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/** Single conversation message */
|
||||
function ConversationMessage({ msg }: { msg: NonNullable<ScenarioRun['conversation']>[number] }) {
|
||||
if (msg.role === 'USER') {
|
||||
return (
|
||||
<div className="flex justify-end">
|
||||
<div className="max-w-[80%] rounded-lg p-4 bg-primary text-primary-foreground">
|
||||
<div className="text-sm whitespace-pre-wrap">{msg.content}</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (msg.role === 'ASSISTANT') {
|
||||
return msg.content ? (
|
||||
<div className="space-y-2">
|
||||
<div className="flex justify-start">
|
||||
<div className="max-w-[80%] rounded-lg p-4 bg-surface-secondary">
|
||||
<div className="text-sm prose prose-sm dark:prose-invert max-w-none">{msg.content}</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
) : null;
|
||||
}
|
||||
|
||||
if (msg.role === 'TOOL') {
|
||||
return (
|
||||
<div className="flex justify-start">
|
||||
<div className="max-w-[80%] rounded-lg border border-border bg-surface-secondary overflow-hidden">
|
||||
<div className="p-3">
|
||||
<div className="text-sm font-medium text-foreground">
|
||||
{msg.toolName || 'Unknown Tool'}
|
||||
</div>
|
||||
{msg.toolResult != null && (
|
||||
<div className="mt-2 pt-2 border-t border-border/50">
|
||||
<pre className="text-xs text-success overflow-x-auto whitespace-pre-wrap break-all">
|
||||
{typeof msg.toolResult === 'string'
|
||||
? msg.toolResult
|
||||
: JSON.stringify(msg.toolResult, null, 2)}
|
||||
</pre>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function ExpandedRunDetails({ run }: ExpandedRunDetailsProps) {
|
||||
const hasAssertions =
|
||||
run.assertions && (run.assertions.passed.length > 0 || run.assertions.failed.length > 0);
|
||||
|
||||
return (
|
||||
<div className="px-4 pb-4 border-t border-border/50">
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-4 mt-4">
|
||||
|
|
@ -15,7 +135,24 @@ export function ExpandedRunDetails({ run }: ExpandedRunDetailsProps) {
|
|||
LLM Evaluation
|
||||
</h4>
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
{run.evaluator.model && <span className="text-xs">{run.evaluator.model}</span>}
|
||||
<Badge
|
||||
className={
|
||||
run.evaluator.verdict === 'pass'
|
||||
? 'bg-success/10 text-success border-success/20'
|
||||
: 'bg-error/10 text-error border-error/20'
|
||||
}
|
||||
>
|
||||
<Icon
|
||||
icon={run.evaluator.verdict === 'pass' ? 'check' : 'x'}
|
||||
className="w-3 h-3 mr-1"
|
||||
/>
|
||||
{run.evaluator.verdict === 'pass' ? 'Pass' : 'Fail'}
|
||||
</Badge>
|
||||
{run.evaluator.model && (
|
||||
<Badge variant="secondary" size="sm">
|
||||
{run.evaluator.model}
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
{run.evaluator?.reason && (
|
||||
<p className="text-sm text-foreground-secondary">{run.evaluator.reason}</p>
|
||||
|
|
@ -55,6 +192,9 @@ export function ExpandedRunDetails({ run }: ExpandedRunDetailsProps) {
|
|||
</div>
|
||||
</div>
|
||||
|
||||
{/* Assertions Results */}
|
||||
{hasAssertions && run.assertions && <AssertionsSection assertions={run.assertions} />}
|
||||
|
||||
{/* Output (if owner) */}
|
||||
{run.output && (
|
||||
<div className="mt-4">
|
||||
|
|
@ -80,61 +220,7 @@ export function ExpandedRunDetails({ run }: ExpandedRunDetailsProps) {
|
|||
)}
|
||||
|
||||
{/* Conversation History */}
|
||||
{run.conversation && (
|
||||
<div className="mt-4">
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<h4 className="text-xs font-semibold text-foreground-secondary uppercase tracking-wide">
|
||||
Conversation History
|
||||
</h4>
|
||||
</div>
|
||||
<div className="space-y-4">
|
||||
{run.conversation.map((msg) => (
|
||||
<div key={msg.id}>
|
||||
{msg.role === 'USER' && (
|
||||
<div className="flex justify-end">
|
||||
<div className="max-w-[80%] rounded-lg p-4 bg-primary text-primary-foreground">
|
||||
<div className="text-sm whitespace-pre-wrap">{msg.content}</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{msg.role === 'ASSISTANT' && (
|
||||
<div className="space-y-2">
|
||||
{msg.content && (
|
||||
<div className="flex justify-start">
|
||||
<div className="max-w-[80%] rounded-lg p-4 bg-surface-secondary">
|
||||
<div className="text-sm prose prose-sm dark:prose-invert max-w-none">
|
||||
{msg.content}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{msg.role === 'TOOL' && (
|
||||
<div className="flex justify-start">
|
||||
<div className="max-w-[80%] rounded-lg border border-border bg-surface-secondary overflow-hidden">
|
||||
<div className="p-3">
|
||||
<div className="text-sm font-medium text-foreground">
|
||||
{msg.toolName || 'Unknown Tool'}
|
||||
</div>
|
||||
{msg.toolResult != null && (
|
||||
<div className="mt-2 pt-2 border-t border-border/50">
|
||||
<pre className="text-xs text-success overflow-x-auto whitespace-pre-wrap break-all">
|
||||
{typeof msg.toolResult === 'string'
|
||||
? msg.toolResult
|
||||
: JSON.stringify(msg.toolResult, null, 2)}
|
||||
</pre>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{run.conversation && <ConversationSection conversation={run.conversation} />}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,13 +3,20 @@
|
|||
*
|
||||
* Tests for the evaluation logic including:
|
||||
* - Regex assertion matching
|
||||
* - JSON Schema validation
|
||||
* - JSON extraction from various formats
|
||||
* - Final verdict determination
|
||||
* - Assertion result handling
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { determineFinalVerdict, runAssertions } from './evaluate';
|
||||
import {
|
||||
determineFinalVerdict,
|
||||
extractJsonFromOutput,
|
||||
runAssertions,
|
||||
validateJsonSchema,
|
||||
} from './evaluate';
|
||||
|
||||
describe('runAssertions', () => {
|
||||
describe('regex assertions', () => {
|
||||
|
|
@ -93,29 +100,262 @@ describe('runAssertions', () => {
|
|||
});
|
||||
|
||||
describe('schema assertions', () => {
|
||||
it('should note when schema is provided', () => {
|
||||
const output = '{"name": "test"}';
|
||||
const assertions = { schema: { type: 'object' } };
|
||||
it('should pass when JSON validates against schema', () => {
|
||||
const output = '{"name": "test", "age": 25}';
|
||||
const assertions = {
|
||||
schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
name: { type: 'string' },
|
||||
age: { type: 'number' },
|
||||
},
|
||||
required: ['name'],
|
||||
},
|
||||
};
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toContain('schema:provided (validation pending)');
|
||||
expect(result.passed).toContain('schema: JSON validates against schema');
|
||||
expect(result.failed).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('combined assertions', () => {
|
||||
it('should handle both regex and schema assertions', () => {
|
||||
const output = '{"status": "success"}';
|
||||
it('should fail when JSON does not validate against schema', () => {
|
||||
const output = '{"name": 123}';
|
||||
const assertions = {
|
||||
schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
name: { type: 'string' },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.failed.length).toBeGreaterThan(0);
|
||||
expect(result.failed[0]).toContain('schema:');
|
||||
expect(result.passed).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should fail when output is not valid JSON', () => {
|
||||
const output = 'This is not JSON at all';
|
||||
const assertions = {
|
||||
regex: ['success'],
|
||||
schema: { type: 'object' },
|
||||
};
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toContain('regex:success');
|
||||
expect(result.passed).toContain('schema:provided (validation pending)');
|
||||
expect(result.failed).toContain('schema: Output does not contain valid JSON');
|
||||
});
|
||||
|
||||
it('should handle schema with required fields', () => {
|
||||
const output = '{"name": "test"}';
|
||||
const assertions = {
|
||||
schema: {
|
||||
type: 'object',
|
||||
required: ['name', 'email'],
|
||||
},
|
||||
};
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.failed.length).toBeGreaterThan(0);
|
||||
expect(result.failed[0]).toContain('schema:');
|
||||
});
|
||||
|
||||
it('should handle empty schema object', () => {
|
||||
const output = '{"anything": "goes"}';
|
||||
const assertions = { schema: {} };
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
// Empty schema should not run validation
|
||||
expect(result.passed).toHaveLength(0);
|
||||
expect(result.failed).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('combined assertions', () => {
|
||||
it('should handle both regex and schema assertions passing', () => {
|
||||
const output = '{"status": "success", "count": 42}';
|
||||
const assertions = {
|
||||
regex: ['success', '42'],
|
||||
schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
status: { type: 'string' },
|
||||
count: { type: 'number' },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toContain('regex:success');
|
||||
expect(result.passed).toContain('regex:42');
|
||||
expect(result.passed).toContain('schema: JSON validates against schema');
|
||||
expect(result.failed).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should handle regex passing but schema failing', () => {
|
||||
const output = '{"status": "success", "count": "not a number"}';
|
||||
const assertions = {
|
||||
regex: ['success'],
|
||||
schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
count: { type: 'number' },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toContain('regex:success');
|
||||
expect(result.failed.length).toBeGreaterThan(0);
|
||||
expect(result.failed[0]).toContain('schema:');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractJsonFromOutput', () => {
|
||||
it('should extract direct JSON object', () => {
|
||||
const output = '{"name": "test"}';
|
||||
const result = extractJsonFromOutput(output);
|
||||
expect(result).toEqual({ name: 'test' });
|
||||
});
|
||||
|
||||
it('should extract direct JSON array', () => {
|
||||
const output = '[1, 2, 3]';
|
||||
const result = extractJsonFromOutput(output);
|
||||
expect(result).toEqual([1, 2, 3]);
|
||||
});
|
||||
|
||||
it('should extract JSON from markdown code block with json tag', () => {
|
||||
const output = 'Here is the result:\n```json\n{"status": "ok"}\n```';
|
||||
const result = extractJsonFromOutput(output);
|
||||
expect(result).toEqual({ status: 'ok' });
|
||||
});
|
||||
|
||||
it('should extract JSON from markdown code block without tag', () => {
|
||||
const output = 'Result:\n```\n{"value": 42}\n```\nDone.';
|
||||
const result = extractJsonFromOutput(output);
|
||||
expect(result).toEqual({ value: 42 });
|
||||
});
|
||||
|
||||
it('should extract JSON embedded in text', () => {
|
||||
const output = 'The response is {"data": "found"} as expected.';
|
||||
const result = extractJsonFromOutput(output);
|
||||
expect(result).toEqual({ data: 'found' });
|
||||
});
|
||||
|
||||
it('should return null for non-JSON output', () => {
|
||||
const output = 'This is just plain text with no JSON.';
|
||||
const result = extractJsonFromOutput(output);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it('should handle whitespace around JSON', () => {
|
||||
const output = ' \n {"trimmed": true} \n ';
|
||||
const result = extractJsonFromOutput(output);
|
||||
expect(result).toEqual({ trimmed: true });
|
||||
});
|
||||
|
||||
it('should handle nested JSON objects', () => {
|
||||
const output = '{"outer": {"inner": {"deep": "value"}}}';
|
||||
const result = extractJsonFromOutput(output);
|
||||
expect(result).toEqual({ outer: { inner: { deep: 'value' } } });
|
||||
});
|
||||
});
|
||||
|
||||
describe('validateJsonSchema', () => {
|
||||
it('should validate simple object schema', () => {
|
||||
const data = { name: 'test', age: 25 };
|
||||
const schema = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
name: { type: 'string' },
|
||||
age: { type: 'number' },
|
||||
},
|
||||
};
|
||||
|
||||
const result = validateJsonSchema(data, schema);
|
||||
expect(result.valid).toBe(true);
|
||||
expect(result.errors).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should return errors for invalid data', () => {
|
||||
const data = { name: 123 };
|
||||
const schema = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
name: { type: 'string' },
|
||||
},
|
||||
};
|
||||
|
||||
const result = validateJsonSchema(data, schema);
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.errors.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should validate required fields', () => {
|
||||
const data = { name: 'test' };
|
||||
const schema = {
|
||||
type: 'object',
|
||||
required: ['name', 'email'],
|
||||
};
|
||||
|
||||
const result = validateJsonSchema(data, schema);
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.errors.some((e) => e.includes('email'))).toBe(true);
|
||||
});
|
||||
|
||||
it('should validate array schemas', () => {
|
||||
const data = [1, 2, 3];
|
||||
const schema = {
|
||||
type: 'array',
|
||||
items: { type: 'number' },
|
||||
};
|
||||
|
||||
const result = validateJsonSchema(data, schema);
|
||||
expect(result.valid).toBe(true);
|
||||
});
|
||||
|
||||
it('should handle format validation (email)', () => {
|
||||
const data = { email: 'invalid-email' };
|
||||
const schema = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
email: { type: 'string', format: 'email' },
|
||||
},
|
||||
};
|
||||
|
||||
const result = validateJsonSchema(data, schema);
|
||||
expect(result.valid).toBe(false);
|
||||
});
|
||||
|
||||
it('should pass format validation for valid email', () => {
|
||||
const data = { email: 'test@example.com' };
|
||||
const schema = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
email: { type: 'string', format: 'email' },
|
||||
},
|
||||
};
|
||||
|
||||
const result = validateJsonSchema(data, schema);
|
||||
expect(result.valid).toBe(true);
|
||||
});
|
||||
|
||||
it('should handle invalid schema gracefully', () => {
|
||||
const data = { test: true };
|
||||
const schema = {
|
||||
type: 'invalid-type-that-does-not-exist',
|
||||
};
|
||||
|
||||
const result = validateJsonSchema(data, schema as Record<string, unknown>);
|
||||
// Ajv with strict: false will still try to validate
|
||||
expect(result.valid).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -2,13 +2,20 @@
|
|||
* Scenario Evaluation Service
|
||||
*
|
||||
* Uses LLM judgment to evaluate if a scenario execution was successful.
|
||||
* Supports both regex pattern matching and JSON Schema validation for assertions.
|
||||
*/
|
||||
|
||||
import { anthropic } from '@ai-sdk/anthropic';
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import { generateObject } from 'ai';
|
||||
import Ajv from 'ajv';
|
||||
import addFormats from 'ajv-formats';
|
||||
import { z } from 'zod';
|
||||
|
||||
// Initialize Ajv with common formats (email, uri, date-time, etc.)
|
||||
const ajv = new Ajv({ allErrors: true, strict: false });
|
||||
addFormats(ajv);
|
||||
|
||||
const EvaluationSchema = z.object({
|
||||
verdict: z.enum(['pass', 'fail']).describe('Whether the scenario was completed successfully'),
|
||||
reason: z.string().describe('Brief explanation of why the scenario passed or failed'),
|
||||
|
|
@ -94,9 +101,90 @@ Provide your verdict, a brief reason, and your confidence level.`,
|
|||
return object;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract JSON from text output
|
||||
*
|
||||
* Attempts to find and parse JSON from various formats:
|
||||
* - Direct JSON
|
||||
* - JSON wrapped in markdown code blocks
|
||||
* - JSON embedded in text
|
||||
*
|
||||
* @param output The text to extract JSON from
|
||||
* @returns Parsed JSON or null if not found
|
||||
*/
|
||||
export function extractJsonFromOutput(output: string): unknown | null {
|
||||
// Try direct parse first
|
||||
try {
|
||||
return JSON.parse(output.trim());
|
||||
} catch {
|
||||
// Continue to other strategies
|
||||
}
|
||||
|
||||
// Try extracting from markdown code blocks (```json ... ``` or ``` ... ```)
|
||||
const codeBlockMatch = output.match(/```(?:json)?\s*([\s\S]*?)```/);
|
||||
if (codeBlockMatch?.[1]) {
|
||||
try {
|
||||
return JSON.parse(codeBlockMatch[1].trim());
|
||||
} catch {
|
||||
// Continue to other strategies
|
||||
}
|
||||
}
|
||||
|
||||
// Try finding JSON object or array in the text
|
||||
const jsonMatch = output.match(/(\{[\s\S]*\}|\[[\s\S]*\])/);
|
||||
if (jsonMatch?.[1]) {
|
||||
try {
|
||||
return JSON.parse(jsonMatch[1]);
|
||||
} catch {
|
||||
// Could not parse as JSON
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate data against a JSON Schema
|
||||
*
|
||||
* @param data The data to validate
|
||||
* @param schema The JSON Schema to validate against
|
||||
* @returns Validation result with errors if any
|
||||
*/
|
||||
export function validateJsonSchema(
|
||||
data: unknown,
|
||||
schema: Record<string, unknown>
|
||||
): { valid: boolean; errors: string[] } {
|
||||
try {
|
||||
const validate = ajv.compile(schema);
|
||||
const valid = validate(data);
|
||||
|
||||
if (valid) {
|
||||
return { valid: true, errors: [] };
|
||||
}
|
||||
|
||||
// Format errors into readable strings
|
||||
const errors = (validate.errors || []).map((err) => {
|
||||
const path = err.instancePath || 'root';
|
||||
const message = err.message || 'Unknown error';
|
||||
return `${path}: ${message}`;
|
||||
});
|
||||
|
||||
return { valid: false, errors };
|
||||
} catch (err) {
|
||||
return {
|
||||
valid: false,
|
||||
errors: [`Schema compilation error: ${err instanceof Error ? err.message : 'Unknown error'}`],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run assertions against the output
|
||||
*
|
||||
* Supports two types of assertions:
|
||||
* - regex: Array of regex patterns that must match the output
|
||||
* - schema: JSON Schema that the output (parsed as JSON) must validate against
|
||||
*
|
||||
* @param output The agent output to check
|
||||
* @param assertions The assertions to run
|
||||
*/
|
||||
|
|
@ -123,11 +211,25 @@ export function runAssertions(
|
|||
}
|
||||
}
|
||||
|
||||
// Schema assertions would require more complex validation
|
||||
// For now, we'll just note if schema was provided
|
||||
if (assertions.schema) {
|
||||
// TODO: Implement JSON schema validation against parsed output
|
||||
passed.push('schema:provided (validation pending)');
|
||||
// Validate against JSON Schema if provided
|
||||
if (assertions.schema && Object.keys(assertions.schema).length > 0) {
|
||||
const extractedJson = extractJsonFromOutput(output);
|
||||
|
||||
if (extractedJson === null) {
|
||||
failed.push('schema: Output does not contain valid JSON');
|
||||
} else {
|
||||
const validation = validateJsonSchema(extractedJson, assertions.schema);
|
||||
|
||||
if (validation.valid) {
|
||||
passed.push('schema: JSON validates against schema');
|
||||
} else {
|
||||
// Include first 3 errors for clarity
|
||||
const errorSummary = validation.errors.slice(0, 3).join('; ');
|
||||
const moreErrors =
|
||||
validation.errors.length > 3 ? ` (+${validation.errors.length - 3} more)` : '';
|
||||
failed.push(`schema: ${errorSummary}${moreErrors}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { passed, failed };
|
||||
|
|
|
|||
447
apps/web/src/lib/scenarios/execute.test.ts
Normal file
447
apps/web/src/lib/scenarios/execute.test.ts
Normal file
|
|
@ -0,0 +1,447 @@
|
|||
/**
|
||||
* Scenario Execution Unit Tests
|
||||
*
|
||||
* Tests for the execution logic including:
|
||||
* - Quota management
|
||||
* - Quality score metrics
|
||||
* - Error handling
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { ScenarioExecutionError } from './execute';
|
||||
|
||||
// Mock prisma
|
||||
vi.mock('@tpmjs/db', () => ({
|
||||
prisma: {
|
||||
scenarioQuota: {
|
||||
findUnique: vi.fn(),
|
||||
create: vi.fn(),
|
||||
update: vi.fn(),
|
||||
},
|
||||
scenario: {
|
||||
findUnique: vi.fn(),
|
||||
update: vi.fn(),
|
||||
},
|
||||
scenarioRun: {
|
||||
create: vi.fn(),
|
||||
update: vi.fn(),
|
||||
},
|
||||
collection: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
describe('ScenarioExecutionError', () => {
|
||||
it('should create error with category', () => {
|
||||
const error = new ScenarioExecutionError('Test error message', 'COLLECTION_NOT_FOUND', {
|
||||
scenarioId: 'test-123',
|
||||
});
|
||||
|
||||
expect(error.message).toBe('Test error message');
|
||||
expect(error.category).toBe('COLLECTION_NOT_FOUND');
|
||||
expect(error.context).toEqual({ scenarioId: 'test-123' });
|
||||
expect(error.name).toBe('ScenarioExecutionError');
|
||||
});
|
||||
|
||||
it('should work without context', () => {
|
||||
const error = new ScenarioExecutionError('No context error', 'NO_TOOLS');
|
||||
|
||||
expect(error.message).toBe('No context error');
|
||||
expect(error.category).toBe('NO_TOOLS');
|
||||
expect(error.context).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should be an instance of Error', () => {
|
||||
const error = new ScenarioExecutionError('Test', 'UNKNOWN_ERROR');
|
||||
expect(error instanceof Error).toBe(true);
|
||||
});
|
||||
|
||||
it('should support all error categories', () => {
|
||||
const categories = [
|
||||
'COLLECTION_NOT_FOUND',
|
||||
'NO_COLLECTION',
|
||||
'NO_TOOLS',
|
||||
'TOOL_BUILD_ERROR',
|
||||
'EXECUTION_ERROR',
|
||||
'EVALUATION_ERROR',
|
||||
'QUOTA_EXCEEDED',
|
||||
'UNKNOWN_ERROR',
|
||||
] as const;
|
||||
|
||||
for (const category of categories) {
|
||||
const error = new ScenarioExecutionError(`Error: ${category}`, category);
|
||||
expect(error.category).toBe(category);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Quota Management', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date('2025-01-15T12:00:00Z'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('checkAndDecrementQuota', () => {
|
||||
it('should create quota for new user', async () => {
|
||||
const { prisma } = await import('@tpmjs/db');
|
||||
const { checkAndDecrementQuota } = await import('./execute');
|
||||
|
||||
vi.mocked(prisma.scenarioQuota.findUnique).mockResolvedValue(null);
|
||||
vi.mocked(prisma.scenarioQuota.create).mockResolvedValue({
|
||||
id: 'quota-1',
|
||||
userId: 'user-1',
|
||||
dailyLimit: 50,
|
||||
dailyUsed: 0,
|
||||
lastResetAt: new Date('2025-01-15T00:00:00Z'),
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
vi.mocked(prisma.scenarioQuota.update).mockResolvedValue({
|
||||
id: 'quota-1',
|
||||
userId: 'user-1',
|
||||
dailyLimit: 50,
|
||||
dailyUsed: 1,
|
||||
lastResetAt: new Date('2025-01-15T00:00:00Z'),
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
|
||||
const result = await checkAndDecrementQuota('user-1');
|
||||
|
||||
expect(result.allowed).toBe(true);
|
||||
expect(result.remaining).toBe(49);
|
||||
expect(prisma.scenarioQuota.create).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should deny when quota exceeded', async () => {
|
||||
const { prisma } = await import('@tpmjs/db');
|
||||
const { checkAndDecrementQuota } = await import('./execute');
|
||||
|
||||
vi.mocked(prisma.scenarioQuota.findUnique).mockResolvedValue({
|
||||
id: 'quota-1',
|
||||
userId: 'user-1',
|
||||
dailyLimit: 50,
|
||||
dailyUsed: 50,
|
||||
lastResetAt: new Date('2025-01-15T00:00:00Z'),
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
|
||||
const result = await checkAndDecrementQuota('user-1');
|
||||
|
||||
expect(result.allowed).toBe(false);
|
||||
expect(result.remaining).toBe(0);
|
||||
});
|
||||
|
||||
it('should reset quota on new day', async () => {
|
||||
const { prisma } = await import('@tpmjs/db');
|
||||
const { checkAndDecrementQuota } = await import('./execute');
|
||||
|
||||
// Quota from yesterday
|
||||
vi.mocked(prisma.scenarioQuota.findUnique).mockResolvedValue({
|
||||
id: 'quota-1',
|
||||
userId: 'user-1',
|
||||
dailyLimit: 50,
|
||||
dailyUsed: 45,
|
||||
lastResetAt: new Date('2025-01-14T00:00:00Z'), // Yesterday
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
|
||||
// After reset
|
||||
vi.mocked(prisma.scenarioQuota.update).mockResolvedValue({
|
||||
id: 'quota-1',
|
||||
userId: 'user-1',
|
||||
dailyLimit: 50,
|
||||
dailyUsed: 0,
|
||||
lastResetAt: new Date('2025-01-15T00:00:00Z'),
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
|
||||
const result = await checkAndDecrementQuota('user-1');
|
||||
|
||||
expect(result.allowed).toBe(true);
|
||||
// After reset: 50 - 0 - 1 = 49
|
||||
expect(result.remaining).toBe(49);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getQuotaStatus', () => {
|
||||
it('should return default quota for new user', async () => {
|
||||
const { prisma } = await import('@tpmjs/db');
|
||||
const { getQuotaStatus } = await import('./execute');
|
||||
|
||||
vi.mocked(prisma.scenarioQuota.findUnique).mockResolvedValue(null);
|
||||
|
||||
const result = await getQuotaStatus('user-1');
|
||||
|
||||
expect(result.used).toBe(0);
|
||||
expect(result.limit).toBe(50);
|
||||
expect(result.remaining).toBe(50);
|
||||
});
|
||||
|
||||
it('should return current quota for existing user', async () => {
|
||||
const { prisma } = await import('@tpmjs/db');
|
||||
const { getQuotaStatus } = await import('./execute');
|
||||
|
||||
vi.mocked(prisma.scenarioQuota.findUnique).mockResolvedValue({
|
||||
id: 'quota-1',
|
||||
userId: 'user-1',
|
||||
dailyLimit: 50,
|
||||
dailyUsed: 10,
|
||||
lastResetAt: new Date('2025-01-15T00:00:00Z'),
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
|
||||
const result = await getQuotaStatus('user-1');
|
||||
|
||||
expect(result.used).toBe(10);
|
||||
expect(result.limit).toBe(50);
|
||||
expect(result.remaining).toBe(40);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Scenario Metrics', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('updateScenarioMetrics', () => {
|
||||
it('should increase quality score on pass', async () => {
|
||||
const { prisma } = await import('@tpmjs/db');
|
||||
const { updateScenarioMetrics } = await import('./execute');
|
||||
|
||||
vi.mocked(prisma.scenario.findUnique).mockResolvedValue({
|
||||
id: 'scenario-1',
|
||||
collectionId: 'col-1',
|
||||
prompt: 'Test',
|
||||
name: 'Test',
|
||||
description: null,
|
||||
tags: [],
|
||||
assertions: null,
|
||||
qualityScore: 0.5,
|
||||
consecutivePasses: 0,
|
||||
consecutiveFails: 2,
|
||||
totalRuns: 5,
|
||||
lastRunAt: null,
|
||||
lastRunStatus: 'fail',
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
|
||||
vi.mocked(prisma.scenario.update).mockResolvedValue({} as any);
|
||||
|
||||
await updateScenarioMetrics('scenario-1', 'pass');
|
||||
|
||||
expect(prisma.scenario.update).toHaveBeenCalledWith({
|
||||
where: { id: 'scenario-1' },
|
||||
data: expect.objectContaining({
|
||||
consecutivePasses: 1,
|
||||
consecutiveFails: 0,
|
||||
totalRuns: 6,
|
||||
lastRunStatus: 'pass',
|
||||
}),
|
||||
});
|
||||
|
||||
// Check quality score increased
|
||||
const updateCall = vi.mocked(prisma.scenario.update).mock.calls[0]?.[0];
|
||||
expect(updateCall?.data.qualityScore).toBeGreaterThan(0.5);
|
||||
});
|
||||
|
||||
it('should decrease quality score on fail', async () => {
|
||||
const { prisma } = await import('@tpmjs/db');
|
||||
const { updateScenarioMetrics } = await import('./execute');
|
||||
|
||||
vi.mocked(prisma.scenario.findUnique).mockResolvedValue({
|
||||
id: 'scenario-1',
|
||||
collectionId: 'col-1',
|
||||
prompt: 'Test',
|
||||
name: 'Test',
|
||||
description: null,
|
||||
tags: [],
|
||||
assertions: null,
|
||||
qualityScore: 0.5,
|
||||
consecutivePasses: 3,
|
||||
consecutiveFails: 0,
|
||||
totalRuns: 5,
|
||||
lastRunAt: null,
|
||||
lastRunStatus: 'pass',
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
|
||||
vi.mocked(prisma.scenario.update).mockResolvedValue({} as any);
|
||||
|
||||
await updateScenarioMetrics('scenario-1', 'fail');
|
||||
|
||||
expect(prisma.scenario.update).toHaveBeenCalledWith({
|
||||
where: { id: 'scenario-1' },
|
||||
data: expect.objectContaining({
|
||||
consecutivePasses: 0,
|
||||
consecutiveFails: 1,
|
||||
totalRuns: 6,
|
||||
lastRunStatus: 'fail',
|
||||
}),
|
||||
});
|
||||
|
||||
// Check quality score decreased
|
||||
const updateCall = vi.mocked(prisma.scenario.update).mock.calls[0]?.[0];
|
||||
expect(updateCall?.data.qualityScore).toBeLessThan(0.5);
|
||||
});
|
||||
|
||||
it('should not exceed quality score bounds', async () => {
|
||||
const { prisma } = await import('@tpmjs/db');
|
||||
const { updateScenarioMetrics } = await import('./execute');
|
||||
|
||||
// Test max bound
|
||||
vi.mocked(prisma.scenario.findUnique).mockResolvedValue({
|
||||
id: 'scenario-1',
|
||||
collectionId: 'col-1',
|
||||
prompt: 'Test',
|
||||
name: 'Test',
|
||||
description: null,
|
||||
tags: [],
|
||||
assertions: null,
|
||||
qualityScore: 0.99,
|
||||
consecutivePasses: 10,
|
||||
consecutiveFails: 0,
|
||||
totalRuns: 20,
|
||||
lastRunAt: null,
|
||||
lastRunStatus: 'pass',
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
|
||||
vi.mocked(prisma.scenario.update).mockResolvedValue({} as any);
|
||||
|
||||
await updateScenarioMetrics('scenario-1', 'pass');
|
||||
|
||||
const updateCall = vi.mocked(prisma.scenario.update).mock.calls[0]?.[0];
|
||||
expect(updateCall?.data.qualityScore).toBeLessThanOrEqual(1.0);
|
||||
});
|
||||
|
||||
it('should not go below zero', async () => {
|
||||
const { prisma } = await import('@tpmjs/db');
|
||||
const { updateScenarioMetrics } = await import('./execute');
|
||||
|
||||
// Test min bound
|
||||
vi.mocked(prisma.scenario.findUnique).mockResolvedValue({
|
||||
id: 'scenario-1',
|
||||
collectionId: 'col-1',
|
||||
prompt: 'Test',
|
||||
name: 'Test',
|
||||
description: null,
|
||||
tags: [],
|
||||
assertions: null,
|
||||
qualityScore: 0.05,
|
||||
consecutivePasses: 0,
|
||||
consecutiveFails: 5,
|
||||
totalRuns: 10,
|
||||
lastRunAt: null,
|
||||
lastRunStatus: 'fail',
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
|
||||
vi.mocked(prisma.scenario.update).mockResolvedValue({} as any);
|
||||
|
||||
await updateScenarioMetrics('scenario-1', 'fail');
|
||||
|
||||
const updateCall = vi.mocked(prisma.scenario.update).mock.calls[0]?.[0];
|
||||
expect(updateCall?.data.qualityScore).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
it('should handle non-existent scenario gracefully', async () => {
|
||||
const { prisma } = await import('@tpmjs/db');
|
||||
const { updateScenarioMetrics } = await import('./execute');
|
||||
|
||||
vi.mocked(prisma.scenario.findUnique).mockResolvedValue(null);
|
||||
|
||||
// Should not throw
|
||||
await expect(updateScenarioMetrics('non-existent', 'pass')).resolves.toBeUndefined();
|
||||
expect(prisma.scenario.update).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Quality Score Algorithm', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should reward consecutive passes with streak bonus', async () => {
|
||||
const { prisma } = await import('@tpmjs/db');
|
||||
const { updateScenarioMetrics } = await import('./execute');
|
||||
|
||||
// Starting with 3 consecutive passes
|
||||
vi.mocked(prisma.scenario.findUnique).mockResolvedValue({
|
||||
id: 'scenario-1',
|
||||
collectionId: 'col-1',
|
||||
prompt: 'Test',
|
||||
name: 'Test',
|
||||
description: null,
|
||||
tags: [],
|
||||
assertions: null,
|
||||
qualityScore: 0.3,
|
||||
consecutivePasses: 3,
|
||||
consecutiveFails: 0,
|
||||
totalRuns: 5,
|
||||
lastRunAt: null,
|
||||
lastRunStatus: 'pass',
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
|
||||
vi.mocked(prisma.scenario.update).mockResolvedValue({} as any);
|
||||
|
||||
await updateScenarioMetrics('scenario-1', 'pass');
|
||||
|
||||
const updateCall = vi.mocked(prisma.scenario.update).mock.calls[0]?.[0];
|
||||
// Base: 0.3, +0.05 for pass, +0.01*4 for streak bonus = 0.3 + 0.05 + 0.04 = 0.39
|
||||
expect(updateCall?.data.qualityScore).toBeCloseTo(0.39, 2);
|
||||
expect(updateCall?.data.consecutivePasses).toBe(4);
|
||||
});
|
||||
|
||||
it('should penalize consecutive fails with streak penalty', async () => {
|
||||
const { prisma } = await import('@tpmjs/db');
|
||||
const { updateScenarioMetrics } = await import('./execute');
|
||||
|
||||
// Starting with 2 consecutive fails
|
||||
vi.mocked(prisma.scenario.findUnique).mockResolvedValue({
|
||||
id: 'scenario-1',
|
||||
collectionId: 'col-1',
|
||||
prompt: 'Test',
|
||||
name: 'Test',
|
||||
description: null,
|
||||
tags: [],
|
||||
assertions: null,
|
||||
qualityScore: 0.5,
|
||||
consecutivePasses: 0,
|
||||
consecutiveFails: 2,
|
||||
totalRuns: 5,
|
||||
lastRunAt: null,
|
||||
lastRunStatus: 'fail',
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
|
||||
vi.mocked(prisma.scenario.update).mockResolvedValue({} as any);
|
||||
|
||||
await updateScenarioMetrics('scenario-1', 'fail');
|
||||
|
||||
const updateCall = vi.mocked(prisma.scenario.update).mock.calls[0]?.[0];
|
||||
// Base: 0.5, -0.1 for fail, -0.02*3 for streak penalty = 0.5 - 0.1 - 0.06 = 0.34
|
||||
expect(updateCall?.data.qualityScore).toBeCloseTo(0.34, 2);
|
||||
expect(updateCall?.data.consecutiveFails).toBe(3);
|
||||
});
|
||||
});
|
||||
|
|
@ -15,6 +15,7 @@ import { createToolDefinition } from '../ai-agent/tool-executor-agent';
|
|||
import { parseExecutorConfig, resolveExecutorConfig } from '../executors';
|
||||
import {
|
||||
determineFinalVerdict,
|
||||
type EvaluationResult,
|
||||
type EvaluatorModelId,
|
||||
evaluateScenarioRun,
|
||||
runAssertions,
|
||||
|
|
@ -25,6 +26,55 @@ const DEFAULT_MODEL = 'gpt-4.1-mini';
|
|||
const MAX_RETRIES = 1;
|
||||
const MAX_TOOL_STEPS = 10;
|
||||
|
||||
/**
|
||||
* Error categories for scenario execution
|
||||
*/
|
||||
export type ScenarioErrorCategory =
|
||||
| 'COLLECTION_NOT_FOUND'
|
||||
| 'NO_COLLECTION'
|
||||
| 'NO_TOOLS'
|
||||
| 'TOOL_BUILD_ERROR'
|
||||
| 'EXECUTION_ERROR'
|
||||
| 'EVALUATION_ERROR'
|
||||
| 'QUOTA_EXCEEDED'
|
||||
| 'UNKNOWN_ERROR';
|
||||
|
||||
/**
|
||||
* Structured error for scenario execution
|
||||
*/
|
||||
export class ScenarioExecutionError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly category: ScenarioErrorCategory,
|
||||
public readonly context?: Record<string, unknown>
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'ScenarioExecutionError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Structured logging for scenario execution
|
||||
*/
|
||||
function logScenarioEvent(
|
||||
event: 'start' | 'tools_built' | 'execution_complete' | 'evaluation_complete' | 'error' | 'retry',
|
||||
scenarioId: string,
|
||||
data?: Record<string, unknown>
|
||||
): void {
|
||||
const logEntry = {
|
||||
timestamp: new Date().toISOString(),
|
||||
event: `scenario.${event}`,
|
||||
scenarioId,
|
||||
...data,
|
||||
};
|
||||
|
||||
// In production, this could be sent to a logging service
|
||||
// For now, we use structured console logging that can be parsed
|
||||
if (process.env.NODE_ENV === 'development') {
|
||||
console.log('[Scenario]', JSON.stringify(logEntry, null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Collection type with full tool relations
|
||||
*/
|
||||
|
|
@ -156,6 +206,12 @@ export async function executeScenario(
|
|||
): Promise<ExecutionResult> {
|
||||
const { evaluatorModel = DEFAULT_EVALUATOR } = options;
|
||||
|
||||
logScenarioEvent('start', scenario.id, {
|
||||
userId,
|
||||
collectionId: scenario.collectionId,
|
||||
evaluatorModel,
|
||||
});
|
||||
|
||||
// Create run record
|
||||
const run = await prisma.scenarioRun.create({
|
||||
data: {
|
||||
|
|
@ -172,6 +228,13 @@ export async function executeScenario(
|
|||
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
||||
retryCount = attempt;
|
||||
|
||||
if (attempt > 0) {
|
||||
logScenarioEvent('retry', scenario.id, {
|
||||
attempt,
|
||||
previousError: lastError?.message,
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
// Update status to running
|
||||
await prisma.scenarioRun.update({
|
||||
|
|
@ -187,12 +250,21 @@ export async function executeScenario(
|
|||
const executionResult = await executeWithAgent(scenario);
|
||||
|
||||
// Evaluate with LLM
|
||||
const evaluation = await evaluateScenarioRun(
|
||||
scenario.prompt,
|
||||
executionResult.output,
|
||||
executionResult.conversation,
|
||||
evaluatorModel
|
||||
);
|
||||
let evaluation: EvaluationResult;
|
||||
try {
|
||||
evaluation = await evaluateScenarioRun(
|
||||
scenario.prompt,
|
||||
executionResult.output,
|
||||
executionResult.conversation,
|
||||
evaluatorModel
|
||||
);
|
||||
} catch (evalError) {
|
||||
throw new ScenarioExecutionError(
|
||||
`Evaluation failed: ${evalError instanceof Error ? evalError.message : 'Unknown error'}`,
|
||||
'EVALUATION_ERROR',
|
||||
{ scenarioId: scenario.id, evaluatorModel, error: String(evalError) }
|
||||
);
|
||||
}
|
||||
|
||||
// Run assertions if defined
|
||||
const assertions = scenario.assertions as {
|
||||
|
|
@ -206,6 +278,14 @@ export async function executeScenario(
|
|||
// Determine final verdict
|
||||
const finalStatus = determineFinalVerdict(evaluation, assertionResults);
|
||||
|
||||
logScenarioEvent('evaluation_complete', scenario.id, {
|
||||
verdict: finalStatus,
|
||||
evaluatorVerdict: evaluation.verdict,
|
||||
evaluatorConfidence: evaluation.confidence,
|
||||
assertionsPassed: assertionResults?.passed.length ?? 0,
|
||||
assertionsFailed: assertionResults?.failed.length ?? 0,
|
||||
});
|
||||
|
||||
// Update run record
|
||||
const updatedRun = await prisma.scenarioRun.update({
|
||||
where: { id: run.id },
|
||||
|
|
@ -234,12 +314,29 @@ export async function executeScenario(
|
|||
|
||||
// If this is the last attempt, mark as error
|
||||
if (attempt === MAX_RETRIES) {
|
||||
// Extract error category if it's a ScenarioExecutionError
|
||||
const errorCategory =
|
||||
error instanceof ScenarioExecutionError ? error.category : 'UNKNOWN_ERROR';
|
||||
const errorContext = error instanceof ScenarioExecutionError ? error.context : undefined;
|
||||
|
||||
logScenarioEvent('error', scenario.id, {
|
||||
category: errorCategory,
|
||||
message: lastError.message,
|
||||
context: errorContext,
|
||||
retryCount,
|
||||
});
|
||||
|
||||
const errorRun = await prisma.scenarioRun.update({
|
||||
where: { id: run.id },
|
||||
data: {
|
||||
status: 'error',
|
||||
retryCount,
|
||||
errorLog: (error as Error).stack || (error as Error).message,
|
||||
errorLog: JSON.stringify({
|
||||
message: lastError.message,
|
||||
category: errorCategory,
|
||||
context: errorContext,
|
||||
stack: lastError.stack,
|
||||
}),
|
||||
completedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
|
@ -353,23 +450,45 @@ async function executeWithAgent(scenario: Scenario): Promise<{
|
|||
|
||||
// Verify scenario has a collection
|
||||
if (!scenario.collectionId) {
|
||||
throw new Error('Scenario has no associated collection');
|
||||
throw new ScenarioExecutionError('Scenario has no associated collection', 'NO_COLLECTION', {
|
||||
scenarioId: scenario.id,
|
||||
});
|
||||
}
|
||||
|
||||
// Fetch collection with tools
|
||||
const collection = await fetchCollectionWithTools(scenario.collectionId);
|
||||
if (!collection) {
|
||||
throw new Error(`Collection not found: ${scenario.collectionId}`);
|
||||
throw new ScenarioExecutionError(
|
||||
`Collection not found: ${scenario.collectionId}`,
|
||||
'COLLECTION_NOT_FOUND',
|
||||
{ scenarioId: scenario.id, collectionId: scenario.collectionId }
|
||||
);
|
||||
}
|
||||
|
||||
if (collection.tools.length === 0) {
|
||||
throw new Error('Collection has no tools configured');
|
||||
throw new ScenarioExecutionError('Collection has no tools configured', 'NO_TOOLS', {
|
||||
scenarioId: scenario.id,
|
||||
collectionId: collection.id,
|
||||
collectionName: collection.name,
|
||||
});
|
||||
}
|
||||
|
||||
// Build tools from collection
|
||||
const tools = buildCollectionTools(collection);
|
||||
let tools: Record<string, ReturnType<typeof createToolDefinition>>;
|
||||
try {
|
||||
tools = buildCollectionTools(collection);
|
||||
} catch (err) {
|
||||
throw new ScenarioExecutionError(
|
||||
`Failed to build tools: ${err instanceof Error ? err.message : 'Unknown error'}`,
|
||||
'TOOL_BUILD_ERROR',
|
||||
{ scenarioId: scenario.id, collectionId: collection.id, error: String(err) }
|
||||
);
|
||||
}
|
||||
|
||||
console.log('[executeWithAgent] Executing scenario with tools:', Object.keys(tools));
|
||||
logScenarioEvent('tools_built', scenario.id, {
|
||||
toolCount: Object.keys(tools).length,
|
||||
toolNames: Object.keys(tools),
|
||||
});
|
||||
|
||||
// Build system prompt for scenario execution
|
||||
const systemPrompt = `You are an AI assistant tasked with completing the following scenario using the available tools.
|
||||
|
|
@ -381,15 +500,30 @@ Available tools: ${Object.keys(tools).join(', ')}
|
|||
Complete the user's task to the best of your ability using the tools provided.`;
|
||||
|
||||
// Execute with generateText and multi-step tool loop
|
||||
const result = await generateText({
|
||||
model: openai(DEFAULT_MODEL),
|
||||
messages: [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: scenario.prompt },
|
||||
],
|
||||
tools,
|
||||
stopWhen: stepCountIs(MAX_TOOL_STEPS),
|
||||
});
|
||||
const result = await (async () => {
|
||||
try {
|
||||
return await generateText({
|
||||
model: openai(DEFAULT_MODEL),
|
||||
messages: [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: scenario.prompt },
|
||||
],
|
||||
tools,
|
||||
stopWhen: stepCountIs(MAX_TOOL_STEPS),
|
||||
});
|
||||
} catch (err) {
|
||||
throw new ScenarioExecutionError(
|
||||
`AI execution failed: ${err instanceof Error ? err.message : 'Unknown error'}`,
|
||||
'EXECUTION_ERROR',
|
||||
{
|
||||
scenarioId: scenario.id,
|
||||
error: String(err),
|
||||
model: DEFAULT_MODEL,
|
||||
maxSteps: MAX_TOOL_STEPS,
|
||||
}
|
||||
);
|
||||
}
|
||||
})();
|
||||
|
||||
// Build conversation history from response
|
||||
const conversation: unknown[] = [{ role: 'user', content: scenario.prompt }];
|
||||
|
|
@ -435,13 +569,13 @@ Complete the user's task to the best of your ability using the tools provided.`;
|
|||
// Get token usage
|
||||
const usage = result.usage || { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
|
||||
|
||||
console.log(
|
||||
'[executeWithAgent] Completed in',
|
||||
logScenarioEvent('execution_complete', scenario.id, {
|
||||
durationMs,
|
||||
'ms with',
|
||||
result.steps?.length || 0,
|
||||
'steps'
|
||||
);
|
||||
stepCount: result.steps?.length || 0,
|
||||
inputTokens: usage.inputTokens ?? 0,
|
||||
outputTokens: usage.outputTokens ?? 0,
|
||||
hasOutput: !!result.text,
|
||||
});
|
||||
|
||||
return {
|
||||
output: result.text || '[No output generated]',
|
||||
|
|
|
|||
15
pnpm-lock.yaml
generated
15
pnpm-lock.yaml
generated
|
|
@ -303,6 +303,12 @@ importers:
|
|||
ai:
|
||||
specifier: 6.0.49
|
||||
version: 6.0.49(zod@4.3.5)
|
||||
ajv:
|
||||
specifier: ^8.17.1
|
||||
version: 8.17.1
|
||||
ajv-formats:
|
||||
specifier: ^3.0.1
|
||||
version: 3.0.1(ajv@8.17.1)
|
||||
better-auth:
|
||||
specifier: ^1.4.10
|
||||
version: 1.4.10(@prisma/client@6.19.1(prisma@6.19.1(typescript@5.9.3))(typescript@5.9.3))(better-sqlite3@12.5.0)(drizzle-kit@0.31.8)(drizzle-orm@0.45.1(@electric-sql/pglite@0.3.2)(@opentelemetry/api@1.9.0)(@prisma/client@6.19.1(prisma@6.19.1(typescript@5.9.3))(typescript@5.9.3))(@types/better-sqlite3@7.6.13)(@upstash/redis@1.36.1)(better-sqlite3@12.5.0)(gel@2.2.0)(kysely@0.28.9)(mysql2@3.15.3)(pg@8.16.3)(postgres@3.4.7)(prisma@6.19.1(typescript@5.9.3)))(mongodb@6.20.0)(mysql2@3.15.3)(next@16.1.1(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(pg@8.16.3)(prisma@6.19.1(typescript@5.9.3))(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(vitest@4.0.16(@opentelemetry/api@1.9.0)(@types/node@25.0.3)(happy-dom@20.1.0)(jiti@2.6.1)(jsdom@27.4.0)(lightningcss@1.30.2)(msw@2.12.7(@types/node@25.0.3)(typescript@5.9.3))(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))
|
||||
|
|
@ -7413,6 +7419,7 @@ packages:
|
|||
'@vercel/kv@3.0.0':
|
||||
resolution: {integrity: sha512-pKT8fRnfyYk2MgvyB6fn6ipJPCdfZwiKDdw7vB+HL50rjboEBHDVBEcnwfkEpVSp2AjNtoaOUH7zG+bVC/rvSg==}
|
||||
engines: {node: '>=14.6'}
|
||||
deprecated: 'Vercel KV is deprecated. If you had an existing KV store, it should have moved to Upstash Redis which you will see under Vercel Integrations. For new projects, install a Redis integration from Vercel Marketplace: https://vercel.com/marketplace?category=storage&search=redis'
|
||||
|
||||
'@vercel/oidc@3.1.0':
|
||||
resolution: {integrity: sha512-Fw28YZpRnA3cAHHDlkt7xQHiJ0fcL+NRcIqsocZQUSmbzeIKRpwttJjik5ZGanXP+vlA4SbTg+AbA3bP363l+w==}
|
||||
|
|
@ -15583,14 +15590,14 @@ snapshots:
|
|||
'@remotion/media-parser': 4.0.409
|
||||
'@remotion/studio': 4.0.409(react-dom@19.2.3(react@19.2.3))(react@19.2.3)
|
||||
'@remotion/studio-shared': 4.0.409(react-dom@19.2.3(react@19.2.3))(react@19.2.3)
|
||||
css-loader: 5.2.7(webpack@5.96.1(esbuild@0.25.0))
|
||||
css-loader: 5.2.7(webpack@5.96.1)
|
||||
esbuild: 0.25.0
|
||||
react: 19.2.3
|
||||
react-dom: 19.2.3(react@19.2.3)
|
||||
react-refresh: 0.9.0
|
||||
remotion: 4.0.409(react-dom@19.2.3(react@19.2.3))(react@19.2.3)
|
||||
source-map: 0.7.3
|
||||
style-loader: 4.0.0(webpack@5.96.1(esbuild@0.25.0))
|
||||
style-loader: 4.0.0(webpack@5.96.1)
|
||||
webpack: 5.96.1(esbuild@0.25.0)
|
||||
transitivePeerDependencies:
|
||||
- '@swc/core'
|
||||
|
|
@ -17968,7 +17975,7 @@ snapshots:
|
|||
shebang-command: 2.0.0
|
||||
which: 2.0.2
|
||||
|
||||
css-loader@5.2.7(webpack@5.96.1(esbuild@0.25.0)):
|
||||
css-loader@5.2.7(webpack@5.96.1):
|
||||
dependencies:
|
||||
icss-utils: 5.1.0(postcss@8.5.6)
|
||||
loader-utils: 2.0.4
|
||||
|
|
@ -22802,7 +22809,7 @@ snapshots:
|
|||
|
||||
stubborn-utils@1.0.2: {}
|
||||
|
||||
style-loader@4.0.0(webpack@5.96.1(esbuild@0.25.0)):
|
||||
style-loader@4.0.0(webpack@5.96.1):
|
||||
dependencies:
|
||||
webpack: 5.96.1(esbuild@0.25.0)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue