feat: add scenarios system for collection testing
Implements the complete scenarios feature for TPMJS: API endpoints: - GET/POST /api/scenarios - list/create scenarios - GET/PATCH/DELETE /api/scenarios/:id - scenario CRUD - POST /api/scenarios/:id/run - execute scenario - GET /api/scenarios/:id/runs - run history - POST /api/scenarios/check-similarity - vector similarity check - GET /api/scenarios/featured - featured scenarios - POST /api/collections/:id/scenarios/generate - AI scenario generation - GET /api/collections/:id/scenarios - list collection scenarios Services: - generate-prompt.ts - AI prompt generation using GPT-4o-mini - similarity.ts - vector embedding and cosine similarity - evaluate.ts - LLM-based success evaluation - execute.ts - scenario execution orchestration CLI commands: - tpm scenario list [collection] - list scenarios - tpm scenario run <collection> - run all scenarios - tpm scenario test <id> - run single scenario - tpm scenario generate <collection> - generate scenarios - tpm scenario info <id> - scenario details Database: - Scenario, ScenarioEmbedding, ScenarioRun, ScenarioQuota models - Streak-based quality scoring - Daily quota management Migration script included for converting existing useCases.
This commit is contained in:
parent
154f000505
commit
ea742c9386
26 changed files with 6922 additions and 282 deletions
2
apps/web/next-env.d.ts
vendored
2
apps/web/next-env.d.ts
vendored
|
|
@ -1,6 +1,6 @@
|
|||
/// <reference types="next" />
|
||||
/// <reference types="next/image-types/global" />
|
||||
import "./.next/types/routes.d.ts";
|
||||
import "./.next/dev/types/routes.d.ts";
|
||||
|
||||
// NOTE: This file should not be edited
|
||||
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,208 @@
|
|||
/**
|
||||
* Scenario Generation API
|
||||
*
|
||||
* POST /api/collections/[id]/scenarios/generate Generate AI-powered scenarios for a collection
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import type { NextRequest } from 'next/server';
|
||||
import { z } from 'zod';
|
||||
import { authenticateRequest } from '~/lib/api-keys/middleware';
|
||||
import {
|
||||
apiForbidden,
|
||||
apiInternalError,
|
||||
apiNotFound,
|
||||
apiSuccess,
|
||||
apiValidationError,
|
||||
} from '~/lib/api-response';
|
||||
import { AI_GENERATION_RATE_LIMIT, checkRateLimitDistributed } from '~/lib/rate-limit';
|
||||
import { generateScenarios } from '~/lib/scenarios/generate-prompt';
|
||||
import { checkSimilarity, generateAndStoreEmbedding } from '~/lib/scenarios/similarity';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 120; // AI generation + embeddings can take time
|
||||
|
||||
type RouteContext = {
|
||||
params: Promise<{ id: string }>;
|
||||
};
|
||||
|
||||
const GenerateRequestSchema = z.object({
|
||||
count: z.number().int().min(1).max(10).default(1),
|
||||
skipSimilarityCheck: z.boolean().default(false),
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /api/collections/[id]/scenarios/generate
|
||||
* Generate AI-powered scenarios for a collection
|
||||
*
|
||||
* Body:
|
||||
* - count: Number of scenarios to generate (1-10, default: 1)
|
||||
* - skipSimilarityCheck: If true, skip duplicate warning (default: false)
|
||||
*
|
||||
* Returns generated scenarios with similarity warnings if applicable
|
||||
*/
|
||||
export async function POST(request: NextRequest, context: RouteContext) {
|
||||
const requestId = crypto.randomUUID();
|
||||
|
||||
try {
|
||||
// Check authentication
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
if (!authResult.authenticated || !authResult.userId) {
|
||||
return apiForbidden('Authentication required to generate scenarios', requestId);
|
||||
}
|
||||
|
||||
// Rate limit check (strict for AI operations)
|
||||
const rateLimitResponse = await checkRateLimitDistributed(request, AI_GENERATION_RATE_LIMIT);
|
||||
if (rateLimitResponse) {
|
||||
return rateLimitResponse;
|
||||
}
|
||||
|
||||
const { id } = await context.params;
|
||||
|
||||
// Parse request body
|
||||
let body: { count?: number; skipSimilarityCheck?: boolean } = {};
|
||||
try {
|
||||
body = await request.json();
|
||||
} catch {
|
||||
// Empty body is fine, use defaults
|
||||
}
|
||||
|
||||
const parseResult = GenerateRequestSchema.safeParse(body);
|
||||
if (!parseResult.success) {
|
||||
return apiValidationError(
|
||||
'Invalid request body',
|
||||
{ errors: parseResult.error.flatten().fieldErrors },
|
||||
requestId
|
||||
);
|
||||
}
|
||||
|
||||
const { count, skipSimilarityCheck } = parseResult.data;
|
||||
|
||||
// Fetch collection with tools
|
||||
const collection = await prisma.collection.findUnique({
|
||||
where: { id },
|
||||
include: {
|
||||
tools: {
|
||||
include: {
|
||||
tool: {
|
||||
select: {
|
||||
name: true,
|
||||
description: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
orderBy: { position: 'asc' },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!collection) {
|
||||
return apiNotFound('Collection', requestId);
|
||||
}
|
||||
|
||||
// User must own the collection to generate scenarios
|
||||
if (collection.userId !== authResult.userId) {
|
||||
return apiForbidden('You can only generate scenarios for your own collections', requestId);
|
||||
}
|
||||
|
||||
// Validate collection has tools
|
||||
if (collection.tools.length === 0) {
|
||||
return apiForbidden(
|
||||
'Collection must have at least one tool to generate scenarios',
|
||||
requestId
|
||||
);
|
||||
}
|
||||
|
||||
// Prepare collection info for AI generation
|
||||
const collectionInfo = {
|
||||
name: collection.name,
|
||||
description: collection.description,
|
||||
tools: collection.tools.map((ct) => ({
|
||||
name: ct.tool.name,
|
||||
description: ct.tool.description,
|
||||
})),
|
||||
};
|
||||
|
||||
// Generate scenarios with AI
|
||||
const generatedScenarios = await generateScenarios(collectionInfo, count);
|
||||
|
||||
// Process each generated scenario
|
||||
const results: Array<{
|
||||
scenario: {
|
||||
id: string;
|
||||
prompt: string;
|
||||
name: string | null;
|
||||
tags: string[];
|
||||
};
|
||||
similarity?: {
|
||||
hasSimilar: boolean;
|
||||
maxSimilarity: number;
|
||||
similar: Array<{ id: string; name: string | null; similarity: number }>;
|
||||
};
|
||||
}> = [];
|
||||
|
||||
for (const generated of generatedScenarios) {
|
||||
// Check similarity unless skipped
|
||||
let similarityResult: {
|
||||
hasSimilar: boolean;
|
||||
maxSimilarity: number;
|
||||
similarScenarios: Array<{
|
||||
scenario: { id: string; name: string | null };
|
||||
similarity: number;
|
||||
}>;
|
||||
} | null = null;
|
||||
|
||||
if (!skipSimilarityCheck) {
|
||||
similarityResult = await checkSimilarity(generated.prompt, collection.id);
|
||||
}
|
||||
|
||||
// Create the scenario
|
||||
const scenario = await prisma.scenario.create({
|
||||
data: {
|
||||
collectionId: collection.id,
|
||||
prompt: generated.prompt,
|
||||
name: generated.name,
|
||||
tags: generated.tags,
|
||||
},
|
||||
});
|
||||
|
||||
// Generate and store embedding for future similarity checks
|
||||
await generateAndStoreEmbedding(scenario.id, generated.prompt);
|
||||
|
||||
results.push({
|
||||
scenario: {
|
||||
id: scenario.id,
|
||||
prompt: scenario.prompt,
|
||||
name: scenario.name,
|
||||
tags: scenario.tags,
|
||||
},
|
||||
...(similarityResult && similarityResult.hasSimilar
|
||||
? {
|
||||
similarity: {
|
||||
hasSimilar: true,
|
||||
maxSimilarity: Math.round(similarityResult.maxSimilarity * 100),
|
||||
similar: similarityResult.similarScenarios.slice(0, 3).map((s) => ({
|
||||
id: s.scenario.id,
|
||||
name: s.scenario.name,
|
||||
similarity: Math.round(s.similarity * 100),
|
||||
})),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
return apiSuccess(
|
||||
{
|
||||
scenarios: results,
|
||||
generatedAt: new Date().toISOString(),
|
||||
},
|
||||
{ requestId, status: 201 }
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('[API Error] POST /api/collections/[id]/scenarios/generate:', error);
|
||||
return apiInternalError('Failed to generate scenarios', requestId);
|
||||
}
|
||||
}
|
||||
141
apps/web/src/app/api/collections/[id]/scenarios/route.ts
Normal file
141
apps/web/src/app/api/collections/[id]/scenarios/route.ts
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
/**
|
||||
* Collection Scenarios API
|
||||
*
|
||||
* GET /api/collections/[id]/scenarios List scenarios for a collection
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import type { NextRequest } from 'next/server';
|
||||
import { authenticateRequest } from '~/lib/api-keys/middleware';
|
||||
import { apiForbidden, apiInternalError, apiNotFound, apiSuccess } from '~/lib/api-response';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 30;
|
||||
|
||||
type RouteContext = {
|
||||
params: Promise<{ id: string }>;
|
||||
};
|
||||
|
||||
/**
|
||||
* GET /api/collections/[id]/scenarios
|
||||
* List scenarios for a collection
|
||||
*
|
||||
* Query params:
|
||||
* - limit: Max results (default: 50, max: 100)
|
||||
* - offset: Pagination offset (default: 0)
|
||||
* - status: Filter by lastRunStatus (optional)
|
||||
*/
|
||||
export async function GET(request: NextRequest, context: RouteContext) {
|
||||
const requestId = crypto.randomUUID();
|
||||
|
||||
try {
|
||||
const { id } = await context.params;
|
||||
const { searchParams } = new URL(request.url);
|
||||
const limit = Math.min(
|
||||
Math.max(Number.parseInt(searchParams.get('limit') || '50', 10), 1),
|
||||
100
|
||||
);
|
||||
const offset = Math.max(Number.parseInt(searchParams.get('offset') || '0', 10), 0);
|
||||
const statusFilter = searchParams.get('status');
|
||||
|
||||
// Check authentication (optional - affects what data is shown)
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
// Fetch collection
|
||||
const collection = await prisma.collection.findUnique({
|
||||
where: { id },
|
||||
select: {
|
||||
id: true,
|
||||
isPublic: true,
|
||||
userId: true,
|
||||
name: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!collection) {
|
||||
return apiNotFound('Collection', requestId);
|
||||
}
|
||||
|
||||
// Check access
|
||||
const isOwner = authResult.authenticated && collection.userId === authResult.userId;
|
||||
if (!collection.isPublic && !isOwner) {
|
||||
return apiForbidden('Access denied', requestId);
|
||||
}
|
||||
|
||||
// Build where clause
|
||||
const where: {
|
||||
collectionId: string;
|
||||
lastRunStatus?: string;
|
||||
} = { collectionId: id };
|
||||
|
||||
if (statusFilter) {
|
||||
where.lastRunStatus = statusFilter;
|
||||
}
|
||||
|
||||
// Fetch scenarios with pagination
|
||||
const scenarios = await prisma.scenario.findMany({
|
||||
where,
|
||||
orderBy: [{ qualityScore: 'desc' }, { createdAt: 'desc' }],
|
||||
take: limit + 1,
|
||||
skip: offset,
|
||||
select: {
|
||||
id: true,
|
||||
prompt: true,
|
||||
name: true,
|
||||
description: true,
|
||||
tags: true,
|
||||
qualityScore: true,
|
||||
totalRuns: true,
|
||||
lastRunAt: true,
|
||||
lastRunStatus: true,
|
||||
consecutivePasses: true,
|
||||
consecutiveFails: true,
|
||||
createdAt: true,
|
||||
updatedAt: true,
|
||||
},
|
||||
});
|
||||
|
||||
const hasMore = scenarios.length > limit;
|
||||
const data = hasMore ? scenarios.slice(0, limit) : scenarios;
|
||||
|
||||
return apiSuccess(
|
||||
{
|
||||
collection: {
|
||||
id: collection.id,
|
||||
name: collection.name,
|
||||
},
|
||||
scenarios: data.map((s) => ({
|
||||
id: s.id,
|
||||
prompt: s.prompt.slice(0, 200) + (s.prompt.length > 200 ? '...' : ''),
|
||||
name: s.name,
|
||||
description: s.description,
|
||||
tags: s.tags,
|
||||
metrics: {
|
||||
qualityScore: s.qualityScore,
|
||||
totalRuns: s.totalRuns,
|
||||
consecutivePasses: s.consecutivePasses,
|
||||
consecutiveFails: s.consecutiveFails,
|
||||
lastRunStatus: s.lastRunStatus,
|
||||
lastRunAt: s.lastRunAt,
|
||||
},
|
||||
timestamps: {
|
||||
createdAt: s.createdAt,
|
||||
updatedAt: s.updatedAt,
|
||||
},
|
||||
})),
|
||||
},
|
||||
{
|
||||
requestId,
|
||||
pagination: {
|
||||
limit,
|
||||
offset,
|
||||
hasMore,
|
||||
},
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('[API Error] GET /api/collections/[id]/scenarios:', error);
|
||||
return apiInternalError('Failed to fetch scenarios', requestId);
|
||||
}
|
||||
}
|
||||
405
apps/web/src/app/api/scenarios/[id]/route.ts
Normal file
405
apps/web/src/app/api/scenarios/[id]/route.ts
Normal file
|
|
@ -0,0 +1,405 @@
|
|||
/**
|
||||
* Scenario API - Get, Update, Delete individual scenario
|
||||
*
|
||||
* GET /api/scenarios/[id] Get scenario details
|
||||
* PATCH /api/scenarios/[id] Update scenario
|
||||
* DELETE /api/scenarios/[id] Delete scenario
|
||||
*/
|
||||
|
||||
import { Prisma, prisma } from '@tpmjs/db';
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
import { z } from 'zod';
|
||||
import { authenticateRequest } from '~/lib/api-keys/middleware';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 60;
|
||||
|
||||
const API_VERSION = '1.0.0';
|
||||
|
||||
interface ApiResponse<T = unknown> {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
error?: {
|
||||
code: string;
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
meta: {
|
||||
version: string;
|
||||
timestamp: string;
|
||||
requestId?: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface RouteContext {
|
||||
params: Promise<{ id: string }>;
|
||||
}
|
||||
|
||||
// Validation schema for updating a scenario
|
||||
const UpdateScenarioSchema = z.object({
|
||||
prompt: z.string().min(10).optional(),
|
||||
name: z.string().max(200).nullish(),
|
||||
description: z.string().nullish(),
|
||||
assertions: z
|
||||
.object({
|
||||
regex: z.array(z.string()).optional(),
|
||||
schema: z.record(z.string(), z.unknown()).optional(),
|
||||
})
|
||||
.nullish(),
|
||||
tags: z.array(z.string()).optional(),
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/scenarios/[id]
|
||||
* Get scenario details with recent runs
|
||||
*
|
||||
* Query params:
|
||||
* - runsLimit: Max runs to return (default: 10, max: 50)
|
||||
*/
|
||||
export async function GET(
|
||||
request: NextRequest,
|
||||
context: RouteContext
|
||||
): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
const { id } = await context.params;
|
||||
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const runsLimit = Math.min(
|
||||
Math.max(Number.parseInt(searchParams.get('runsLimit') || '10', 10), 1),
|
||||
50
|
||||
);
|
||||
|
||||
// Check authentication (optional - affects what data is shown)
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
const scenario = await prisma.scenario.findUnique({
|
||||
where: { id },
|
||||
include: {
|
||||
collection: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
slug: true,
|
||||
isPublic: true,
|
||||
userId: true,
|
||||
user: {
|
||||
select: {
|
||||
username: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
runs: {
|
||||
orderBy: { createdAt: 'desc' },
|
||||
take: runsLimit,
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
evaluatorVerdict: true,
|
||||
executionTimeMs: true,
|
||||
totalTokens: true,
|
||||
startedAt: true,
|
||||
completedAt: true,
|
||||
createdAt: true,
|
||||
},
|
||||
},
|
||||
_count: {
|
||||
select: { runs: true },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!scenario) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'NOT_FOUND', message: 'Scenario not found' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
// Check access - collection must be public OR user must own it
|
||||
const isOwner = authResult.authenticated && scenario.collection?.userId === authResult.userId;
|
||||
const isPublic = scenario.collection?.isPublic ?? false;
|
||||
|
||||
if (!isOwner && !isPublic) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'FORBIDDEN', message: 'Access denied' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
id: scenario.id,
|
||||
collectionId: scenario.collectionId,
|
||||
prompt: scenario.prompt,
|
||||
name: scenario.name,
|
||||
description: scenario.description,
|
||||
assertions: scenario.assertions,
|
||||
tags: scenario.tags,
|
||||
qualityScore: scenario.qualityScore,
|
||||
consecutivePasses: scenario.consecutivePasses,
|
||||
consecutiveFails: scenario.consecutiveFails,
|
||||
totalRuns: scenario.totalRuns,
|
||||
lastRunAt: scenario.lastRunAt,
|
||||
lastRunStatus: scenario.lastRunStatus,
|
||||
createdAt: scenario.createdAt,
|
||||
updatedAt: scenario.updatedAt,
|
||||
isOwner,
|
||||
collection: scenario.collection
|
||||
? {
|
||||
id: scenario.collection.id,
|
||||
name: scenario.collection.name,
|
||||
slug: scenario.collection.slug,
|
||||
username: scenario.collection.user.username,
|
||||
}
|
||||
: null,
|
||||
recentRuns: scenario.runs,
|
||||
runCount: scenario._count.runs,
|
||||
},
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[API Error] GET /api/scenarios/[id]:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to fetch scenario' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* PATCH /api/scenarios/[id]
|
||||
* Update a scenario
|
||||
*
|
||||
* Requires authentication and ownership of the collection
|
||||
*/
|
||||
export async function PATCH(
|
||||
request: NextRequest,
|
||||
context: RouteContext
|
||||
): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
const { id } = await context.params;
|
||||
|
||||
try {
|
||||
// Check authentication
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
if (!authResult.authenticated || !authResult.userId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'UNAUTHORIZED', message: 'Authentication required' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 401 }
|
||||
);
|
||||
}
|
||||
|
||||
// Find scenario and check ownership
|
||||
const existing = await prisma.scenario.findUnique({
|
||||
where: { id },
|
||||
include: {
|
||||
collection: {
|
||||
select: { userId: true },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!existing) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'NOT_FOUND', message: 'Scenario not found' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
if (existing.collection?.userId !== authResult.userId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'FORBIDDEN', message: 'Access denied' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
|
||||
// Parse and validate request body
|
||||
const body = await request.json();
|
||||
const parseResult = UpdateScenarioSchema.safeParse(body);
|
||||
|
||||
if (!parseResult.success) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: {
|
||||
code: 'VALIDATION_ERROR',
|
||||
message: 'Invalid request body',
|
||||
details: { errors: parseResult.error.flatten().fieldErrors },
|
||||
},
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const { prompt, name, description, assertions, tags } = parseResult.data;
|
||||
|
||||
// Update scenario (transform null to Prisma.JsonNull for JSON fields)
|
||||
const scenario = await prisma.scenario.update({
|
||||
where: { id },
|
||||
data: {
|
||||
...(prompt !== undefined && { prompt }),
|
||||
...(name !== undefined && { name }),
|
||||
...(description !== undefined && { description }),
|
||||
...(assertions !== undefined && {
|
||||
assertions:
|
||||
assertions === null
|
||||
? Prisma.JsonNull
|
||||
: (assertions as unknown as Prisma.InputJsonValue),
|
||||
}),
|
||||
...(tags !== undefined && { tags }),
|
||||
},
|
||||
});
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
id: scenario.id,
|
||||
collectionId: scenario.collectionId,
|
||||
prompt: scenario.prompt,
|
||||
name: scenario.name,
|
||||
description: scenario.description,
|
||||
assertions: scenario.assertions,
|
||||
tags: scenario.tags,
|
||||
qualityScore: scenario.qualityScore,
|
||||
totalRuns: scenario.totalRuns,
|
||||
updatedAt: scenario.updatedAt,
|
||||
},
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[API Error] PATCH /api/scenarios/[id]:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to update scenario' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* DELETE /api/scenarios/[id]
|
||||
* Delete a scenario
|
||||
*
|
||||
* Requires authentication and ownership of the collection
|
||||
*/
|
||||
export async function DELETE(
|
||||
_request: NextRequest,
|
||||
context: RouteContext
|
||||
): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
const { id } = await context.params;
|
||||
|
||||
try {
|
||||
// Check authentication
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
if (!authResult.authenticated || !authResult.userId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'UNAUTHORIZED', message: 'Authentication required' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 401 }
|
||||
);
|
||||
}
|
||||
|
||||
// Find scenario and check ownership
|
||||
const existing = await prisma.scenario.findUnique({
|
||||
where: { id },
|
||||
include: {
|
||||
collection: {
|
||||
select: { userId: true },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!existing) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'NOT_FOUND', message: 'Scenario not found' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
if (existing.collection?.userId !== authResult.userId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'FORBIDDEN', message: 'Access denied' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
|
||||
// Delete scenario (cascade will delete runs and embedding)
|
||||
await prisma.scenario.delete({
|
||||
where: { id },
|
||||
});
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: { deleted: true },
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[API Error] DELETE /api/scenarios/[id]:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to delete scenario' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
178
apps/web/src/app/api/scenarios/[id]/run/route.ts
Normal file
178
apps/web/src/app/api/scenarios/[id]/run/route.ts
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
/**
|
||||
* Scenario Run API - Trigger a scenario execution
|
||||
*
|
||||
* POST /api/scenarios/[id]/run Execute a scenario
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
import { authenticateRequest } from '~/lib/api-keys/middleware';
|
||||
import { checkAndDecrementQuota, executeScenario, getQuotaStatus } from '~/lib/scenarios/execute';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 300; // 5 minutes for scenario execution
|
||||
|
||||
const API_VERSION = '1.0.0';
|
||||
|
||||
interface ApiResponse<T = unknown> {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
error?: {
|
||||
code: string;
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
meta: {
|
||||
version: string;
|
||||
timestamp: string;
|
||||
requestId?: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface RouteContext {
|
||||
params: Promise<{ id: string }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/scenarios/[id]/run
|
||||
* Execute a scenario
|
||||
*
|
||||
* Requires authentication
|
||||
* Subject to daily quota limits
|
||||
*/
|
||||
export async function POST(
|
||||
_request: NextRequest,
|
||||
context: RouteContext
|
||||
): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
const { id: scenarioId } = await context.params;
|
||||
|
||||
try {
|
||||
// Check authentication
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
if (!authResult.authenticated || !authResult.userId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'UNAUTHORIZED', message: 'Authentication required' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 401 }
|
||||
);
|
||||
}
|
||||
|
||||
// Check scenario exists
|
||||
const scenario = await prisma.scenario.findUnique({
|
||||
where: { id: scenarioId },
|
||||
include: {
|
||||
collection: {
|
||||
select: {
|
||||
id: true,
|
||||
isPublic: true,
|
||||
userId: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!scenario) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'NOT_FOUND', message: 'Scenario not found' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
// Check access - must be owner or collection must be public
|
||||
const isOwner = scenario.collection?.userId === authResult.userId;
|
||||
const isPublic = scenario.collection?.isPublic ?? false;
|
||||
|
||||
if (!isOwner && !isPublic) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'FORBIDDEN', message: 'Access denied' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
|
||||
// Check quota
|
||||
const quota = await checkAndDecrementQuota(authResult.userId);
|
||||
|
||||
if (!quota.allowed) {
|
||||
const quotaStatus = await getQuotaStatus(authResult.userId);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: {
|
||||
code: 'QUOTA_EXCEEDED',
|
||||
message: 'Daily scenario run quota exceeded. Try again tomorrow.',
|
||||
details: {
|
||||
used: quotaStatus.used,
|
||||
limit: quotaStatus.limit,
|
||||
resetsAt: quotaStatus.resetsAt.toISOString(),
|
||||
},
|
||||
},
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 429 }
|
||||
);
|
||||
}
|
||||
|
||||
// Execute the scenario
|
||||
const { run, success } = await executeScenario(scenario, authResult.userId);
|
||||
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: true,
|
||||
data: {
|
||||
runId: run.id,
|
||||
status: run.status,
|
||||
success,
|
||||
evaluator: {
|
||||
model: run.evaluatorModel,
|
||||
verdict: run.evaluatorVerdict,
|
||||
reason: run.evaluatorReason,
|
||||
},
|
||||
assertions: run.assertionResults,
|
||||
usage: {
|
||||
inputTokens: run.inputTokens,
|
||||
outputTokens: run.outputTokens,
|
||||
totalTokens: run.totalTokens,
|
||||
executionTimeMs: run.executionTimeMs,
|
||||
},
|
||||
timestamps: {
|
||||
startedAt: run.startedAt,
|
||||
completedAt: run.completedAt,
|
||||
createdAt: run.createdAt,
|
||||
},
|
||||
quotaRemaining: quota.remaining,
|
||||
},
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
note: 'Execution uses simulated agent. Full agent integration coming soon.',
|
||||
},
|
||||
},
|
||||
{ status: 201 }
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('[API Error] POST /api/scenarios/[id]/run:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to execute scenario' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
196
apps/web/src/app/api/scenarios/[id]/runs/route.ts
Normal file
196
apps/web/src/app/api/scenarios/[id]/runs/route.ts
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
/**
|
||||
* Scenario Runs API - List run history
|
||||
*
|
||||
* GET /api/scenarios/[id]/runs Get all runs for a scenario (paginated)
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
import { authenticateRequest } from '~/lib/api-keys/middleware';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 60;
|
||||
|
||||
const API_VERSION = '1.0.0';
|
||||
|
||||
interface ApiResponse<T = unknown> {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
error?: {
|
||||
code: string;
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
meta: {
|
||||
version: string;
|
||||
timestamp: string;
|
||||
requestId?: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface RouteContext {
|
||||
params: Promise<{ id: string }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/scenarios/[id]/runs
|
||||
* Get run history for a scenario
|
||||
*
|
||||
* Query params:
|
||||
* - limit: Max results (default: 20, max: 100)
|
||||
* - offset: Pagination offset (default: 0)
|
||||
* - status: Filter by status (optional)
|
||||
*/
|
||||
export async function GET(
|
||||
request: NextRequest,
|
||||
context: RouteContext
|
||||
): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
const { id: scenarioId } = await context.params;
|
||||
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const limit = Math.min(
|
||||
Math.max(Number.parseInt(searchParams.get('limit') || '20', 10), 1),
|
||||
100
|
||||
);
|
||||
const offset = Math.max(Number.parseInt(searchParams.get('offset') || '0', 10), 0);
|
||||
const statusFilter = searchParams.get('status');
|
||||
|
||||
// Check authentication (optional - affects what data is shown)
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
// Verify scenario exists and check access
|
||||
const scenario = await prisma.scenario.findUnique({
|
||||
where: { id: scenarioId },
|
||||
include: {
|
||||
collection: {
|
||||
select: {
|
||||
isPublic: true,
|
||||
userId: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!scenario) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'NOT_FOUND', message: 'Scenario not found' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
// Check access
|
||||
const isOwner = authResult.authenticated && scenario.collection?.userId === authResult.userId;
|
||||
const isPublic = scenario.collection?.isPublic ?? false;
|
||||
|
||||
if (!isOwner && !isPublic) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'FORBIDDEN', message: 'Access denied' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
|
||||
// Build where clause
|
||||
const where: { scenarioId: string; status?: string } = { scenarioId };
|
||||
if (statusFilter) {
|
||||
where.status = statusFilter;
|
||||
}
|
||||
|
||||
// Fetch runs
|
||||
const runs = await prisma.scenarioRun.findMany({
|
||||
where,
|
||||
orderBy: { createdAt: 'desc' },
|
||||
take: limit + 1,
|
||||
skip: offset,
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
retryCount: true,
|
||||
evaluatorModel: true,
|
||||
evaluatorVerdict: true,
|
||||
evaluatorReason: true,
|
||||
assertionResults: true,
|
||||
inputTokens: true,
|
||||
outputTokens: true,
|
||||
totalTokens: true,
|
||||
executionTimeMs: true,
|
||||
estimatedCost: true,
|
||||
startedAt: true,
|
||||
completedAt: true,
|
||||
createdAt: true,
|
||||
// Only include sensitive data if owner
|
||||
...(isOwner && {
|
||||
output: true,
|
||||
errorLog: true,
|
||||
conversation: true,
|
||||
}),
|
||||
},
|
||||
});
|
||||
|
||||
const hasMore = runs.length > limit;
|
||||
const data = hasMore ? runs.slice(0, limit) : runs;
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: data.map((run) => ({
|
||||
id: run.id,
|
||||
status: run.status,
|
||||
retryCount: run.retryCount,
|
||||
evaluator: {
|
||||
model: run.evaluatorModel,
|
||||
verdict: run.evaluatorVerdict,
|
||||
reason: run.evaluatorReason,
|
||||
},
|
||||
assertions: run.assertionResults,
|
||||
usage: {
|
||||
inputTokens: run.inputTokens,
|
||||
outputTokens: run.outputTokens,
|
||||
totalTokens: run.totalTokens,
|
||||
executionTimeMs: run.executionTimeMs,
|
||||
estimatedCost: run.estimatedCost,
|
||||
},
|
||||
timestamps: {
|
||||
startedAt: run.startedAt,
|
||||
completedAt: run.completedAt,
|
||||
createdAt: run.createdAt,
|
||||
},
|
||||
// Only include if owner
|
||||
...('output' in run && {
|
||||
output: run.output,
|
||||
errorLog: run.errorLog,
|
||||
conversation: run.conversation,
|
||||
}),
|
||||
})),
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
},
|
||||
pagination: {
|
||||
limit,
|
||||
offset,
|
||||
hasMore,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[API Error] GET /api/scenarios/[id]/runs:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to fetch runs' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
147
apps/web/src/app/api/scenarios/check-similarity/route.ts
Normal file
147
apps/web/src/app/api/scenarios/check-similarity/route.ts
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
/**
|
||||
* Scenario Similarity Check API
|
||||
*
|
||||
* POST /api/scenarios/check-similarity Check if a prompt is similar to existing scenarios
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
import { z } from 'zod';
|
||||
import { authenticateRequest } from '~/lib/api-keys/middleware';
|
||||
import { checkSimilarity } from '~/lib/scenarios/similarity';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 30;
|
||||
|
||||
const API_VERSION = '1.0.0';
|
||||
|
||||
interface ApiResponse<T = unknown> {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
error?: {
|
||||
code: string;
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
meta: {
|
||||
version: string;
|
||||
timestamp: string;
|
||||
requestId?: string;
|
||||
};
|
||||
}
|
||||
|
||||
const CheckSimilaritySchema = z.object({
|
||||
prompt: z.string().min(10, 'Prompt must be at least 10 characters'),
|
||||
collectionId: z.string().min(1, 'Collection ID is required'),
|
||||
excludeScenarioId: z.string().optional(), // For updates, exclude the scenario being edited
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /api/scenarios/check-similarity
|
||||
* Check if a prompt is similar to existing scenarios
|
||||
*
|
||||
* Returns warning if similarity >= 70%
|
||||
*/
|
||||
export async function POST(request: NextRequest): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
|
||||
try {
|
||||
// Check authentication
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
if (!authResult.authenticated || !authResult.userId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'UNAUTHORIZED', message: 'Authentication required' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 401 }
|
||||
);
|
||||
}
|
||||
|
||||
// Parse and validate request body
|
||||
const body = await request.json();
|
||||
const parseResult = CheckSimilaritySchema.safeParse(body);
|
||||
|
||||
if (!parseResult.success) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: {
|
||||
code: 'VALIDATION_ERROR',
|
||||
message: 'Invalid request body',
|
||||
details: { errors: parseResult.error.flatten().fieldErrors },
|
||||
},
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const { prompt, collectionId, excludeScenarioId } = parseResult.data;
|
||||
|
||||
// Verify collection exists and user has access
|
||||
const collection = await prisma.collection.findUnique({
|
||||
where: { id: collectionId },
|
||||
select: { id: true, userId: true, isPublic: true },
|
||||
});
|
||||
|
||||
if (!collection) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'NOT_FOUND', message: 'Collection not found' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
// User must own the collection or it must be public
|
||||
if (collection.userId !== authResult.userId && !collection.isPublic) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'FORBIDDEN', message: 'Access denied' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
|
||||
// Check similarity
|
||||
const result = await checkSimilarity(prompt, collectionId, excludeScenarioId);
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
hasSimilar: result.hasSimilar,
|
||||
maxSimilarity: Math.round(result.maxSimilarity * 100), // Return as percentage
|
||||
warningThreshold: 70,
|
||||
similar: result.similarScenarios.map((s) => ({
|
||||
id: s.scenario.id,
|
||||
name: s.scenario.name,
|
||||
prompt: s.scenario.prompt.slice(0, 200) + (s.scenario.prompt.length > 200 ? '...' : ''),
|
||||
similarity: Math.round(s.similarity * 100),
|
||||
})),
|
||||
},
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[API Error] POST /api/scenarios/check-similarity:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to check similarity' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
168
apps/web/src/app/api/scenarios/featured/route.ts
Normal file
168
apps/web/src/app/api/scenarios/featured/route.ts
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
/**
|
||||
* Featured Scenarios API
|
||||
*
|
||||
* GET /api/scenarios/featured Get featured scenarios for homepage showcase
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 30;
|
||||
|
||||
const API_VERSION = '1.0.0';
|
||||
|
||||
interface ApiResponse<T = unknown> {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
error?: {
|
||||
code: string;
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
meta: {
|
||||
version: string;
|
||||
timestamp: string;
|
||||
requestId?: string;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/scenarios/featured
|
||||
* Get featured scenarios for homepage showcase
|
||||
*
|
||||
* Returns an algorithmic mix of:
|
||||
* - High quality scenarios (by qualityScore)
|
||||
* - Diverse scenarios (different collections/tags)
|
||||
* - Fresh scenarios (recently created)
|
||||
*
|
||||
* Query params:
|
||||
* - limit: Max results (default: 6, max: 20)
|
||||
*/
|
||||
export async function GET(request: NextRequest): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const limit = Math.min(Math.max(Number.parseInt(searchParams.get('limit') || '6', 10), 1), 20);
|
||||
|
||||
// Get high quality scenarios (top 40%)
|
||||
const highQuality = await prisma.scenario.findMany({
|
||||
where: {
|
||||
collection: { isPublic: true },
|
||||
qualityScore: { gte: 0.3 },
|
||||
totalRuns: { gte: 1 },
|
||||
},
|
||||
orderBy: { qualityScore: 'desc' },
|
||||
take: Math.ceil(limit * 0.4),
|
||||
include: {
|
||||
collection: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
slug: true,
|
||||
user: { select: { username: true } },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Get diverse scenarios - different collections (30%)
|
||||
const seenCollections = new Set(highQuality.map((s) => s.collectionId));
|
||||
const diverse = await prisma.scenario.findMany({
|
||||
where: {
|
||||
collection: { isPublic: true },
|
||||
collectionId: { notIn: Array.from(seenCollections).filter(Boolean) as string[] },
|
||||
totalRuns: { gte: 1 },
|
||||
},
|
||||
orderBy: { qualityScore: 'desc' },
|
||||
take: Math.ceil(limit * 0.3),
|
||||
include: {
|
||||
collection: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
slug: true,
|
||||
user: { select: { username: true } },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Get fresh scenarios (30%)
|
||||
const seenIds = new Set([...highQuality, ...diverse].map((s) => s.id));
|
||||
const fresh = await prisma.scenario.findMany({
|
||||
where: {
|
||||
collection: { isPublic: true },
|
||||
id: { notIn: Array.from(seenIds) },
|
||||
},
|
||||
orderBy: { createdAt: 'desc' },
|
||||
take: Math.ceil(limit * 0.3),
|
||||
include: {
|
||||
collection: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
slug: true,
|
||||
user: { select: { username: true } },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Combine and shuffle slightly for variety
|
||||
const combined = [...highQuality, ...diverse, ...fresh];
|
||||
|
||||
// Simple shuffle to mix the categories
|
||||
for (let i = combined.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(Math.random() * (i + 1));
|
||||
const temp = combined[i];
|
||||
const swapWith = combined[j];
|
||||
if (temp !== undefined && swapWith !== undefined) {
|
||||
combined[i] = swapWith;
|
||||
combined[j] = temp;
|
||||
}
|
||||
}
|
||||
|
||||
// Take only the requested limit
|
||||
const featured = combined.slice(0, limit);
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: featured.map((s) => ({
|
||||
id: s.id,
|
||||
name: s.name,
|
||||
prompt: s.prompt.slice(0, 150) + (s.prompt.length > 150 ? '...' : ''),
|
||||
tags: s.tags.slice(0, 3),
|
||||
qualityScore: s.qualityScore,
|
||||
totalRuns: s.totalRuns,
|
||||
lastRunStatus: s.lastRunStatus,
|
||||
collection: s.collection
|
||||
? {
|
||||
id: s.collection.id,
|
||||
name: s.collection.name,
|
||||
slug: s.collection.slug,
|
||||
username: s.collection.user.username,
|
||||
}
|
||||
: null,
|
||||
})),
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
algorithm: 'mixed-quality-diversity-freshness',
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[API Error] GET /api/scenarios/featured:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to fetch featured scenarios' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
313
apps/web/src/app/api/scenarios/route.ts
Normal file
313
apps/web/src/app/api/scenarios/route.ts
Normal file
|
|
@ -0,0 +1,313 @@
|
|||
/**
|
||||
* Scenarios API - List and Create
|
||||
*
|
||||
* GET /api/scenarios List all public scenarios (paginated)
|
||||
* POST /api/scenarios Create a new scenario
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
import { z } from 'zod';
|
||||
import { authenticateRequest } from '~/lib/api-keys/middleware';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 60;
|
||||
|
||||
const API_VERSION = '1.0.0';
|
||||
|
||||
interface ApiResponse<T = unknown> {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
error?: {
|
||||
code: string;
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
meta: {
|
||||
version: string;
|
||||
timestamp: string;
|
||||
requestId?: string;
|
||||
};
|
||||
}
|
||||
|
||||
// Validation schema for creating a scenario
|
||||
const CreateScenarioSchema = z.object({
|
||||
collectionId: z.string().min(1, 'Collection ID is required'),
|
||||
prompt: z.string().min(10, 'Prompt must be at least 10 characters'),
|
||||
name: z.string().max(200).optional(),
|
||||
description: z.string().optional(),
|
||||
assertions: z
|
||||
.object({
|
||||
regex: z.array(z.string()).optional(),
|
||||
schema: z.record(z.string(), z.unknown()).optional(),
|
||||
})
|
||||
.optional(),
|
||||
tags: z.array(z.string()).optional(),
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/scenarios
|
||||
* List all public scenarios (paginated)
|
||||
*
|
||||
* Query params:
|
||||
* - limit: Max results (default: 20, max: 100)
|
||||
* - offset: Pagination offset (default: 0)
|
||||
* - collectionId: Filter by collection (optional)
|
||||
* - tags: Comma-separated tags to filter by (optional)
|
||||
* - sortBy: Sort field (default: 'qualityScore')
|
||||
*/
|
||||
export async function GET(request: NextRequest): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const limit = Math.min(
|
||||
Math.max(Number.parseInt(searchParams.get('limit') || '20', 10), 1),
|
||||
100
|
||||
);
|
||||
const offset = Math.max(Number.parseInt(searchParams.get('offset') || '0', 10), 0);
|
||||
const collectionId = searchParams.get('collectionId');
|
||||
const tagsParam = searchParams.get('tags');
|
||||
const sortBy = searchParams.get('sortBy') || 'qualityScore';
|
||||
|
||||
// Build where clause
|
||||
const where: {
|
||||
collectionId?: string;
|
||||
tags?: { hasSome: string[] };
|
||||
collection?: { isPublic: boolean };
|
||||
} = {};
|
||||
|
||||
if (collectionId) {
|
||||
where.collectionId = collectionId;
|
||||
} else {
|
||||
// Only show scenarios from public collections when not filtering by collectionId
|
||||
where.collection = { isPublic: true };
|
||||
}
|
||||
|
||||
if (tagsParam) {
|
||||
where.tags = { hasSome: tagsParam.split(',').map((t) => t.trim()) };
|
||||
}
|
||||
|
||||
// Build orderBy
|
||||
const orderByMap: Record<string, object> = {
|
||||
qualityScore: { qualityScore: 'desc' },
|
||||
totalRuns: { totalRuns: 'desc' },
|
||||
createdAt: { createdAt: 'desc' },
|
||||
lastRunAt: { lastRunAt: 'desc' },
|
||||
};
|
||||
const orderBy = orderByMap[sortBy] || orderByMap.qualityScore;
|
||||
|
||||
// Fetch scenarios with pagination
|
||||
const scenarios = await prisma.scenario.findMany({
|
||||
where,
|
||||
orderBy,
|
||||
take: limit + 1,
|
||||
skip: offset,
|
||||
include: {
|
||||
collection: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
slug: true,
|
||||
user: {
|
||||
select: {
|
||||
username: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
_count: {
|
||||
select: { runs: true },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const hasMore = scenarios.length > limit;
|
||||
const data = hasMore ? scenarios.slice(0, limit) : scenarios;
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: data.map((s) => ({
|
||||
id: s.id,
|
||||
collectionId: s.collectionId,
|
||||
prompt: s.prompt,
|
||||
name: s.name,
|
||||
description: s.description,
|
||||
tags: s.tags,
|
||||
qualityScore: s.qualityScore,
|
||||
totalRuns: s.totalRuns,
|
||||
lastRunAt: s.lastRunAt,
|
||||
lastRunStatus: s.lastRunStatus,
|
||||
createdAt: s.createdAt,
|
||||
collection: s.collection
|
||||
? {
|
||||
id: s.collection.id,
|
||||
name: s.collection.name,
|
||||
slug: s.collection.slug,
|
||||
username: s.collection.user.username,
|
||||
}
|
||||
: null,
|
||||
runCount: s._count.runs,
|
||||
})),
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
},
|
||||
pagination: {
|
||||
limit,
|
||||
offset,
|
||||
hasMore,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[API Error] GET /api/scenarios:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to fetch scenarios' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/scenarios
|
||||
* Create a new scenario
|
||||
*
|
||||
* Requires authentication
|
||||
* User must own the collection
|
||||
*/
|
||||
export async function POST(request: NextRequest): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
|
||||
try {
|
||||
// Check authentication
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
if (!authResult.authenticated || !authResult.userId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'UNAUTHORIZED', message: 'Authentication required' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 401 }
|
||||
);
|
||||
}
|
||||
|
||||
// Parse and validate request body
|
||||
const body = await request.json();
|
||||
const parseResult = CreateScenarioSchema.safeParse(body);
|
||||
|
||||
if (!parseResult.success) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: {
|
||||
code: 'VALIDATION_ERROR',
|
||||
message: 'Invalid request body',
|
||||
details: { errors: parseResult.error.flatten().fieldErrors },
|
||||
},
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const { collectionId, prompt, name, description, assertions, tags } = parseResult.data;
|
||||
|
||||
// Verify collection exists and user owns it
|
||||
const collection = await prisma.collection.findUnique({
|
||||
where: { id: collectionId },
|
||||
select: { id: true, userId: true },
|
||||
});
|
||||
|
||||
if (!collection) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'NOT_FOUND', message: 'Collection not found' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
if (collection.userId !== authResult.userId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'FORBIDDEN', message: 'You do not own this collection' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
|
||||
// Create the scenario
|
||||
const scenario = await prisma.scenario.create({
|
||||
data: {
|
||||
collectionId,
|
||||
prompt,
|
||||
name,
|
||||
description,
|
||||
assertions: assertions ? (assertions as object) : undefined,
|
||||
tags: tags || [],
|
||||
},
|
||||
include: {
|
||||
collection: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
slug: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: true,
|
||||
data: {
|
||||
id: scenario.id,
|
||||
collectionId: scenario.collectionId,
|
||||
prompt: scenario.prompt,
|
||||
name: scenario.name,
|
||||
description: scenario.description,
|
||||
assertions: scenario.assertions,
|
||||
tags: scenario.tags,
|
||||
qualityScore: scenario.qualityScore,
|
||||
totalRuns: scenario.totalRuns,
|
||||
createdAt: scenario.createdAt,
|
||||
collection: scenario.collection
|
||||
? {
|
||||
id: scenario.collection.id,
|
||||
name: scenario.collection.name,
|
||||
slug: scenario.collection.slug,
|
||||
}
|
||||
: null,
|
||||
},
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
},
|
||||
},
|
||||
{ status: 201 }
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('[API Error] POST /api/scenarios:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to create scenario' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
151
apps/web/src/lib/scenarios/evaluate.ts
Normal file
151
apps/web/src/lib/scenarios/evaluate.ts
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
/**
|
||||
* Scenario Evaluation Service
|
||||
*
|
||||
* Uses LLM judgment to evaluate if a scenario execution was successful.
|
||||
*/
|
||||
|
||||
import { anthropic } from '@ai-sdk/anthropic';
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import { generateObject } from 'ai';
|
||||
import { z } from 'zod';
|
||||
|
||||
const EvaluationSchema = z.object({
|
||||
verdict: z.enum(['pass', 'fail']).describe('Whether the scenario was completed successfully'),
|
||||
reason: z.string().describe('Brief explanation of why the scenario passed or failed'),
|
||||
confidence: z.number().min(0).max(1).describe('Confidence level in the evaluation (0-1)'),
|
||||
});
|
||||
|
||||
export type EvaluationResult = z.infer<typeof EvaluationSchema>;
|
||||
|
||||
/**
|
||||
* Supported evaluator model IDs
|
||||
*/
|
||||
export type EvaluatorModelId =
|
||||
| 'claude-3-5-sonnet-latest'
|
||||
| 'claude-3-5-haiku-latest'
|
||||
| 'gpt-4o'
|
||||
| 'gpt-4o-mini';
|
||||
|
||||
/**
|
||||
* Get the model instance for an evaluator model ID
|
||||
*/
|
||||
function getEvaluatorModel(modelId: EvaluatorModelId) {
|
||||
switch (modelId) {
|
||||
case 'claude-3-5-sonnet-latest':
|
||||
return anthropic('claude-3-5-sonnet-latest');
|
||||
case 'claude-3-5-haiku-latest':
|
||||
return anthropic('claude-3-5-haiku-latest');
|
||||
case 'gpt-4o':
|
||||
return openai('gpt-4o');
|
||||
case 'gpt-4o-mini':
|
||||
return openai('gpt-4o-mini');
|
||||
default:
|
||||
return anthropic('claude-3-5-haiku-latest');
|
||||
}
|
||||
}
|
||||
|
||||
const DEFAULT_EVALUATOR: EvaluatorModelId = 'claude-3-5-haiku-latest';
|
||||
|
||||
/**
|
||||
* Evaluate if a scenario execution was successful
|
||||
*
|
||||
* @param scenarioPrompt The original scenario prompt/task
|
||||
* @param agentOutput The output produced by the agent
|
||||
* @param conversation Optional conversation history for context
|
||||
* @param modelId Which model to use for evaluation
|
||||
*/
|
||||
export async function evaluateScenarioRun(
|
||||
scenarioPrompt: string,
|
||||
agentOutput: string,
|
||||
conversation?: unknown[],
|
||||
modelId: EvaluatorModelId = DEFAULT_EVALUATOR
|
||||
): Promise<EvaluationResult> {
|
||||
const model = getEvaluatorModel(modelId);
|
||||
|
||||
const conversationContext = conversation
|
||||
? `\n\nConversation history:\n${JSON.stringify(conversation, null, 2)}`
|
||||
: '';
|
||||
|
||||
const { object } = await generateObject({
|
||||
model,
|
||||
schema: EvaluationSchema,
|
||||
prompt: `You are evaluating whether an AI agent successfully completed a task.
|
||||
|
||||
## Task
|
||||
${scenarioPrompt}
|
||||
|
||||
## Agent Output
|
||||
${agentOutput}
|
||||
${conversationContext}
|
||||
|
||||
## Instructions
|
||||
Evaluate if the agent successfully completed the task described above.
|
||||
- A "pass" means the core objective was achieved, even if some minor aspects weren't perfect
|
||||
- A "fail" means the agent failed to accomplish the main goal
|
||||
- Consider partial success as a pass if the primary task was completed
|
||||
- Be fair but rigorous in your evaluation
|
||||
|
||||
Provide your verdict, a brief reason, and your confidence level.`,
|
||||
});
|
||||
|
||||
return object;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run assertions against the output
|
||||
*
|
||||
* @param output The agent output to check
|
||||
* @param assertions The assertions to run
|
||||
*/
|
||||
export function runAssertions(
|
||||
output: string,
|
||||
assertions: { regex?: string[]; schema?: Record<string, unknown> }
|
||||
): { passed: string[]; failed: string[] } {
|
||||
const passed: string[] = [];
|
||||
const failed: string[] = [];
|
||||
|
||||
// Check regex assertions
|
||||
if (assertions.regex) {
|
||||
for (const pattern of assertions.regex) {
|
||||
try {
|
||||
const regex = new RegExp(pattern, 'i');
|
||||
if (regex.test(output)) {
|
||||
passed.push(`regex:${pattern}`);
|
||||
} else {
|
||||
failed.push(`regex:${pattern}`);
|
||||
}
|
||||
} catch {
|
||||
failed.push(`regex:${pattern} (invalid pattern)`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Schema assertions would require more complex validation
|
||||
// For now, we'll just note if schema was provided
|
||||
if (assertions.schema) {
|
||||
// TODO: Implement JSON schema validation against parsed output
|
||||
passed.push('schema:provided (validation pending)');
|
||||
}
|
||||
|
||||
return { passed, failed };
|
||||
}
|
||||
|
||||
/**
|
||||
* Combine LLM evaluation and assertions into final verdict
|
||||
*/
|
||||
export function determineFinalVerdict(
|
||||
evaluation: EvaluationResult,
|
||||
assertions?: { passed: string[]; failed: string[] } | null
|
||||
): 'pass' | 'fail' {
|
||||
// If LLM says fail, it fails
|
||||
if (evaluation.verdict === 'fail') {
|
||||
return 'fail';
|
||||
}
|
||||
|
||||
// If there are failed assertions, it fails
|
||||
if (assertions && assertions.failed.length > 0) {
|
||||
return 'fail';
|
||||
}
|
||||
|
||||
return 'pass';
|
||||
}
|
||||
311
apps/web/src/lib/scenarios/execute.ts
Normal file
311
apps/web/src/lib/scenarios/execute.ts
Normal file
|
|
@ -0,0 +1,311 @@
|
|||
/**
|
||||
* Scenario Execution Service
|
||||
*
|
||||
* Orchestrates scenario execution using ephemeral agents.
|
||||
* Currently implements a simulated execution - full agent integration coming in Phase 3.
|
||||
*/
|
||||
|
||||
import type { Scenario, ScenarioRun } from '@prisma/client';
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import {
|
||||
determineFinalVerdict,
|
||||
type EvaluatorModelId,
|
||||
evaluateScenarioRun,
|
||||
runAssertions,
|
||||
} from './evaluate';
|
||||
|
||||
const DEFAULT_EVALUATOR: EvaluatorModelId = 'claude-3-5-haiku-latest';
|
||||
const MAX_RETRIES = 1;
|
||||
|
||||
interface ExecutionOptions {
|
||||
evaluatorModel?: EvaluatorModelId;
|
||||
}
|
||||
|
||||
interface ExecutionResult {
|
||||
run: ScenarioRun;
|
||||
success: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check and decrement user's daily quota
|
||||
*/
|
||||
export async function checkAndDecrementQuota(
|
||||
userId: string
|
||||
): Promise<{ allowed: boolean; remaining: number }> {
|
||||
const now = new Date();
|
||||
const startOfDay = new Date(now.getFullYear(), now.getMonth(), now.getDate());
|
||||
|
||||
// Get or create quota record
|
||||
let quota = await prisma.scenarioQuota.findUnique({
|
||||
where: { userId },
|
||||
});
|
||||
|
||||
if (!quota) {
|
||||
quota = await prisma.scenarioQuota.create({
|
||||
data: {
|
||||
userId,
|
||||
dailyLimit: 50,
|
||||
dailyUsed: 0,
|
||||
lastResetAt: startOfDay,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Reset quota if it's a new day
|
||||
if (quota.lastResetAt < startOfDay) {
|
||||
quota = await prisma.scenarioQuota.update({
|
||||
where: { userId },
|
||||
data: {
|
||||
dailyUsed: 0,
|
||||
lastResetAt: startOfDay,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Check if quota allows another run
|
||||
if (quota.dailyUsed >= quota.dailyLimit) {
|
||||
return { allowed: false, remaining: 0 };
|
||||
}
|
||||
|
||||
// Decrement quota
|
||||
await prisma.scenarioQuota.update({
|
||||
where: { userId },
|
||||
data: {
|
||||
dailyUsed: { increment: 1 },
|
||||
},
|
||||
});
|
||||
|
||||
return { allowed: true, remaining: quota.dailyLimit - quota.dailyUsed - 1 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Update scenario metrics based on run result
|
||||
*/
|
||||
export async function updateScenarioMetrics(
|
||||
scenarioId: string,
|
||||
status: 'pass' | 'fail' | 'error'
|
||||
): Promise<void> {
|
||||
const scenario = await prisma.scenario.findUnique({
|
||||
where: { id: scenarioId },
|
||||
});
|
||||
|
||||
if (!scenario) return;
|
||||
|
||||
let { consecutivePasses, consecutiveFails, qualityScore, totalRuns } = scenario;
|
||||
|
||||
if (status === 'pass') {
|
||||
consecutivePasses += 1;
|
||||
consecutiveFails = 0;
|
||||
// Bonus for streaks: +0.05 per pass + (streak bonus), max 1.0
|
||||
qualityScore = Math.min(1.0, qualityScore + 0.05 + consecutivePasses * 0.01);
|
||||
} else {
|
||||
consecutiveFails += 1;
|
||||
consecutivePasses = 0;
|
||||
// Penalty for fails: -0.1 per fail + (streak penalty), min 0
|
||||
qualityScore = Math.max(0, qualityScore - 0.1 - consecutiveFails * 0.02);
|
||||
}
|
||||
|
||||
await prisma.scenario.update({
|
||||
where: { id: scenarioId },
|
||||
data: {
|
||||
consecutivePasses,
|
||||
consecutiveFails,
|
||||
qualityScore,
|
||||
totalRuns: totalRuns + 1,
|
||||
lastRunAt: new Date(),
|
||||
lastRunStatus: status,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a scenario
|
||||
*
|
||||
* NOTE: This currently uses simulated execution.
|
||||
* Full agent integration will be added in Phase 3.
|
||||
*
|
||||
* @param scenario The scenario to execute
|
||||
* @param userId The user triggering the execution
|
||||
* @param options Execution options
|
||||
*/
|
||||
export async function executeScenario(
|
||||
scenario: Scenario,
|
||||
userId: string,
|
||||
options: ExecutionOptions = {}
|
||||
): Promise<ExecutionResult> {
|
||||
const { evaluatorModel = DEFAULT_EVALUATOR } = options;
|
||||
|
||||
// Create run record
|
||||
const run = await prisma.scenarioRun.create({
|
||||
data: {
|
||||
scenarioId: scenario.id,
|
||||
userId,
|
||||
status: 'pending',
|
||||
},
|
||||
});
|
||||
|
||||
let lastError: Error | null = null;
|
||||
let retryCount = 0;
|
||||
|
||||
// Retry loop
|
||||
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
||||
retryCount = attempt;
|
||||
|
||||
try {
|
||||
// Update status to running
|
||||
await prisma.scenarioRun.update({
|
||||
where: { id: run.id },
|
||||
data: {
|
||||
status: 'running',
|
||||
startedAt: new Date(),
|
||||
retryCount: attempt,
|
||||
},
|
||||
});
|
||||
|
||||
// TODO: Phase 3 - Replace with actual agent execution
|
||||
// For now, we simulate execution
|
||||
const executionResult = await simulateExecution(scenario);
|
||||
|
||||
// Evaluate with LLM
|
||||
const evaluation = await evaluateScenarioRun(
|
||||
scenario.prompt,
|
||||
executionResult.output,
|
||||
executionResult.conversation,
|
||||
evaluatorModel
|
||||
);
|
||||
|
||||
// Run assertions if defined
|
||||
const assertions = scenario.assertions as {
|
||||
regex?: string[];
|
||||
schema?: Record<string, unknown>;
|
||||
} | null;
|
||||
const assertionResults = assertions
|
||||
? runAssertions(executionResult.output, assertions)
|
||||
: null;
|
||||
|
||||
// Determine final verdict
|
||||
const finalStatus = determineFinalVerdict(evaluation, assertionResults);
|
||||
|
||||
// Update run record
|
||||
const updatedRun = await prisma.scenarioRun.update({
|
||||
where: { id: run.id },
|
||||
data: {
|
||||
status: finalStatus,
|
||||
conversation: executionResult.conversation as object,
|
||||
output: executionResult.output,
|
||||
evaluatorModel,
|
||||
evaluatorVerdict: evaluation.verdict,
|
||||
evaluatorReason: evaluation.reason,
|
||||
assertionResults: assertionResults as object,
|
||||
inputTokens: executionResult.usage.inputTokens,
|
||||
outputTokens: executionResult.usage.outputTokens,
|
||||
totalTokens: executionResult.usage.totalTokens,
|
||||
executionTimeMs: executionResult.durationMs,
|
||||
completedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
// Update scenario metrics
|
||||
await updateScenarioMetrics(scenario.id, finalStatus);
|
||||
|
||||
return { run: updatedRun, success: finalStatus === 'pass' };
|
||||
} catch (error) {
|
||||
lastError = error as Error;
|
||||
|
||||
// If this is the last attempt, mark as error
|
||||
if (attempt === MAX_RETRIES) {
|
||||
const errorRun = await prisma.scenarioRun.update({
|
||||
where: { id: run.id },
|
||||
data: {
|
||||
status: 'error',
|
||||
retryCount,
|
||||
errorLog: (error as Error).stack || (error as Error).message,
|
||||
completedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
await updateScenarioMetrics(scenario.id, 'error');
|
||||
|
||||
return { run: errorRun, success: false };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Should never reach here, but TypeScript needs this
|
||||
throw lastError || new Error('Unknown execution error');
|
||||
}
|
||||
|
||||
/**
|
||||
* Simulated execution for development/testing
|
||||
*
|
||||
* This will be replaced with actual agent execution in Phase 3.
|
||||
*/
|
||||
async function simulateExecution(scenario: Scenario): Promise<{
|
||||
output: string;
|
||||
conversation: unknown[];
|
||||
usage: {
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
totalTokens: number;
|
||||
};
|
||||
durationMs: number;
|
||||
}> {
|
||||
// Simulate some processing time
|
||||
const durationMs = Math.floor(Math.random() * 3000) + 1000;
|
||||
await new Promise((resolve) => setTimeout(resolve, 100)); // Small delay
|
||||
|
||||
// Generate simulated output based on the prompt
|
||||
const passRate = 0.7; // 70% pass rate for simulated runs
|
||||
const willPass = Math.random() < passRate;
|
||||
|
||||
const output = willPass
|
||||
? `[SIMULATED] Successfully completed task: ${scenario.prompt.slice(0, 100)}...\n\nThe scenario was executed successfully. All requested operations were performed.`
|
||||
: `[SIMULATED] Failed to complete task: ${scenario.prompt.slice(0, 100)}...\n\nEncountered an error during execution. The requested operation could not be completed.`;
|
||||
|
||||
const conversation = [
|
||||
{ role: 'user', content: scenario.prompt },
|
||||
{ role: 'assistant', content: output },
|
||||
];
|
||||
|
||||
const inputTokens = Math.floor(scenario.prompt.length / 4);
|
||||
const outputTokens = Math.floor(output.length / 4);
|
||||
|
||||
return {
|
||||
output,
|
||||
conversation,
|
||||
usage: {
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
totalTokens: inputTokens + outputTokens,
|
||||
},
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get user's current quota status
|
||||
*/
|
||||
export async function getQuotaStatus(
|
||||
userId: string
|
||||
): Promise<{ used: number; limit: number; remaining: number; resetsAt: Date }> {
|
||||
const now = new Date();
|
||||
const tomorrow = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1);
|
||||
|
||||
const quota = await prisma.scenarioQuota.findUnique({
|
||||
where: { userId },
|
||||
});
|
||||
|
||||
if (!quota) {
|
||||
return { used: 0, limit: 50, remaining: 50, resetsAt: tomorrow };
|
||||
}
|
||||
|
||||
const startOfDay = new Date(now.getFullYear(), now.getMonth(), now.getDate());
|
||||
const used = quota.lastResetAt < startOfDay ? 0 : quota.dailyUsed;
|
||||
|
||||
return {
|
||||
used,
|
||||
limit: quota.dailyLimit,
|
||||
remaining: quota.dailyLimit - used,
|
||||
resetsAt: tomorrow,
|
||||
};
|
||||
}
|
||||
143
apps/web/src/lib/scenarios/generate-prompt.ts
Normal file
143
apps/web/src/lib/scenarios/generate-prompt.ts
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
/**
|
||||
* AI Prompt Generation Service
|
||||
*
|
||||
* Uses GPT-4o-mini to generate realistic test scenario prompts
|
||||
* for collections based on their tools.
|
||||
*/
|
||||
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import { generateObject, generateText } from 'ai';
|
||||
import { z } from 'zod';
|
||||
|
||||
const GENERATOR_MODEL = 'gpt-4o-mini';
|
||||
|
||||
interface Tool {
|
||||
name: string;
|
||||
description: string | null;
|
||||
}
|
||||
|
||||
interface Collection {
|
||||
name: string;
|
||||
description: string | null;
|
||||
tools: Tool[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a single scenario prompt for a collection
|
||||
*/
|
||||
export async function generateScenarioPrompt(collection: Collection): Promise<string> {
|
||||
const toolDescriptions = collection.tools
|
||||
.map((t) => `- ${t.name}: ${t.description || 'No description'}`)
|
||||
.join('\n');
|
||||
|
||||
const { text } = await generateText({
|
||||
model: openai(GENERATOR_MODEL),
|
||||
prompt: `Generate a realistic test scenario for this tool collection.
|
||||
|
||||
Collection: ${collection.name}
|
||||
Description: ${collection.description || 'No description provided'}
|
||||
|
||||
Available tools:
|
||||
${toolDescriptions}
|
||||
|
||||
Write a single, specific task that a user might want to accomplish using these tools.
|
||||
Be concrete and include example data where helpful (like URLs, file paths, or specific values).
|
||||
The task should be achievable using the available tools.
|
||||
Output only the scenario prompt, nothing else.`,
|
||||
});
|
||||
|
||||
return text.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate multiple scenario prompts with metadata
|
||||
*/
|
||||
export interface GeneratedScenario {
|
||||
prompt: string;
|
||||
name: string;
|
||||
tags: string[];
|
||||
}
|
||||
|
||||
const GeneratedScenarioSchema = z.object({
|
||||
prompt: z.string().describe('The specific task prompt'),
|
||||
name: z.string().describe('A short descriptive name for the scenario (max 50 chars)'),
|
||||
tags: z.array(z.string()).max(5).describe('Relevant tags for categorization'),
|
||||
});
|
||||
|
||||
const GeneratedScenariosSchema = z.object({
|
||||
scenarios: z.array(GeneratedScenarioSchema),
|
||||
});
|
||||
|
||||
/**
|
||||
* Generate multiple scenarios with names and tags
|
||||
*/
|
||||
export async function generateScenarios(
|
||||
collection: Collection,
|
||||
count: number = 1
|
||||
): Promise<GeneratedScenario[]> {
|
||||
const toolDescriptions = collection.tools
|
||||
.map((t) => `- ${t.name}: ${t.description || 'No description'}`)
|
||||
.join('\n');
|
||||
|
||||
const { object } = await generateObject({
|
||||
model: openai(GENERATOR_MODEL),
|
||||
schema: GeneratedScenariosSchema,
|
||||
prompt: `Generate ${count} realistic test scenario${count > 1 ? 's' : ''} for this tool collection.
|
||||
|
||||
Collection: ${collection.name}
|
||||
Description: ${collection.description || 'No description provided'}
|
||||
|
||||
Available tools:
|
||||
${toolDescriptions}
|
||||
|
||||
For each scenario:
|
||||
1. Write a specific, achievable task using the available tools
|
||||
2. Include concrete example data (URLs, file paths, values) where helpful
|
||||
3. Provide a short descriptive name (max 50 characters)
|
||||
4. Add 1-5 relevant tags for categorization
|
||||
|
||||
${count > 1 ? 'Make the scenarios diverse - cover different use cases and tool combinations.' : ''}`,
|
||||
});
|
||||
|
||||
return object.scenarios;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate tags for an existing prompt
|
||||
*/
|
||||
export async function generateTags(prompt: string): Promise<string[]> {
|
||||
const TagsSchema = z.object({
|
||||
tags: z.array(z.string()).min(1).max(5).describe('Relevant tags for the scenario'),
|
||||
});
|
||||
|
||||
const { object } = await generateObject({
|
||||
model: openai(GENERATOR_MODEL),
|
||||
schema: TagsSchema,
|
||||
prompt: `Generate 1-5 relevant tags for this scenario prompt:
|
||||
|
||||
"${prompt}"
|
||||
|
||||
Tags should be:
|
||||
- Lowercase
|
||||
- Single words or short phrases (max 2 words)
|
||||
- Descriptive of the task type, domain, or tools involved`,
|
||||
});
|
||||
|
||||
return object.tags;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a name for an existing prompt
|
||||
*/
|
||||
export async function generateName(prompt: string): Promise<string> {
|
||||
const { text } = await generateText({
|
||||
model: openai(GENERATOR_MODEL),
|
||||
prompt: `Generate a short, descriptive name (max 50 characters) for this scenario:
|
||||
|
||||
"${prompt}"
|
||||
|
||||
Output only the name, nothing else.`,
|
||||
});
|
||||
|
||||
return text.trim().slice(0, 50);
|
||||
}
|
||||
170
apps/web/src/lib/scenarios/similarity.ts
Normal file
170
apps/web/src/lib/scenarios/similarity.ts
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
/**
|
||||
* Scenario Similarity Service
|
||||
*
|
||||
* Uses OpenAI embeddings to detect similar scenarios.
|
||||
* Embeddings are stored in PostgreSQL JSONB for persistence.
|
||||
*/
|
||||
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import type { Scenario, ScenarioEmbedding } from '@prisma/client';
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { embed } from 'ai';
|
||||
|
||||
const EMBEDDING_MODEL = 'text-embedding-3-small';
|
||||
const SIMILARITY_THRESHOLD = 0.7; // 70% similarity triggers warning
|
||||
|
||||
/**
|
||||
* Compute embedding for a text string
|
||||
*/
|
||||
export async function computeEmbedding(text: string): Promise<number[]> {
|
||||
const { embedding } = await embed({
|
||||
model: openai.embedding(EMBEDDING_MODEL),
|
||||
value: text,
|
||||
});
|
||||
return embedding;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute cosine similarity between two vectors
|
||||
*/
|
||||
export function cosineSimilarity(a: number[], b: number[]): number {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error('Vectors must have same length');
|
||||
}
|
||||
|
||||
let dotProduct = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
const aVal = a[i] ?? 0;
|
||||
const bVal = b[i] ?? 0;
|
||||
dotProduct += aVal * bVal;
|
||||
normA += aVal * aVal;
|
||||
normB += bVal * bVal;
|
||||
}
|
||||
|
||||
const denominator = Math.sqrt(normA) * Math.sqrt(normB);
|
||||
if (denominator === 0) return 0;
|
||||
|
||||
return dotProduct / denominator;
|
||||
}
|
||||
|
||||
export interface SimilarScenario {
|
||||
scenario: Scenario;
|
||||
similarity: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find scenarios similar to a given prompt
|
||||
*
|
||||
* @param prompt The prompt to check for similarity
|
||||
* @param collectionId The collection to search within
|
||||
* @param threshold Minimum similarity score (0-1), default 0.7
|
||||
* @param excludeScenarioId Optional scenario ID to exclude (for updates)
|
||||
*/
|
||||
export async function findSimilarScenarios(
|
||||
prompt: string,
|
||||
collectionId: string,
|
||||
threshold: number = SIMILARITY_THRESHOLD,
|
||||
excludeScenarioId?: string
|
||||
): Promise<SimilarScenario[]> {
|
||||
// Compute embedding for the new prompt
|
||||
const newEmbedding = await computeEmbedding(prompt);
|
||||
|
||||
// Get all scenarios with embeddings for this collection
|
||||
const scenarios = await prisma.scenario.findMany({
|
||||
where: {
|
||||
collectionId,
|
||||
...(excludeScenarioId && { id: { not: excludeScenarioId } }),
|
||||
embedding: { isNot: null },
|
||||
},
|
||||
include: {
|
||||
embedding: true,
|
||||
},
|
||||
});
|
||||
|
||||
// Calculate similarity scores
|
||||
const similar: SimilarScenario[] = [];
|
||||
|
||||
for (const scenario of scenarios) {
|
||||
if (!scenario.embedding) continue;
|
||||
|
||||
const existingEmbedding = scenario.embedding.embedding as number[];
|
||||
const similarity = cosineSimilarity(newEmbedding, existingEmbedding);
|
||||
|
||||
if (similarity >= threshold) {
|
||||
// Remove embedding from returned scenario to keep response light
|
||||
const { embedding: _, ...scenarioWithoutEmbedding } = scenario;
|
||||
similar.push({
|
||||
scenario: scenarioWithoutEmbedding as Scenario,
|
||||
similarity,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by similarity descending
|
||||
return similar.sort((a, b) => b.similarity - a.similarity);
|
||||
}
|
||||
|
||||
/**
|
||||
* Store embedding for a scenario
|
||||
*/
|
||||
export async function storeScenarioEmbedding(
|
||||
scenarioId: string,
|
||||
embedding: number[]
|
||||
): Promise<ScenarioEmbedding> {
|
||||
// Upsert to handle both create and update cases
|
||||
return prisma.scenarioEmbedding.upsert({
|
||||
where: { scenarioId },
|
||||
update: {
|
||||
embedding: embedding as unknown as object,
|
||||
model: EMBEDDING_MODEL,
|
||||
},
|
||||
create: {
|
||||
scenarioId,
|
||||
embedding: embedding as unknown as object,
|
||||
model: EMBEDDING_MODEL,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate and store embedding for a scenario
|
||||
*/
|
||||
export async function generateAndStoreEmbedding(
|
||||
scenarioId: string,
|
||||
prompt: string
|
||||
): Promise<ScenarioEmbedding> {
|
||||
const embedding = await computeEmbedding(prompt);
|
||||
return storeScenarioEmbedding(scenarioId, embedding);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check similarity and return warning if similar scenarios exist
|
||||
*/
|
||||
export interface SimilarityCheckResult {
|
||||
hasSimilar: boolean;
|
||||
maxSimilarity: number;
|
||||
similarScenarios: SimilarScenario[];
|
||||
}
|
||||
|
||||
export async function checkSimilarity(
|
||||
prompt: string,
|
||||
collectionId: string,
|
||||
excludeScenarioId?: string
|
||||
): Promise<SimilarityCheckResult> {
|
||||
const similar = await findSimilarScenarios(
|
||||
prompt,
|
||||
collectionId,
|
||||
SIMILARITY_THRESHOLD,
|
||||
excludeScenarioId
|
||||
);
|
||||
|
||||
const firstSimilar = similar[0];
|
||||
return {
|
||||
hasSimilar: similar.length > 0,
|
||||
maxSimilarity: firstSimilar?.similarity ?? 0,
|
||||
similarScenarios: similar.slice(0, 5), // Return top 5 similar
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue