feat: add scenarios system for collection testing
Implements the complete scenarios feature for TPMJS: API endpoints: - GET/POST /api/scenarios - list/create scenarios - GET/PATCH/DELETE /api/scenarios/:id - scenario CRUD - POST /api/scenarios/:id/run - execute scenario - GET /api/scenarios/:id/runs - run history - POST /api/scenarios/check-similarity - vector similarity check - GET /api/scenarios/featured - featured scenarios - POST /api/collections/:id/scenarios/generate - AI scenario generation - GET /api/collections/:id/scenarios - list collection scenarios Services: - generate-prompt.ts - AI prompt generation using GPT-4o-mini - similarity.ts - vector embedding and cosine similarity - evaluate.ts - LLM-based success evaluation - execute.ts - scenario execution orchestration CLI commands: - tpm scenario list [collection] - list scenarios - tpm scenario run <collection> - run all scenarios - tpm scenario test <id> - run single scenario - tpm scenario generate <collection> - generate scenarios - tpm scenario info <id> - scenario details Database: - Scenario, ScenarioEmbedding, ScenarioRun, ScenarioQuota models - Streak-based quality scoring - Daily quota management Migration script included for converting existing useCases.
This commit is contained in:
Ajax Davis
parent
154f000505
commit
ea742c9386
26 changed files with 6922 additions and 282 deletions
2
apps/web/next-env.d.ts
vendored
2
apps/web/next-env.d.ts
vendored
|
|
@ -1,6 +1,6 @@
|
|||
/// <reference types="next" />
|
||||
/// <reference types="next/image-types/global" />
|
||||
import "./.next/types/routes.d.ts";
|
||||
import "./.next/dev/types/routes.d.ts";
|
||||
|
||||
// NOTE: This file should not be edited
|
||||
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,208 @@
|
|||
/**
|
||||
* Scenario Generation API
|
||||
*
|
||||
* POST /api/collections/[id]/scenarios/generate Generate AI-powered scenarios for a collection
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import type { NextRequest } from 'next/server';
|
||||
import { z } from 'zod';
|
||||
import { authenticateRequest } from '~/lib/api-keys/middleware';
|
||||
import {
|
||||
apiForbidden,
|
||||
apiInternalError,
|
||||
apiNotFound,
|
||||
apiSuccess,
|
||||
apiValidationError,
|
||||
} from '~/lib/api-response';
|
||||
import { AI_GENERATION_RATE_LIMIT, checkRateLimitDistributed } from '~/lib/rate-limit';
|
||||
import { generateScenarios } from '~/lib/scenarios/generate-prompt';
|
||||
import { checkSimilarity, generateAndStoreEmbedding } from '~/lib/scenarios/similarity';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 120; // AI generation + embeddings can take time
|
||||
|
||||
type RouteContext = {
|
||||
params: Promise<{ id: string }>;
|
||||
};
|
||||
|
||||
const GenerateRequestSchema = z.object({
|
||||
count: z.number().int().min(1).max(10).default(1),
|
||||
skipSimilarityCheck: z.boolean().default(false),
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /api/collections/[id]/scenarios/generate
|
||||
* Generate AI-powered scenarios for a collection
|
||||
*
|
||||
* Body:
|
||||
* - count: Number of scenarios to generate (1-10, default: 1)
|
||||
* - skipSimilarityCheck: If true, skip duplicate warning (default: false)
|
||||
*
|
||||
* Returns generated scenarios with similarity warnings if applicable
|
||||
*/
|
||||
export async function POST(request: NextRequest, context: RouteContext) {
|
||||
const requestId = crypto.randomUUID();
|
||||
|
||||
try {
|
||||
// Check authentication
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
if (!authResult.authenticated || !authResult.userId) {
|
||||
return apiForbidden('Authentication required to generate scenarios', requestId);
|
||||
}
|
||||
|
||||
// Rate limit check (strict for AI operations)
|
||||
const rateLimitResponse = await checkRateLimitDistributed(request, AI_GENERATION_RATE_LIMIT);
|
||||
if (rateLimitResponse) {
|
||||
return rateLimitResponse;
|
||||
}
|
||||
|
||||
const { id } = await context.params;
|
||||
|
||||
// Parse request body
|
||||
let body: { count?: number; skipSimilarityCheck?: boolean } = {};
|
||||
try {
|
||||
body = await request.json();
|
||||
} catch {
|
||||
// Empty body is fine, use defaults
|
||||
}
|
||||
|
||||
const parseResult = GenerateRequestSchema.safeParse(body);
|
||||
if (!parseResult.success) {
|
||||
return apiValidationError(
|
||||
'Invalid request body',
|
||||
{ errors: parseResult.error.flatten().fieldErrors },
|
||||
requestId
|
||||
);
|
||||
}
|
||||
|
||||
const { count, skipSimilarityCheck } = parseResult.data;
|
||||
|
||||
// Fetch collection with tools
|
||||
const collection = await prisma.collection.findUnique({
|
||||
where: { id },
|
||||
include: {
|
||||
tools: {
|
||||
include: {
|
||||
tool: {
|
||||
select: {
|
||||
name: true,
|
||||
description: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
orderBy: { position: 'asc' },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!collection) {
|
||||
return apiNotFound('Collection', requestId);
|
||||
}
|
||||
|
||||
// User must own the collection to generate scenarios
|
||||
if (collection.userId !== authResult.userId) {
|
||||
return apiForbidden('You can only generate scenarios for your own collections', requestId);
|
||||
}
|
||||
|
||||
// Validate collection has tools
|
||||
if (collection.tools.length === 0) {
|
||||
return apiForbidden(
|
||||
'Collection must have at least one tool to generate scenarios',
|
||||
requestId
|
||||
);
|
||||
}
|
||||
|
||||
// Prepare collection info for AI generation
|
||||
const collectionInfo = {
|
||||
name: collection.name,
|
||||
description: collection.description,
|
||||
tools: collection.tools.map((ct) => ({
|
||||
name: ct.tool.name,
|
||||
description: ct.tool.description,
|
||||
})),
|
||||
};
|
||||
|
||||
// Generate scenarios with AI
|
||||
const generatedScenarios = await generateScenarios(collectionInfo, count);
|
||||
|
||||
// Process each generated scenario
|
||||
const results: Array<{
|
||||
scenario: {
|
||||
id: string;
|
||||
prompt: string;
|
||||
name: string | null;
|
||||
tags: string[];
|
||||
};
|
||||
similarity?: {
|
||||
hasSimilar: boolean;
|
||||
maxSimilarity: number;
|
||||
similar: Array<{ id: string; name: string | null; similarity: number }>;
|
||||
};
|
||||
}> = [];
|
||||
|
||||
for (const generated of generatedScenarios) {
|
||||
// Check similarity unless skipped
|
||||
let similarityResult: {
|
||||
hasSimilar: boolean;
|
||||
maxSimilarity: number;
|
||||
similarScenarios: Array<{
|
||||
scenario: { id: string; name: string | null };
|
||||
similarity: number;
|
||||
}>;
|
||||
} | null = null;
|
||||
|
||||
if (!skipSimilarityCheck) {
|
||||
similarityResult = await checkSimilarity(generated.prompt, collection.id);
|
||||
}
|
||||
|
||||
// Create the scenario
|
||||
const scenario = await prisma.scenario.create({
|
||||
data: {
|
||||
collectionId: collection.id,
|
||||
prompt: generated.prompt,
|
||||
name: generated.name,
|
||||
tags: generated.tags,
|
||||
},
|
||||
});
|
||||
|
||||
// Generate and store embedding for future similarity checks
|
||||
await generateAndStoreEmbedding(scenario.id, generated.prompt);
|
||||
|
||||
results.push({
|
||||
scenario: {
|
||||
id: scenario.id,
|
||||
prompt: scenario.prompt,
|
||||
name: scenario.name,
|
||||
tags: scenario.tags,
|
||||
},
|
||||
...(similarityResult && similarityResult.hasSimilar
|
||||
? {
|
||||
similarity: {
|
||||
hasSimilar: true,
|
||||
maxSimilarity: Math.round(similarityResult.maxSimilarity * 100),
|
||||
similar: similarityResult.similarScenarios.slice(0, 3).map((s) => ({
|
||||
id: s.scenario.id,
|
||||
name: s.scenario.name,
|
||||
similarity: Math.round(s.similarity * 100),
|
||||
})),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
return apiSuccess(
|
||||
{
|
||||
scenarios: results,
|
||||
generatedAt: new Date().toISOString(),
|
||||
},
|
||||
{ requestId, status: 201 }
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('[API Error] POST /api/collections/[id]/scenarios/generate:', error);
|
||||
return apiInternalError('Failed to generate scenarios', requestId);
|
||||
}
|
||||
}
|
||||
141
apps/web/src/app/api/collections/[id]/scenarios/route.ts
Normal file
141
apps/web/src/app/api/collections/[id]/scenarios/route.ts
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
/**
|
||||
* Collection Scenarios API
|
||||
*
|
||||
* GET /api/collections/[id]/scenarios List scenarios for a collection
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import type { NextRequest } from 'next/server';
|
||||
import { authenticateRequest } from '~/lib/api-keys/middleware';
|
||||
import { apiForbidden, apiInternalError, apiNotFound, apiSuccess } from '~/lib/api-response';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 30;
|
||||
|
||||
type RouteContext = {
|
||||
params: Promise<{ id: string }>;
|
||||
};
|
||||
|
||||
/**
|
||||
* GET /api/collections/[id]/scenarios
|
||||
* List scenarios for a collection
|
||||
*
|
||||
* Query params:
|
||||
* - limit: Max results (default: 50, max: 100)
|
||||
* - offset: Pagination offset (default: 0)
|
||||
* - status: Filter by lastRunStatus (optional)
|
||||
*/
|
||||
export async function GET(request: NextRequest, context: RouteContext) {
|
||||
const requestId = crypto.randomUUID();
|
||||
|
||||
try {
|
||||
const { id } = await context.params;
|
||||
const { searchParams } = new URL(request.url);
|
||||
const limit = Math.min(
|
||||
Math.max(Number.parseInt(searchParams.get('limit') || '50', 10), 1),
|
||||
100
|
||||
);
|
||||
const offset = Math.max(Number.parseInt(searchParams.get('offset') || '0', 10), 0);
|
||||
const statusFilter = searchParams.get('status');
|
||||
|
||||
// Check authentication (optional - affects what data is shown)
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
// Fetch collection
|
||||
const collection = await prisma.collection.findUnique({
|
||||
where: { id },
|
||||
select: {
|
||||
id: true,
|
||||
isPublic: true,
|
||||
userId: true,
|
||||
name: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!collection) {
|
||||
return apiNotFound('Collection', requestId);
|
||||
}
|
||||
|
||||
// Check access
|
||||
const isOwner = authResult.authenticated && collection.userId === authResult.userId;
|
||||
if (!collection.isPublic && !isOwner) {
|
||||
return apiForbidden('Access denied', requestId);
|
||||
}
|
||||
|
||||
// Build where clause
|
||||
const where: {
|
||||
collectionId: string;
|
||||
lastRunStatus?: string;
|
||||
} = { collectionId: id };
|
||||
|
||||
if (statusFilter) {
|
||||
where.lastRunStatus = statusFilter;
|
||||
}
|
||||
|
||||
// Fetch scenarios with pagination
|
||||
const scenarios = await prisma.scenario.findMany({
|
||||
where,
|
||||
orderBy: [{ qualityScore: 'desc' }, { createdAt: 'desc' }],
|
||||
take: limit + 1,
|
||||
skip: offset,
|
||||
select: {
|
||||
id: true,
|
||||
prompt: true,
|
||||
name: true,
|
||||
description: true,
|
||||
tags: true,
|
||||
qualityScore: true,
|
||||
totalRuns: true,
|
||||
lastRunAt: true,
|
||||
lastRunStatus: true,
|
||||
consecutivePasses: true,
|
||||
consecutiveFails: true,
|
||||
createdAt: true,
|
||||
updatedAt: true,
|
||||
},
|
||||
});
|
||||
|
||||
const hasMore = scenarios.length > limit;
|
||||
const data = hasMore ? scenarios.slice(0, limit) : scenarios;
|
||||
|
||||
return apiSuccess(
|
||||
{
|
||||
collection: {
|
||||
id: collection.id,
|
||||
name: collection.name,
|
||||
},
|
||||
scenarios: data.map((s) => ({
|
||||
id: s.id,
|
||||
prompt: s.prompt.slice(0, 200) + (s.prompt.length > 200 ? '...' : ''),
|
||||
name: s.name,
|
||||
description: s.description,
|
||||
tags: s.tags,
|
||||
metrics: {
|
||||
qualityScore: s.qualityScore,
|
||||
totalRuns: s.totalRuns,
|
||||
consecutivePasses: s.consecutivePasses,
|
||||
consecutiveFails: s.consecutiveFails,
|
||||
lastRunStatus: s.lastRunStatus,
|
||||
lastRunAt: s.lastRunAt,
|
||||
},
|
||||
timestamps: {
|
||||
createdAt: s.createdAt,
|
||||
updatedAt: s.updatedAt,
|
||||
},
|
||||
})),
|
||||
},
|
||||
{
|
||||
requestId,
|
||||
pagination: {
|
||||
limit,
|
||||
offset,
|
||||
hasMore,
|
||||
},
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('[API Error] GET /api/collections/[id]/scenarios:', error);
|
||||
return apiInternalError('Failed to fetch scenarios', requestId);
|
||||
}
|
||||
}
|
||||
405
apps/web/src/app/api/scenarios/[id]/route.ts
Normal file
405
apps/web/src/app/api/scenarios/[id]/route.ts
Normal file
|
|
@ -0,0 +1,405 @@
|
|||
/**
|
||||
* Scenario API - Get, Update, Delete individual scenario
|
||||
*
|
||||
* GET /api/scenarios/[id] Get scenario details
|
||||
* PATCH /api/scenarios/[id] Update scenario
|
||||
* DELETE /api/scenarios/[id] Delete scenario
|
||||
*/
|
||||
|
||||
import { Prisma, prisma } from '@tpmjs/db';
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
import { z } from 'zod';
|
||||
import { authenticateRequest } from '~/lib/api-keys/middleware';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 60;
|
||||
|
||||
const API_VERSION = '1.0.0';
|
||||
|
||||
interface ApiResponse<T = unknown> {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
error?: {
|
||||
code: string;
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
meta: {
|
||||
version: string;
|
||||
timestamp: string;
|
||||
requestId?: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface RouteContext {
|
||||
params: Promise<{ id: string }>;
|
||||
}
|
||||
|
||||
// Validation schema for updating a scenario
|
||||
const UpdateScenarioSchema = z.object({
|
||||
prompt: z.string().min(10).optional(),
|
||||
name: z.string().max(200).nullish(),
|
||||
description: z.string().nullish(),
|
||||
assertions: z
|
||||
.object({
|
||||
regex: z.array(z.string()).optional(),
|
||||
schema: z.record(z.string(), z.unknown()).optional(),
|
||||
})
|
||||
.nullish(),
|
||||
tags: z.array(z.string()).optional(),
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/scenarios/[id]
|
||||
* Get scenario details with recent runs
|
||||
*
|
||||
* Query params:
|
||||
* - runsLimit: Max runs to return (default: 10, max: 50)
|
||||
*/
|
||||
export async function GET(
|
||||
request: NextRequest,
|
||||
context: RouteContext
|
||||
): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
const { id } = await context.params;
|
||||
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const runsLimit = Math.min(
|
||||
Math.max(Number.parseInt(searchParams.get('runsLimit') || '10', 10), 1),
|
||||
50
|
||||
);
|
||||
|
||||
// Check authentication (optional - affects what data is shown)
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
const scenario = await prisma.scenario.findUnique({
|
||||
where: { id },
|
||||
include: {
|
||||
collection: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
slug: true,
|
||||
isPublic: true,
|
||||
userId: true,
|
||||
user: {
|
||||
select: {
|
||||
username: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
runs: {
|
||||
orderBy: { createdAt: 'desc' },
|
||||
take: runsLimit,
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
evaluatorVerdict: true,
|
||||
executionTimeMs: true,
|
||||
totalTokens: true,
|
||||
startedAt: true,
|
||||
completedAt: true,
|
||||
createdAt: true,
|
||||
},
|
||||
},
|
||||
_count: {
|
||||
select: { runs: true },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!scenario) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'NOT_FOUND', message: 'Scenario not found' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
// Check access - collection must be public OR user must own it
|
||||
const isOwner = authResult.authenticated && scenario.collection?.userId === authResult.userId;
|
||||
const isPublic = scenario.collection?.isPublic ?? false;
|
||||
|
||||
if (!isOwner && !isPublic) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'FORBIDDEN', message: 'Access denied' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
id: scenario.id,
|
||||
collectionId: scenario.collectionId,
|
||||
prompt: scenario.prompt,
|
||||
name: scenario.name,
|
||||
description: scenario.description,
|
||||
assertions: scenario.assertions,
|
||||
tags: scenario.tags,
|
||||
qualityScore: scenario.qualityScore,
|
||||
consecutivePasses: scenario.consecutivePasses,
|
||||
consecutiveFails: scenario.consecutiveFails,
|
||||
totalRuns: scenario.totalRuns,
|
||||
lastRunAt: scenario.lastRunAt,
|
||||
lastRunStatus: scenario.lastRunStatus,
|
||||
createdAt: scenario.createdAt,
|
||||
updatedAt: scenario.updatedAt,
|
||||
isOwner,
|
||||
collection: scenario.collection
|
||||
? {
|
||||
id: scenario.collection.id,
|
||||
name: scenario.collection.name,
|
||||
slug: scenario.collection.slug,
|
||||
username: scenario.collection.user.username,
|
||||
}
|
||||
: null,
|
||||
recentRuns: scenario.runs,
|
||||
runCount: scenario._count.runs,
|
||||
},
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[API Error] GET /api/scenarios/[id]:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to fetch scenario' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* PATCH /api/scenarios/[id]
|
||||
* Update a scenario
|
||||
*
|
||||
* Requires authentication and ownership of the collection
|
||||
*/
|
||||
export async function PATCH(
|
||||
request: NextRequest,
|
||||
context: RouteContext
|
||||
): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
const { id } = await context.params;
|
||||
|
||||
try {
|
||||
// Check authentication
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
if (!authResult.authenticated || !authResult.userId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'UNAUTHORIZED', message: 'Authentication required' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 401 }
|
||||
);
|
||||
}
|
||||
|
||||
// Find scenario and check ownership
|
||||
const existing = await prisma.scenario.findUnique({
|
||||
where: { id },
|
||||
include: {
|
||||
collection: {
|
||||
select: { userId: true },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!existing) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'NOT_FOUND', message: 'Scenario not found' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
if (existing.collection?.userId !== authResult.userId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'FORBIDDEN', message: 'Access denied' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
|
||||
// Parse and validate request body
|
||||
const body = await request.json();
|
||||
const parseResult = UpdateScenarioSchema.safeParse(body);
|
||||
|
||||
if (!parseResult.success) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: {
|
||||
code: 'VALIDATION_ERROR',
|
||||
message: 'Invalid request body',
|
||||
details: { errors: parseResult.error.flatten().fieldErrors },
|
||||
},
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const { prompt, name, description, assertions, tags } = parseResult.data;
|
||||
|
||||
// Update scenario (transform null to Prisma.JsonNull for JSON fields)
|
||||
const scenario = await prisma.scenario.update({
|
||||
where: { id },
|
||||
data: {
|
||||
...(prompt !== undefined && { prompt }),
|
||||
...(name !== undefined && { name }),
|
||||
...(description !== undefined && { description }),
|
||||
...(assertions !== undefined && {
|
||||
assertions:
|
||||
assertions === null
|
||||
? Prisma.JsonNull
|
||||
: (assertions as unknown as Prisma.InputJsonValue),
|
||||
}),
|
||||
...(tags !== undefined && { tags }),
|
||||
},
|
||||
});
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
id: scenario.id,
|
||||
collectionId: scenario.collectionId,
|
||||
prompt: scenario.prompt,
|
||||
name: scenario.name,
|
||||
description: scenario.description,
|
||||
assertions: scenario.assertions,
|
||||
tags: scenario.tags,
|
||||
qualityScore: scenario.qualityScore,
|
||||
totalRuns: scenario.totalRuns,
|
||||
updatedAt: scenario.updatedAt,
|
||||
},
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[API Error] PATCH /api/scenarios/[id]:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to update scenario' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* DELETE /api/scenarios/[id]
|
||||
* Delete a scenario
|
||||
*
|
||||
* Requires authentication and ownership of the collection
|
||||
*/
|
||||
export async function DELETE(
|
||||
_request: NextRequest,
|
||||
context: RouteContext
|
||||
): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
const { id } = await context.params;
|
||||
|
||||
try {
|
||||
// Check authentication
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
if (!authResult.authenticated || !authResult.userId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'UNAUTHORIZED', message: 'Authentication required' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 401 }
|
||||
);
|
||||
}
|
||||
|
||||
// Find scenario and check ownership
|
||||
const existing = await prisma.scenario.findUnique({
|
||||
where: { id },
|
||||
include: {
|
||||
collection: {
|
||||
select: { userId: true },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!existing) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'NOT_FOUND', message: 'Scenario not found' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
if (existing.collection?.userId !== authResult.userId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'FORBIDDEN', message: 'Access denied' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
|
||||
// Delete scenario (cascade will delete runs and embedding)
|
||||
await prisma.scenario.delete({
|
||||
where: { id },
|
||||
});
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: { deleted: true },
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[API Error] DELETE /api/scenarios/[id]:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to delete scenario' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
178
apps/web/src/app/api/scenarios/[id]/run/route.ts
Normal file
178
apps/web/src/app/api/scenarios/[id]/run/route.ts
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
/**
|
||||
* Scenario Run API - Trigger a scenario execution
|
||||
*
|
||||
* POST /api/scenarios/[id]/run Execute a scenario
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
import { authenticateRequest } from '~/lib/api-keys/middleware';
|
||||
import { checkAndDecrementQuota, executeScenario, getQuotaStatus } from '~/lib/scenarios/execute';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 300; // 5 minutes for scenario execution
|
||||
|
||||
const API_VERSION = '1.0.0';
|
||||
|
||||
interface ApiResponse<T = unknown> {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
error?: {
|
||||
code: string;
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
meta: {
|
||||
version: string;
|
||||
timestamp: string;
|
||||
requestId?: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface RouteContext {
|
||||
params: Promise<{ id: string }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/scenarios/[id]/run
|
||||
* Execute a scenario
|
||||
*
|
||||
* Requires authentication
|
||||
* Subject to daily quota limits
|
||||
*/
|
||||
export async function POST(
|
||||
_request: NextRequest,
|
||||
context: RouteContext
|
||||
): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
const { id: scenarioId } = await context.params;
|
||||
|
||||
try {
|
||||
// Check authentication
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
if (!authResult.authenticated || !authResult.userId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'UNAUTHORIZED', message: 'Authentication required' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 401 }
|
||||
);
|
||||
}
|
||||
|
||||
// Check scenario exists
|
||||
const scenario = await prisma.scenario.findUnique({
|
||||
where: { id: scenarioId },
|
||||
include: {
|
||||
collection: {
|
||||
select: {
|
||||
id: true,
|
||||
isPublic: true,
|
||||
userId: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!scenario) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'NOT_FOUND', message: 'Scenario not found' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
// Check access - must be owner or collection must be public
|
||||
const isOwner = scenario.collection?.userId === authResult.userId;
|
||||
const isPublic = scenario.collection?.isPublic ?? false;
|
||||
|
||||
if (!isOwner && !isPublic) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'FORBIDDEN', message: 'Access denied' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
|
||||
// Check quota
|
||||
const quota = await checkAndDecrementQuota(authResult.userId);
|
||||
|
||||
if (!quota.allowed) {
|
||||
const quotaStatus = await getQuotaStatus(authResult.userId);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: {
|
||||
code: 'QUOTA_EXCEEDED',
|
||||
message: 'Daily scenario run quota exceeded. Try again tomorrow.',
|
||||
details: {
|
||||
used: quotaStatus.used,
|
||||
limit: quotaStatus.limit,
|
||||
resetsAt: quotaStatus.resetsAt.toISOString(),
|
||||
},
|
||||
},
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 429 }
|
||||
);
|
||||
}
|
||||
|
||||
// Execute the scenario
|
||||
const { run, success } = await executeScenario(scenario, authResult.userId);
|
||||
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: true,
|
||||
data: {
|
||||
runId: run.id,
|
||||
status: run.status,
|
||||
success,
|
||||
evaluator: {
|
||||
model: run.evaluatorModel,
|
||||
verdict: run.evaluatorVerdict,
|
||||
reason: run.evaluatorReason,
|
||||
},
|
||||
assertions: run.assertionResults,
|
||||
usage: {
|
||||
inputTokens: run.inputTokens,
|
||||
outputTokens: run.outputTokens,
|
||||
totalTokens: run.totalTokens,
|
||||
executionTimeMs: run.executionTimeMs,
|
||||
},
|
||||
timestamps: {
|
||||
startedAt: run.startedAt,
|
||||
completedAt: run.completedAt,
|
||||
createdAt: run.createdAt,
|
||||
},
|
||||
quotaRemaining: quota.remaining,
|
||||
},
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
note: 'Execution uses simulated agent. Full agent integration coming soon.',
|
||||
},
|
||||
},
|
||||
{ status: 201 }
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('[API Error] POST /api/scenarios/[id]/run:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to execute scenario' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
196
apps/web/src/app/api/scenarios/[id]/runs/route.ts
Normal file
196
apps/web/src/app/api/scenarios/[id]/runs/route.ts
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
/**
|
||||
* Scenario Runs API - List run history
|
||||
*
|
||||
* GET /api/scenarios/[id]/runs Get all runs for a scenario (paginated)
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
import { authenticateRequest } from '~/lib/api-keys/middleware';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 60;
|
||||
|
||||
const API_VERSION = '1.0.0';
|
||||
|
||||
interface ApiResponse<T = unknown> {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
error?: {
|
||||
code: string;
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
meta: {
|
||||
version: string;
|
||||
timestamp: string;
|
||||
requestId?: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface RouteContext {
|
||||
params: Promise<{ id: string }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/scenarios/[id]/runs
|
||||
* Get run history for a scenario
|
||||
*
|
||||
* Query params:
|
||||
* - limit: Max results (default: 20, max: 100)
|
||||
* - offset: Pagination offset (default: 0)
|
||||
* - status: Filter by status (optional)
|
||||
*/
|
||||
export async function GET(
|
||||
request: NextRequest,
|
||||
context: RouteContext
|
||||
): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
const { id: scenarioId } = await context.params;
|
||||
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const limit = Math.min(
|
||||
Math.max(Number.parseInt(searchParams.get('limit') || '20', 10), 1),
|
||||
100
|
||||
);
|
||||
const offset = Math.max(Number.parseInt(searchParams.get('offset') || '0', 10), 0);
|
||||
const statusFilter = searchParams.get('status');
|
||||
|
||||
// Check authentication (optional - affects what data is shown)
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
// Verify scenario exists and check access
|
||||
const scenario = await prisma.scenario.findUnique({
|
||||
where: { id: scenarioId },
|
||||
include: {
|
||||
collection: {
|
||||
select: {
|
||||
isPublic: true,
|
||||
userId: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!scenario) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'NOT_FOUND', message: 'Scenario not found' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
// Check access
|
||||
const isOwner = authResult.authenticated && scenario.collection?.userId === authResult.userId;
|
||||
const isPublic = scenario.collection?.isPublic ?? false;
|
||||
|
||||
if (!isOwner && !isPublic) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'FORBIDDEN', message: 'Access denied' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
|
||||
// Build where clause
|
||||
const where: { scenarioId: string; status?: string } = { scenarioId };
|
||||
if (statusFilter) {
|
||||
where.status = statusFilter;
|
||||
}
|
||||
|
||||
// Fetch runs
|
||||
const runs = await prisma.scenarioRun.findMany({
|
||||
where,
|
||||
orderBy: { createdAt: 'desc' },
|
||||
take: limit + 1,
|
||||
skip: offset,
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
retryCount: true,
|
||||
evaluatorModel: true,
|
||||
evaluatorVerdict: true,
|
||||
evaluatorReason: true,
|
||||
assertionResults: true,
|
||||
inputTokens: true,
|
||||
outputTokens: true,
|
||||
totalTokens: true,
|
||||
executionTimeMs: true,
|
||||
estimatedCost: true,
|
||||
startedAt: true,
|
||||
completedAt: true,
|
||||
createdAt: true,
|
||||
// Only include sensitive data if owner
|
||||
...(isOwner && {
|
||||
output: true,
|
||||
errorLog: true,
|
||||
conversation: true,
|
||||
}),
|
||||
},
|
||||
});
|
||||
|
||||
const hasMore = runs.length > limit;
|
||||
const data = hasMore ? runs.slice(0, limit) : runs;
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: data.map((run) => ({
|
||||
id: run.id,
|
||||
status: run.status,
|
||||
retryCount: run.retryCount,
|
||||
evaluator: {
|
||||
model: run.evaluatorModel,
|
||||
verdict: run.evaluatorVerdict,
|
||||
reason: run.evaluatorReason,
|
||||
},
|
||||
assertions: run.assertionResults,
|
||||
usage: {
|
||||
inputTokens: run.inputTokens,
|
||||
outputTokens: run.outputTokens,
|
||||
totalTokens: run.totalTokens,
|
||||
executionTimeMs: run.executionTimeMs,
|
||||
estimatedCost: run.estimatedCost,
|
||||
},
|
||||
timestamps: {
|
||||
startedAt: run.startedAt,
|
||||
completedAt: run.completedAt,
|
||||
createdAt: run.createdAt,
|
||||
},
|
||||
// Only include if owner
|
||||
...('output' in run && {
|
||||
output: run.output,
|
||||
errorLog: run.errorLog,
|
||||
conversation: run.conversation,
|
||||
}),
|
||||
})),
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
},
|
||||
pagination: {
|
||||
limit,
|
||||
offset,
|
||||
hasMore,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[API Error] GET /api/scenarios/[id]/runs:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to fetch runs' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
147
apps/web/src/app/api/scenarios/check-similarity/route.ts
Normal file
147
apps/web/src/app/api/scenarios/check-similarity/route.ts
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
/**
|
||||
* Scenario Similarity Check API
|
||||
*
|
||||
* POST /api/scenarios/check-similarity Check if a prompt is similar to existing scenarios
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
import { z } from 'zod';
|
||||
import { authenticateRequest } from '~/lib/api-keys/middleware';
|
||||
import { checkSimilarity } from '~/lib/scenarios/similarity';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 30;
|
||||
|
||||
const API_VERSION = '1.0.0';
|
||||
|
||||
interface ApiResponse<T = unknown> {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
error?: {
|
||||
code: string;
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
meta: {
|
||||
version: string;
|
||||
timestamp: string;
|
||||
requestId?: string;
|
||||
};
|
||||
}
|
||||
|
||||
const CheckSimilaritySchema = z.object({
|
||||
prompt: z.string().min(10, 'Prompt must be at least 10 characters'),
|
||||
collectionId: z.string().min(1, 'Collection ID is required'),
|
||||
excludeScenarioId: z.string().optional(), // For updates, exclude the scenario being edited
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /api/scenarios/check-similarity
|
||||
* Check if a prompt is similar to existing scenarios
|
||||
*
|
||||
* Returns warning if similarity >= 70%
|
||||
*/
|
||||
export async function POST(request: NextRequest): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
|
||||
try {
|
||||
// Check authentication
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
if (!authResult.authenticated || !authResult.userId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'UNAUTHORIZED', message: 'Authentication required' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 401 }
|
||||
);
|
||||
}
|
||||
|
||||
// Parse and validate request body
|
||||
const body = await request.json();
|
||||
const parseResult = CheckSimilaritySchema.safeParse(body);
|
||||
|
||||
if (!parseResult.success) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: {
|
||||
code: 'VALIDATION_ERROR',
|
||||
message: 'Invalid request body',
|
||||
details: { errors: parseResult.error.flatten().fieldErrors },
|
||||
},
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const { prompt, collectionId, excludeScenarioId } = parseResult.data;
|
||||
|
||||
// Verify collection exists and user has access
|
||||
const collection = await prisma.collection.findUnique({
|
||||
where: { id: collectionId },
|
||||
select: { id: true, userId: true, isPublic: true },
|
||||
});
|
||||
|
||||
if (!collection) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'NOT_FOUND', message: 'Collection not found' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
// User must own the collection or it must be public
|
||||
if (collection.userId !== authResult.userId && !collection.isPublic) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'FORBIDDEN', message: 'Access denied' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
|
||||
// Check similarity
|
||||
const result = await checkSimilarity(prompt, collectionId, excludeScenarioId);
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
hasSimilar: result.hasSimilar,
|
||||
maxSimilarity: Math.round(result.maxSimilarity * 100), // Return as percentage
|
||||
warningThreshold: 70,
|
||||
similar: result.similarScenarios.map((s) => ({
|
||||
id: s.scenario.id,
|
||||
name: s.scenario.name,
|
||||
prompt: s.scenario.prompt.slice(0, 200) + (s.scenario.prompt.length > 200 ? '...' : ''),
|
||||
similarity: Math.round(s.similarity * 100),
|
||||
})),
|
||||
},
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[API Error] POST /api/scenarios/check-similarity:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to check similarity' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
168
apps/web/src/app/api/scenarios/featured/route.ts
Normal file
168
apps/web/src/app/api/scenarios/featured/route.ts
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
/**
|
||||
* Featured Scenarios API
|
||||
*
|
||||
* GET /api/scenarios/featured Get featured scenarios for homepage showcase
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 30;
|
||||
|
||||
const API_VERSION = '1.0.0';
|
||||
|
||||
interface ApiResponse<T = unknown> {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
error?: {
|
||||
code: string;
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
meta: {
|
||||
version: string;
|
||||
timestamp: string;
|
||||
requestId?: string;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/scenarios/featured
|
||||
* Get featured scenarios for homepage showcase
|
||||
*
|
||||
* Returns an algorithmic mix of:
|
||||
* - High quality scenarios (by qualityScore)
|
||||
* - Diverse scenarios (different collections/tags)
|
||||
* - Fresh scenarios (recently created)
|
||||
*
|
||||
* Query params:
|
||||
* - limit: Max results (default: 6, max: 20)
|
||||
*/
|
||||
export async function GET(request: NextRequest): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const limit = Math.min(Math.max(Number.parseInt(searchParams.get('limit') || '6', 10), 1), 20);
|
||||
|
||||
// Get high quality scenarios (top 40%)
|
||||
const highQuality = await prisma.scenario.findMany({
|
||||
where: {
|
||||
collection: { isPublic: true },
|
||||
qualityScore: { gte: 0.3 },
|
||||
totalRuns: { gte: 1 },
|
||||
},
|
||||
orderBy: { qualityScore: 'desc' },
|
||||
take: Math.ceil(limit * 0.4),
|
||||
include: {
|
||||
collection: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
slug: true,
|
||||
user: { select: { username: true } },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Get diverse scenarios - different collections (30%)
|
||||
const seenCollections = new Set(highQuality.map((s) => s.collectionId));
|
||||
const diverse = await prisma.scenario.findMany({
|
||||
where: {
|
||||
collection: { isPublic: true },
|
||||
collectionId: { notIn: Array.from(seenCollections).filter(Boolean) as string[] },
|
||||
totalRuns: { gte: 1 },
|
||||
},
|
||||
orderBy: { qualityScore: 'desc' },
|
||||
take: Math.ceil(limit * 0.3),
|
||||
include: {
|
||||
collection: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
slug: true,
|
||||
user: { select: { username: true } },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Get fresh scenarios (30%)
|
||||
const seenIds = new Set([...highQuality, ...diverse].map((s) => s.id));
|
||||
const fresh = await prisma.scenario.findMany({
|
||||
where: {
|
||||
collection: { isPublic: true },
|
||||
id: { notIn: Array.from(seenIds) },
|
||||
},
|
||||
orderBy: { createdAt: 'desc' },
|
||||
take: Math.ceil(limit * 0.3),
|
||||
include: {
|
||||
collection: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
slug: true,
|
||||
user: { select: { username: true } },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Combine and shuffle slightly for variety
|
||||
const combined = [...highQuality, ...diverse, ...fresh];
|
||||
|
||||
// Simple shuffle to mix the categories
|
||||
for (let i = combined.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(Math.random() * (i + 1));
|
||||
const temp = combined[i];
|
||||
const swapWith = combined[j];
|
||||
if (temp !== undefined && swapWith !== undefined) {
|
||||
combined[i] = swapWith;
|
||||
combined[j] = temp;
|
||||
}
|
||||
}
|
||||
|
||||
// Take only the requested limit
|
||||
const featured = combined.slice(0, limit);
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: featured.map((s) => ({
|
||||
id: s.id,
|
||||
name: s.name,
|
||||
prompt: s.prompt.slice(0, 150) + (s.prompt.length > 150 ? '...' : ''),
|
||||
tags: s.tags.slice(0, 3),
|
||||
qualityScore: s.qualityScore,
|
||||
totalRuns: s.totalRuns,
|
||||
lastRunStatus: s.lastRunStatus,
|
||||
collection: s.collection
|
||||
? {
|
||||
id: s.collection.id,
|
||||
name: s.collection.name,
|
||||
slug: s.collection.slug,
|
||||
username: s.collection.user.username,
|
||||
}
|
||||
: null,
|
||||
})),
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
algorithm: 'mixed-quality-diversity-freshness',
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[API Error] GET /api/scenarios/featured:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to fetch featured scenarios' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
313
apps/web/src/app/api/scenarios/route.ts
Normal file
313
apps/web/src/app/api/scenarios/route.ts
Normal file
|
|
@ -0,0 +1,313 @@
|
|||
/**
|
||||
* Scenarios API - List and Create
|
||||
*
|
||||
* GET /api/scenarios List all public scenarios (paginated)
|
||||
* POST /api/scenarios Create a new scenario
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
import { z } from 'zod';
|
||||
import { authenticateRequest } from '~/lib/api-keys/middleware';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 60;
|
||||
|
||||
const API_VERSION = '1.0.0';
|
||||
|
||||
interface ApiResponse<T = unknown> {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
error?: {
|
||||
code: string;
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
meta: {
|
||||
version: string;
|
||||
timestamp: string;
|
||||
requestId?: string;
|
||||
};
|
||||
}
|
||||
|
||||
// Validation schema for creating a scenario
|
||||
const CreateScenarioSchema = z.object({
|
||||
collectionId: z.string().min(1, 'Collection ID is required'),
|
||||
prompt: z.string().min(10, 'Prompt must be at least 10 characters'),
|
||||
name: z.string().max(200).optional(),
|
||||
description: z.string().optional(),
|
||||
assertions: z
|
||||
.object({
|
||||
regex: z.array(z.string()).optional(),
|
||||
schema: z.record(z.string(), z.unknown()).optional(),
|
||||
})
|
||||
.optional(),
|
||||
tags: z.array(z.string()).optional(),
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/scenarios
|
||||
* List all public scenarios (paginated)
|
||||
*
|
||||
* Query params:
|
||||
* - limit: Max results (default: 20, max: 100)
|
||||
* - offset: Pagination offset (default: 0)
|
||||
* - collectionId: Filter by collection (optional)
|
||||
* - tags: Comma-separated tags to filter by (optional)
|
||||
* - sortBy: Sort field (default: 'qualityScore')
|
||||
*/
|
||||
export async function GET(request: NextRequest): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const limit = Math.min(
|
||||
Math.max(Number.parseInt(searchParams.get('limit') || '20', 10), 1),
|
||||
100
|
||||
);
|
||||
const offset = Math.max(Number.parseInt(searchParams.get('offset') || '0', 10), 0);
|
||||
const collectionId = searchParams.get('collectionId');
|
||||
const tagsParam = searchParams.get('tags');
|
||||
const sortBy = searchParams.get('sortBy') || 'qualityScore';
|
||||
|
||||
// Build where clause
|
||||
const where: {
|
||||
collectionId?: string;
|
||||
tags?: { hasSome: string[] };
|
||||
collection?: { isPublic: boolean };
|
||||
} = {};
|
||||
|
||||
if (collectionId) {
|
||||
where.collectionId = collectionId;
|
||||
} else {
|
||||
// Only show scenarios from public collections when not filtering by collectionId
|
||||
where.collection = { isPublic: true };
|
||||
}
|
||||
|
||||
if (tagsParam) {
|
||||
where.tags = { hasSome: tagsParam.split(',').map((t) => t.trim()) };
|
||||
}
|
||||
|
||||
// Build orderBy
|
||||
const orderByMap: Record<string, object> = {
|
||||
qualityScore: { qualityScore: 'desc' },
|
||||
totalRuns: { totalRuns: 'desc' },
|
||||
createdAt: { createdAt: 'desc' },
|
||||
lastRunAt: { lastRunAt: 'desc' },
|
||||
};
|
||||
const orderBy = orderByMap[sortBy] || orderByMap.qualityScore;
|
||||
|
||||
// Fetch scenarios with pagination
|
||||
const scenarios = await prisma.scenario.findMany({
|
||||
where,
|
||||
orderBy,
|
||||
take: limit + 1,
|
||||
skip: offset,
|
||||
include: {
|
||||
collection: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
slug: true,
|
||||
user: {
|
||||
select: {
|
||||
username: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
_count: {
|
||||
select: { runs: true },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const hasMore = scenarios.length > limit;
|
||||
const data = hasMore ? scenarios.slice(0, limit) : scenarios;
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: data.map((s) => ({
|
||||
id: s.id,
|
||||
collectionId: s.collectionId,
|
||||
prompt: s.prompt,
|
||||
name: s.name,
|
||||
description: s.description,
|
||||
tags: s.tags,
|
||||
qualityScore: s.qualityScore,
|
||||
totalRuns: s.totalRuns,
|
||||
lastRunAt: s.lastRunAt,
|
||||
lastRunStatus: s.lastRunStatus,
|
||||
createdAt: s.createdAt,
|
||||
collection: s.collection
|
||||
? {
|
||||
id: s.collection.id,
|
||||
name: s.collection.name,
|
||||
slug: s.collection.slug,
|
||||
username: s.collection.user.username,
|
||||
}
|
||||
: null,
|
||||
runCount: s._count.runs,
|
||||
})),
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
},
|
||||
pagination: {
|
||||
limit,
|
||||
offset,
|
||||
hasMore,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[API Error] GET /api/scenarios:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to fetch scenarios' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/scenarios
|
||||
* Create a new scenario
|
||||
*
|
||||
* Requires authentication
|
||||
* User must own the collection
|
||||
*/
|
||||
export async function POST(request: NextRequest): Promise<NextResponse<ApiResponse>> {
|
||||
const requestId = crypto.randomUUID();
|
||||
|
||||
try {
|
||||
// Check authentication
|
||||
const authResult = await authenticateRequest();
|
||||
|
||||
if (!authResult.authenticated || !authResult.userId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'UNAUTHORIZED', message: 'Authentication required' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 401 }
|
||||
);
|
||||
}
|
||||
|
||||
// Parse and validate request body
|
||||
const body = await request.json();
|
||||
const parseResult = CreateScenarioSchema.safeParse(body);
|
||||
|
||||
if (!parseResult.success) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: {
|
||||
code: 'VALIDATION_ERROR',
|
||||
message: 'Invalid request body',
|
||||
details: { errors: parseResult.error.flatten().fieldErrors },
|
||||
},
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const { collectionId, prompt, name, description, assertions, tags } = parseResult.data;
|
||||
|
||||
// Verify collection exists and user owns it
|
||||
const collection = await prisma.collection.findUnique({
|
||||
where: { id: collectionId },
|
||||
select: { id: true, userId: true },
|
||||
});
|
||||
|
||||
if (!collection) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'NOT_FOUND', message: 'Collection not found' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
if (collection.userId !== authResult.userId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'FORBIDDEN', message: 'You do not own this collection' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
|
||||
// Create the scenario
|
||||
const scenario = await prisma.scenario.create({
|
||||
data: {
|
||||
collectionId,
|
||||
prompt,
|
||||
name,
|
||||
description,
|
||||
assertions: assertions ? (assertions as object) : undefined,
|
||||
tags: tags || [],
|
||||
},
|
||||
include: {
|
||||
collection: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
slug: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: true,
|
||||
data: {
|
||||
id: scenario.id,
|
||||
collectionId: scenario.collectionId,
|
||||
prompt: scenario.prompt,
|
||||
name: scenario.name,
|
||||
description: scenario.description,
|
||||
assertions: scenario.assertions,
|
||||
tags: scenario.tags,
|
||||
qualityScore: scenario.qualityScore,
|
||||
totalRuns: scenario.totalRuns,
|
||||
createdAt: scenario.createdAt,
|
||||
collection: scenario.collection
|
||||
? {
|
||||
id: scenario.collection.id,
|
||||
name: scenario.collection.name,
|
||||
slug: scenario.collection.slug,
|
||||
}
|
||||
: null,
|
||||
},
|
||||
meta: {
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
requestId,
|
||||
},
|
||||
},
|
||||
{ status: 201 }
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('[API Error] POST /api/scenarios:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: { code: 'INTERNAL_ERROR', message: 'Failed to create scenario' },
|
||||
meta: { version: API_VERSION, timestamp: new Date().toISOString(), requestId },
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
151
apps/web/src/lib/scenarios/evaluate.ts
Normal file
151
apps/web/src/lib/scenarios/evaluate.ts
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
/**
|
||||
* Scenario Evaluation Service
|
||||
*
|
||||
* Uses LLM judgment to evaluate if a scenario execution was successful.
|
||||
*/
|
||||
|
||||
import { anthropic } from '@ai-sdk/anthropic';
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import { generateObject } from 'ai';
|
||||
import { z } from 'zod';
|
||||
|
||||
const EvaluationSchema = z.object({
|
||||
verdict: z.enum(['pass', 'fail']).describe('Whether the scenario was completed successfully'),
|
||||
reason: z.string().describe('Brief explanation of why the scenario passed or failed'),
|
||||
confidence: z.number().min(0).max(1).describe('Confidence level in the evaluation (0-1)'),
|
||||
});
|
||||
|
||||
export type EvaluationResult = z.infer<typeof EvaluationSchema>;
|
||||
|
||||
/**
|
||||
* Supported evaluator model IDs
|
||||
*/
|
||||
export type EvaluatorModelId =
|
||||
| 'claude-3-5-sonnet-latest'
|
||||
| 'claude-3-5-haiku-latest'
|
||||
| 'gpt-4o'
|
||||
| 'gpt-4o-mini';
|
||||
|
||||
/**
|
||||
* Get the model instance for an evaluator model ID
|
||||
*/
|
||||
function getEvaluatorModel(modelId: EvaluatorModelId) {
|
||||
switch (modelId) {
|
||||
case 'claude-3-5-sonnet-latest':
|
||||
return anthropic('claude-3-5-sonnet-latest');
|
||||
case 'claude-3-5-haiku-latest':
|
||||
return anthropic('claude-3-5-haiku-latest');
|
||||
case 'gpt-4o':
|
||||
return openai('gpt-4o');
|
||||
case 'gpt-4o-mini':
|
||||
return openai('gpt-4o-mini');
|
||||
default:
|
||||
return anthropic('claude-3-5-haiku-latest');
|
||||
}
|
||||
}
|
||||
|
||||
const DEFAULT_EVALUATOR: EvaluatorModelId = 'claude-3-5-haiku-latest';
|
||||
|
||||
/**
|
||||
* Evaluate if a scenario execution was successful
|
||||
*
|
||||
* @param scenarioPrompt The original scenario prompt/task
|
||||
* @param agentOutput The output produced by the agent
|
||||
* @param conversation Optional conversation history for context
|
||||
* @param modelId Which model to use for evaluation
|
||||
*/
|
||||
export async function evaluateScenarioRun(
|
||||
scenarioPrompt: string,
|
||||
agentOutput: string,
|
||||
conversation?: unknown[],
|
||||
modelId: EvaluatorModelId = DEFAULT_EVALUATOR
|
||||
): Promise<EvaluationResult> {
|
||||
const model = getEvaluatorModel(modelId);
|
||||
|
||||
const conversationContext = conversation
|
||||
? `\n\nConversation history:\n${JSON.stringify(conversation, null, 2)}`
|
||||
: '';
|
||||
|
||||
const { object } = await generateObject({
|
||||
model,
|
||||
schema: EvaluationSchema,
|
||||
prompt: `You are evaluating whether an AI agent successfully completed a task.
|
||||
|
||||
## Task
|
||||
${scenarioPrompt}
|
||||
|
||||
## Agent Output
|
||||
${agentOutput}
|
||||
${conversationContext}
|
||||
|
||||
## Instructions
|
||||
Evaluate if the agent successfully completed the task described above.
|
||||
- A "pass" means the core objective was achieved, even if some minor aspects weren't perfect
|
||||
- A "fail" means the agent failed to accomplish the main goal
|
||||
- Consider partial success as a pass if the primary task was completed
|
||||
- Be fair but rigorous in your evaluation
|
||||
|
||||
Provide your verdict, a brief reason, and your confidence level.`,
|
||||
});
|
||||
|
||||
return object;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run assertions against the output
|
||||
*
|
||||
* @param output The agent output to check
|
||||
* @param assertions The assertions to run
|
||||
*/
|
||||
export function runAssertions(
|
||||
output: string,
|
||||
assertions: { regex?: string[]; schema?: Record<string, unknown> }
|
||||
): { passed: string[]; failed: string[] } {
|
||||
const passed: string[] = [];
|
||||
const failed: string[] = [];
|
||||
|
||||
// Check regex assertions
|
||||
if (assertions.regex) {
|
||||
for (const pattern of assertions.regex) {
|
||||
try {
|
||||
const regex = new RegExp(pattern, 'i');
|
||||
if (regex.test(output)) {
|
||||
passed.push(`regex:${pattern}`);
|
||||
} else {
|
||||
failed.push(`regex:${pattern}`);
|
||||
}
|
||||
} catch {
|
||||
failed.push(`regex:${pattern} (invalid pattern)`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Schema assertions would require more complex validation
|
||||
// For now, we'll just note if schema was provided
|
||||
if (assertions.schema) {
|
||||
// TODO: Implement JSON schema validation against parsed output
|
||||
passed.push('schema:provided (validation pending)');
|
||||
}
|
||||
|
||||
return { passed, failed };
|
||||
}
|
||||
|
||||
/**
|
||||
* Combine LLM evaluation and assertions into final verdict
|
||||
*/
|
||||
export function determineFinalVerdict(
|
||||
evaluation: EvaluationResult,
|
||||
assertions?: { passed: string[]; failed: string[] } | null
|
||||
): 'pass' | 'fail' {
|
||||
// If LLM says fail, it fails
|
||||
if (evaluation.verdict === 'fail') {
|
||||
return 'fail';
|
||||
}
|
||||
|
||||
// If there are failed assertions, it fails
|
||||
if (assertions && assertions.failed.length > 0) {
|
||||
return 'fail';
|
||||
}
|
||||
|
||||
return 'pass';
|
||||
}
|
||||
311
apps/web/src/lib/scenarios/execute.ts
Normal file
311
apps/web/src/lib/scenarios/execute.ts
Normal file
|
|
@ -0,0 +1,311 @@
|
|||
/**
|
||||
* Scenario Execution Service
|
||||
*
|
||||
* Orchestrates scenario execution using ephemeral agents.
|
||||
* Currently implements a simulated execution - full agent integration coming in Phase 3.
|
||||
*/
|
||||
|
||||
import type { Scenario, ScenarioRun } from '@prisma/client';
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import {
|
||||
determineFinalVerdict,
|
||||
type EvaluatorModelId,
|
||||
evaluateScenarioRun,
|
||||
runAssertions,
|
||||
} from './evaluate';
|
||||
|
||||
const DEFAULT_EVALUATOR: EvaluatorModelId = 'claude-3-5-haiku-latest';
|
||||
const MAX_RETRIES = 1;
|
||||
|
||||
interface ExecutionOptions {
|
||||
evaluatorModel?: EvaluatorModelId;
|
||||
}
|
||||
|
||||
interface ExecutionResult {
|
||||
run: ScenarioRun;
|
||||
success: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check and decrement user's daily quota
|
||||
*/
|
||||
export async function checkAndDecrementQuota(
|
||||
userId: string
|
||||
): Promise<{ allowed: boolean; remaining: number }> {
|
||||
const now = new Date();
|
||||
const startOfDay = new Date(now.getFullYear(), now.getMonth(), now.getDate());
|
||||
|
||||
// Get or create quota record
|
||||
let quota = await prisma.scenarioQuota.findUnique({
|
||||
where: { userId },
|
||||
});
|
||||
|
||||
if (!quota) {
|
||||
quota = await prisma.scenarioQuota.create({
|
||||
data: {
|
||||
userId,
|
||||
dailyLimit: 50,
|
||||
dailyUsed: 0,
|
||||
lastResetAt: startOfDay,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Reset quota if it's a new day
|
||||
if (quota.lastResetAt < startOfDay) {
|
||||
quota = await prisma.scenarioQuota.update({
|
||||
where: { userId },
|
||||
data: {
|
||||
dailyUsed: 0,
|
||||
lastResetAt: startOfDay,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Check if quota allows another run
|
||||
if (quota.dailyUsed >= quota.dailyLimit) {
|
||||
return { allowed: false, remaining: 0 };
|
||||
}
|
||||
|
||||
// Decrement quota
|
||||
await prisma.scenarioQuota.update({
|
||||
where: { userId },
|
||||
data: {
|
||||
dailyUsed: { increment: 1 },
|
||||
},
|
||||
});
|
||||
|
||||
return { allowed: true, remaining: quota.dailyLimit - quota.dailyUsed - 1 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Update scenario metrics based on run result
|
||||
*/
|
||||
export async function updateScenarioMetrics(
|
||||
scenarioId: string,
|
||||
status: 'pass' | 'fail' | 'error'
|
||||
): Promise<void> {
|
||||
const scenario = await prisma.scenario.findUnique({
|
||||
where: { id: scenarioId },
|
||||
});
|
||||
|
||||
if (!scenario) return;
|
||||
|
||||
let { consecutivePasses, consecutiveFails, qualityScore, totalRuns } = scenario;
|
||||
|
||||
if (status === 'pass') {
|
||||
consecutivePasses += 1;
|
||||
consecutiveFails = 0;
|
||||
// Bonus for streaks: +0.05 per pass + (streak bonus), max 1.0
|
||||
qualityScore = Math.min(1.0, qualityScore + 0.05 + consecutivePasses * 0.01);
|
||||
} else {
|
||||
consecutiveFails += 1;
|
||||
consecutivePasses = 0;
|
||||
// Penalty for fails: -0.1 per fail + (streak penalty), min 0
|
||||
qualityScore = Math.max(0, qualityScore - 0.1 - consecutiveFails * 0.02);
|
||||
}
|
||||
|
||||
await prisma.scenario.update({
|
||||
where: { id: scenarioId },
|
||||
data: {
|
||||
consecutivePasses,
|
||||
consecutiveFails,
|
||||
qualityScore,
|
||||
totalRuns: totalRuns + 1,
|
||||
lastRunAt: new Date(),
|
||||
lastRunStatus: status,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a scenario
|
||||
*
|
||||
* NOTE: This currently uses simulated execution.
|
||||
* Full agent integration will be added in Phase 3.
|
||||
*
|
||||
* @param scenario The scenario to execute
|
||||
* @param userId The user triggering the execution
|
||||
* @param options Execution options
|
||||
*/
|
||||
export async function executeScenario(
|
||||
scenario: Scenario,
|
||||
userId: string,
|
||||
options: ExecutionOptions = {}
|
||||
): Promise<ExecutionResult> {
|
||||
const { evaluatorModel = DEFAULT_EVALUATOR } = options;
|
||||
|
||||
// Create run record
|
||||
const run = await prisma.scenarioRun.create({
|
||||
data: {
|
||||
scenarioId: scenario.id,
|
||||
userId,
|
||||
status: 'pending',
|
||||
},
|
||||
});
|
||||
|
||||
let lastError: Error | null = null;
|
||||
let retryCount = 0;
|
||||
|
||||
// Retry loop
|
||||
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
||||
retryCount = attempt;
|
||||
|
||||
try {
|
||||
// Update status to running
|
||||
await prisma.scenarioRun.update({
|
||||
where: { id: run.id },
|
||||
data: {
|
||||
status: 'running',
|
||||
startedAt: new Date(),
|
||||
retryCount: attempt,
|
||||
},
|
||||
});
|
||||
|
||||
// TODO: Phase 3 - Replace with actual agent execution
|
||||
// For now, we simulate execution
|
||||
const executionResult = await simulateExecution(scenario);
|
||||
|
||||
// Evaluate with LLM
|
||||
const evaluation = await evaluateScenarioRun(
|
||||
scenario.prompt,
|
||||
executionResult.output,
|
||||
executionResult.conversation,
|
||||
evaluatorModel
|
||||
);
|
||||
|
||||
// Run assertions if defined
|
||||
const assertions = scenario.assertions as {
|
||||
regex?: string[];
|
||||
schema?: Record<string, unknown>;
|
||||
} | null;
|
||||
const assertionResults = assertions
|
||||
? runAssertions(executionResult.output, assertions)
|
||||
: null;
|
||||
|
||||
// Determine final verdict
|
||||
const finalStatus = determineFinalVerdict(evaluation, assertionResults);
|
||||
|
||||
// Update run record
|
||||
const updatedRun = await prisma.scenarioRun.update({
|
||||
where: { id: run.id },
|
||||
data: {
|
||||
status: finalStatus,
|
||||
conversation: executionResult.conversation as object,
|
||||
output: executionResult.output,
|
||||
evaluatorModel,
|
||||
evaluatorVerdict: evaluation.verdict,
|
||||
evaluatorReason: evaluation.reason,
|
||||
assertionResults: assertionResults as object,
|
||||
inputTokens: executionResult.usage.inputTokens,
|
||||
outputTokens: executionResult.usage.outputTokens,
|
||||
totalTokens: executionResult.usage.totalTokens,
|
||||
executionTimeMs: executionResult.durationMs,
|
||||
completedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
// Update scenario metrics
|
||||
await updateScenarioMetrics(scenario.id, finalStatus);
|
||||
|
||||
return { run: updatedRun, success: finalStatus === 'pass' };
|
||||
} catch (error) {
|
||||
lastError = error as Error;
|
||||
|
||||
// If this is the last attempt, mark as error
|
||||
if (attempt === MAX_RETRIES) {
|
||||
const errorRun = await prisma.scenarioRun.update({
|
||||
where: { id: run.id },
|
||||
data: {
|
||||
status: 'error',
|
||||
retryCount,
|
||||
errorLog: (error as Error).stack || (error as Error).message,
|
||||
completedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
await updateScenarioMetrics(scenario.id, 'error');
|
||||
|
||||
return { run: errorRun, success: false };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Should never reach here, but TypeScript needs this
|
||||
throw lastError || new Error('Unknown execution error');
|
||||
}
|
||||
|
||||
/**
|
||||
* Simulated execution for development/testing
|
||||
*
|
||||
* This will be replaced with actual agent execution in Phase 3.
|
||||
*/
|
||||
async function simulateExecution(scenario: Scenario): Promise<{
|
||||
output: string;
|
||||
conversation: unknown[];
|
||||
usage: {
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
totalTokens: number;
|
||||
};
|
||||
durationMs: number;
|
||||
}> {
|
||||
// Simulate some processing time
|
||||
const durationMs = Math.floor(Math.random() * 3000) + 1000;
|
||||
await new Promise((resolve) => setTimeout(resolve, 100)); // Small delay
|
||||
|
||||
// Generate simulated output based on the prompt
|
||||
const passRate = 0.7; // 70% pass rate for simulated runs
|
||||
const willPass = Math.random() < passRate;
|
||||
|
||||
const output = willPass
|
||||
? `[SIMULATED] Successfully completed task: ${scenario.prompt.slice(0, 100)}...\n\nThe scenario was executed successfully. All requested operations were performed.`
|
||||
: `[SIMULATED] Failed to complete task: ${scenario.prompt.slice(0, 100)}...\n\nEncountered an error during execution. The requested operation could not be completed.`;
|
||||
|
||||
const conversation = [
|
||||
{ role: 'user', content: scenario.prompt },
|
||||
{ role: 'assistant', content: output },
|
||||
];
|
||||
|
||||
const inputTokens = Math.floor(scenario.prompt.length / 4);
|
||||
const outputTokens = Math.floor(output.length / 4);
|
||||
|
||||
return {
|
||||
output,
|
||||
conversation,
|
||||
usage: {
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
totalTokens: inputTokens + outputTokens,
|
||||
},
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get user's current quota status
|
||||
*/
|
||||
export async function getQuotaStatus(
|
||||
userId: string
|
||||
): Promise<{ used: number; limit: number; remaining: number; resetsAt: Date }> {
|
||||
const now = new Date();
|
||||
const tomorrow = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1);
|
||||
|
||||
const quota = await prisma.scenarioQuota.findUnique({
|
||||
where: { userId },
|
||||
});
|
||||
|
||||
if (!quota) {
|
||||
return { used: 0, limit: 50, remaining: 50, resetsAt: tomorrow };
|
||||
}
|
||||
|
||||
const startOfDay = new Date(now.getFullYear(), now.getMonth(), now.getDate());
|
||||
const used = quota.lastResetAt < startOfDay ? 0 : quota.dailyUsed;
|
||||
|
||||
return {
|
||||
used,
|
||||
limit: quota.dailyLimit,
|
||||
remaining: quota.dailyLimit - used,
|
||||
resetsAt: tomorrow,
|
||||
};
|
||||
}
|
||||
143
apps/web/src/lib/scenarios/generate-prompt.ts
Normal file
143
apps/web/src/lib/scenarios/generate-prompt.ts
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
/**
|
||||
* AI Prompt Generation Service
|
||||
*
|
||||
* Uses GPT-4o-mini to generate realistic test scenario prompts
|
||||
* for collections based on their tools.
|
||||
*/
|
||||
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import { generateObject, generateText } from 'ai';
|
||||
import { z } from 'zod';
|
||||
|
||||
const GENERATOR_MODEL = 'gpt-4o-mini';
|
||||
|
||||
interface Tool {
|
||||
name: string;
|
||||
description: string | null;
|
||||
}
|
||||
|
||||
interface Collection {
|
||||
name: string;
|
||||
description: string | null;
|
||||
tools: Tool[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a single scenario prompt for a collection
|
||||
*/
|
||||
export async function generateScenarioPrompt(collection: Collection): Promise<string> {
|
||||
const toolDescriptions = collection.tools
|
||||
.map((t) => `- ${t.name}: ${t.description || 'No description'}`)
|
||||
.join('\n');
|
||||
|
||||
const { text } = await generateText({
|
||||
model: openai(GENERATOR_MODEL),
|
||||
prompt: `Generate a realistic test scenario for this tool collection.
|
||||
|
||||
Collection: ${collection.name}
|
||||
Description: ${collection.description || 'No description provided'}
|
||||
|
||||
Available tools:
|
||||
${toolDescriptions}
|
||||
|
||||
Write a single, specific task that a user might want to accomplish using these tools.
|
||||
Be concrete and include example data where helpful (like URLs, file paths, or specific values).
|
||||
The task should be achievable using the available tools.
|
||||
Output only the scenario prompt, nothing else.`,
|
||||
});
|
||||
|
||||
return text.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate multiple scenario prompts with metadata
|
||||
*/
|
||||
export interface GeneratedScenario {
|
||||
prompt: string;
|
||||
name: string;
|
||||
tags: string[];
|
||||
}
|
||||
|
||||
const GeneratedScenarioSchema = z.object({
|
||||
prompt: z.string().describe('The specific task prompt'),
|
||||
name: z.string().describe('A short descriptive name for the scenario (max 50 chars)'),
|
||||
tags: z.array(z.string()).max(5).describe('Relevant tags for categorization'),
|
||||
});
|
||||
|
||||
const GeneratedScenariosSchema = z.object({
|
||||
scenarios: z.array(GeneratedScenarioSchema),
|
||||
});
|
||||
|
||||
/**
|
||||
* Generate multiple scenarios with names and tags
|
||||
*/
|
||||
export async function generateScenarios(
|
||||
collection: Collection,
|
||||
count: number = 1
|
||||
): Promise<GeneratedScenario[]> {
|
||||
const toolDescriptions = collection.tools
|
||||
.map((t) => `- ${t.name}: ${t.description || 'No description'}`)
|
||||
.join('\n');
|
||||
|
||||
const { object } = await generateObject({
|
||||
model: openai(GENERATOR_MODEL),
|
||||
schema: GeneratedScenariosSchema,
|
||||
prompt: `Generate ${count} realistic test scenario${count > 1 ? 's' : ''} for this tool collection.
|
||||
|
||||
Collection: ${collection.name}
|
||||
Description: ${collection.description || 'No description provided'}
|
||||
|
||||
Available tools:
|
||||
${toolDescriptions}
|
||||
|
||||
For each scenario:
|
||||
1. Write a specific, achievable task using the available tools
|
||||
2. Include concrete example data (URLs, file paths, values) where helpful
|
||||
3. Provide a short descriptive name (max 50 characters)
|
||||
4. Add 1-5 relevant tags for categorization
|
||||
|
||||
${count > 1 ? 'Make the scenarios diverse - cover different use cases and tool combinations.' : ''}`,
|
||||
});
|
||||
|
||||
return object.scenarios;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate tags for an existing prompt
|
||||
*/
|
||||
export async function generateTags(prompt: string): Promise<string[]> {
|
||||
const TagsSchema = z.object({
|
||||
tags: z.array(z.string()).min(1).max(5).describe('Relevant tags for the scenario'),
|
||||
});
|
||||
|
||||
const { object } = await generateObject({
|
||||
model: openai(GENERATOR_MODEL),
|
||||
schema: TagsSchema,
|
||||
prompt: `Generate 1-5 relevant tags for this scenario prompt:
|
||||
|
||||
"${prompt}"
|
||||
|
||||
Tags should be:
|
||||
- Lowercase
|
||||
- Single words or short phrases (max 2 words)
|
||||
- Descriptive of the task type, domain, or tools involved`,
|
||||
});
|
||||
|
||||
return object.tags;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a name for an existing prompt
|
||||
*/
|
||||
export async function generateName(prompt: string): Promise<string> {
|
||||
const { text } = await generateText({
|
||||
model: openai(GENERATOR_MODEL),
|
||||
prompt: `Generate a short, descriptive name (max 50 characters) for this scenario:
|
||||
|
||||
"${prompt}"
|
||||
|
||||
Output only the name, nothing else.`,
|
||||
});
|
||||
|
||||
return text.trim().slice(0, 50);
|
||||
}
|
||||
170
apps/web/src/lib/scenarios/similarity.ts
Normal file
170
apps/web/src/lib/scenarios/similarity.ts
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
/**
|
||||
* Scenario Similarity Service
|
||||
*
|
||||
* Uses OpenAI embeddings to detect similar scenarios.
|
||||
* Embeddings are stored in PostgreSQL JSONB for persistence.
|
||||
*/
|
||||
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import type { Scenario, ScenarioEmbedding } from '@prisma/client';
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { embed } from 'ai';
|
||||
|
||||
const EMBEDDING_MODEL = 'text-embedding-3-small';
|
||||
const SIMILARITY_THRESHOLD = 0.7; // 70% similarity triggers warning
|
||||
|
||||
/**
|
||||
* Compute embedding for a text string
|
||||
*/
|
||||
export async function computeEmbedding(text: string): Promise<number[]> {
|
||||
const { embedding } = await embed({
|
||||
model: openai.embedding(EMBEDDING_MODEL),
|
||||
value: text,
|
||||
});
|
||||
return embedding;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute cosine similarity between two vectors
|
||||
*/
|
||||
export function cosineSimilarity(a: number[], b: number[]): number {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error('Vectors must have same length');
|
||||
}
|
||||
|
||||
let dotProduct = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
const aVal = a[i] ?? 0;
|
||||
const bVal = b[i] ?? 0;
|
||||
dotProduct += aVal * bVal;
|
||||
normA += aVal * aVal;
|
||||
normB += bVal * bVal;
|
||||
}
|
||||
|
||||
const denominator = Math.sqrt(normA) * Math.sqrt(normB);
|
||||
if (denominator === 0) return 0;
|
||||
|
||||
return dotProduct / denominator;
|
||||
}
|
||||
|
||||
export interface SimilarScenario {
|
||||
scenario: Scenario;
|
||||
similarity: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find scenarios similar to a given prompt
|
||||
*
|
||||
* @param prompt The prompt to check for similarity
|
||||
* @param collectionId The collection to search within
|
||||
* @param threshold Minimum similarity score (0-1), default 0.7
|
||||
* @param excludeScenarioId Optional scenario ID to exclude (for updates)
|
||||
*/
|
||||
export async function findSimilarScenarios(
|
||||
prompt: string,
|
||||
collectionId: string,
|
||||
threshold: number = SIMILARITY_THRESHOLD,
|
||||
excludeScenarioId?: string
|
||||
): Promise<SimilarScenario[]> {
|
||||
// Compute embedding for the new prompt
|
||||
const newEmbedding = await computeEmbedding(prompt);
|
||||
|
||||
// Get all scenarios with embeddings for this collection
|
||||
const scenarios = await prisma.scenario.findMany({
|
||||
where: {
|
||||
collectionId,
|
||||
...(excludeScenarioId && { id: { not: excludeScenarioId } }),
|
||||
embedding: { isNot: null },
|
||||
},
|
||||
include: {
|
||||
embedding: true,
|
||||
},
|
||||
});
|
||||
|
||||
// Calculate similarity scores
|
||||
const similar: SimilarScenario[] = [];
|
||||
|
||||
for (const scenario of scenarios) {
|
||||
if (!scenario.embedding) continue;
|
||||
|
||||
const existingEmbedding = scenario.embedding.embedding as number[];
|
||||
const similarity = cosineSimilarity(newEmbedding, existingEmbedding);
|
||||
|
||||
if (similarity >= threshold) {
|
||||
// Remove embedding from returned scenario to keep response light
|
||||
const { embedding: _, ...scenarioWithoutEmbedding } = scenario;
|
||||
similar.push({
|
||||
scenario: scenarioWithoutEmbedding as Scenario,
|
||||
similarity,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by similarity descending
|
||||
return similar.sort((a, b) => b.similarity - a.similarity);
|
||||
}
|
||||
|
||||
/**
|
||||
* Store embedding for a scenario
|
||||
*/
|
||||
export async function storeScenarioEmbedding(
|
||||
scenarioId: string,
|
||||
embedding: number[]
|
||||
): Promise<ScenarioEmbedding> {
|
||||
// Upsert to handle both create and update cases
|
||||
return prisma.scenarioEmbedding.upsert({
|
||||
where: { scenarioId },
|
||||
update: {
|
||||
embedding: embedding as unknown as object,
|
||||
model: EMBEDDING_MODEL,
|
||||
},
|
||||
create: {
|
||||
scenarioId,
|
||||
embedding: embedding as unknown as object,
|
||||
model: EMBEDDING_MODEL,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate and store embedding for a scenario
|
||||
*/
|
||||
export async function generateAndStoreEmbedding(
|
||||
scenarioId: string,
|
||||
prompt: string
|
||||
): Promise<ScenarioEmbedding> {
|
||||
const embedding = await computeEmbedding(prompt);
|
||||
return storeScenarioEmbedding(scenarioId, embedding);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check similarity and return warning if similar scenarios exist
|
||||
*/
|
||||
export interface SimilarityCheckResult {
|
||||
hasSimilar: boolean;
|
||||
maxSimilarity: number;
|
||||
similarScenarios: SimilarScenario[];
|
||||
}
|
||||
|
||||
export async function checkSimilarity(
|
||||
prompt: string,
|
||||
collectionId: string,
|
||||
excludeScenarioId?: string
|
||||
): Promise<SimilarityCheckResult> {
|
||||
const similar = await findSimilarScenarios(
|
||||
prompt,
|
||||
collectionId,
|
||||
SIMILARITY_THRESHOLD,
|
||||
excludeScenarioId
|
||||
);
|
||||
|
||||
const firstSimilar = similar[0];
|
||||
return {
|
||||
hasSimilar: similar.length > 0,
|
||||
maxSimilarity: firstSimilar?.similarity ?? 0,
|
||||
similarScenarios: similar.slice(0, 5), // Return top 5 similar
|
||||
};
|
||||
}
|
||||
|
|
@ -4,9 +4,7 @@
|
|||
"aliases": [],
|
||||
"args": {},
|
||||
"description": "Run diagnostic checks for TPMJS CLI",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %>"
|
||||
],
|
||||
"examples": ["<%= config.bin %> <%= command.id %>"],
|
||||
"flags": {
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
|
|
@ -31,11 +29,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"doctor.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "doctor.js"]
|
||||
},
|
||||
"playground": {
|
||||
"aliases": [],
|
||||
|
|
@ -79,19 +73,13 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"playground.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "playground.js"]
|
||||
},
|
||||
"update": {
|
||||
"aliases": [],
|
||||
"args": {},
|
||||
"description": "Update the TPMJS CLI to the latest version",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %>"
|
||||
],
|
||||
"examples": ["<%= config.bin %> <%= command.id %>"],
|
||||
"flags": {
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
|
|
@ -122,11 +110,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"update.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "update.js"]
|
||||
},
|
||||
"agent:chat": {
|
||||
"aliases": [],
|
||||
|
|
@ -186,12 +170,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"agent",
|
||||
"chat.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "agent", "chat.js"]
|
||||
},
|
||||
"agent:create": {
|
||||
"aliases": [],
|
||||
|
|
@ -233,13 +212,7 @@
|
|||
"required": true,
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"options": [
|
||||
"ANTHROPIC",
|
||||
"OPENAI",
|
||||
"GOOGLE",
|
||||
"GROQ",
|
||||
"MISTRAL"
|
||||
],
|
||||
"options": ["ANTHROPIC", "OPENAI", "GOOGLE", "GROQ", "MISTRAL"],
|
||||
"type": "option"
|
||||
},
|
||||
"model": {
|
||||
|
|
@ -297,12 +270,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"agent",
|
||||
"create.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "agent", "create.js"]
|
||||
},
|
||||
"agent:delete": {
|
||||
"aliases": [],
|
||||
|
|
@ -349,12 +317,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"agent",
|
||||
"delete.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "agent", "delete.js"]
|
||||
},
|
||||
"agent:list": {
|
||||
"aliases": [],
|
||||
|
|
@ -406,12 +369,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"agent",
|
||||
"list.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "agent", "list.js"]
|
||||
},
|
||||
"agent:update": {
|
||||
"aliases": [],
|
||||
|
|
@ -457,13 +415,7 @@
|
|||
"name": "provider",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"options": [
|
||||
"ANTHROPIC",
|
||||
"OPENAI",
|
||||
"GOOGLE",
|
||||
"GROQ",
|
||||
"MISTRAL"
|
||||
],
|
||||
"options": ["ANTHROPIC", "OPENAI", "GOOGLE", "GROQ", "MISTRAL"],
|
||||
"type": "option"
|
||||
},
|
||||
"model": {
|
||||
|
|
@ -519,12 +471,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"agent",
|
||||
"update.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "agent", "update.js"]
|
||||
},
|
||||
"auth:login": {
|
||||
"aliases": [],
|
||||
|
|
@ -579,20 +526,13 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"auth",
|
||||
"login.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "auth", "login.js"]
|
||||
},
|
||||
"auth:logout": {
|
||||
"aliases": [],
|
||||
"args": {},
|
||||
"description": "Log out from TPMJS",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %>"
|
||||
],
|
||||
"examples": ["<%= config.bin %> <%= command.id %>"],
|
||||
"flags": {
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
|
|
@ -610,20 +550,13 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"auth",
|
||||
"logout.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "auth", "logout.js"]
|
||||
},
|
||||
"auth:status": {
|
||||
"aliases": [],
|
||||
"args": {},
|
||||
"description": "Show authentication status",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %>"
|
||||
],
|
||||
"examples": ["<%= config.bin %> <%= command.id %>"],
|
||||
"flags": {
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
|
|
@ -648,20 +581,13 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"auth",
|
||||
"status.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "auth", "status.js"]
|
||||
},
|
||||
"auth:whoami": {
|
||||
"aliases": [],
|
||||
"args": {},
|
||||
"description": "Show current user information",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %>"
|
||||
],
|
||||
"examples": ["<%= config.bin %> <%= command.id %>"],
|
||||
"flags": {
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
|
|
@ -686,12 +612,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"auth",
|
||||
"whoami.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "auth", "whoami.js"]
|
||||
},
|
||||
"collection:add": {
|
||||
"aliases": [],
|
||||
|
|
@ -731,12 +652,7 @@
|
|||
"strict": false,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"collection",
|
||||
"add.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "collection", "add.js"]
|
||||
},
|
||||
"collection:create": {
|
||||
"aliases": [],
|
||||
|
|
@ -793,12 +709,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"collection",
|
||||
"create.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "collection", "create.js"]
|
||||
},
|
||||
"collection:delete": {
|
||||
"aliases": [],
|
||||
|
|
@ -845,12 +756,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"collection",
|
||||
"delete.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "collection", "delete.js"]
|
||||
},
|
||||
"collection:import": {
|
||||
"aliases": [],
|
||||
|
|
@ -899,12 +805,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"collection",
|
||||
"import.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "collection", "import.js"]
|
||||
},
|
||||
"collection:list": {
|
||||
"aliases": [],
|
||||
|
|
@ -956,12 +857,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"collection",
|
||||
"list.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "collection", "list.js"]
|
||||
},
|
||||
"collection:remove": {
|
||||
"aliases": [],
|
||||
|
|
@ -978,9 +874,7 @@
|
|||
}
|
||||
},
|
||||
"description": "Remove a tool from a collection",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %> my-collection tool-id-1"
|
||||
],
|
||||
"examples": ["<%= config.bin %> <%= command.id %> my-collection tool-id-1"],
|
||||
"flags": {
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
|
|
@ -1005,12 +899,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"collection",
|
||||
"remove.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "collection", "remove.js"]
|
||||
},
|
||||
"collection:update": {
|
||||
"aliases": [],
|
||||
|
|
@ -1072,12 +961,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"collection",
|
||||
"update.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "collection", "update.js"]
|
||||
},
|
||||
"mcp:config": {
|
||||
"aliases": [],
|
||||
|
|
@ -1102,12 +986,7 @@
|
|||
"default": "claude",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"options": [
|
||||
"claude",
|
||||
"cursor",
|
||||
"windsurf",
|
||||
"generic"
|
||||
],
|
||||
"options": ["claude", "cursor", "windsurf", "generic"],
|
||||
"type": "option"
|
||||
},
|
||||
"output": {
|
||||
|
|
@ -1142,12 +1021,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"mcp",
|
||||
"config.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "mcp", "config.js"]
|
||||
},
|
||||
"mcp:serve": {
|
||||
"aliases": [],
|
||||
|
|
@ -1208,12 +1082,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"mcp",
|
||||
"serve.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "mcp", "serve.js"]
|
||||
},
|
||||
"publish:check": {
|
||||
"aliases": [],
|
||||
|
|
@ -1253,12 +1122,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"publish",
|
||||
"check.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "publish", "check.js"]
|
||||
},
|
||||
"publish:preview": {
|
||||
"aliases": [],
|
||||
|
|
@ -1301,12 +1165,271 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"publish",
|
||||
"preview.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "publish", "preview.js"]
|
||||
},
|
||||
"scenario:generate": {
|
||||
"aliases": [],
|
||||
"args": {
|
||||
"collection": {
|
||||
"description": "Collection ID or slug",
|
||||
"name": "collection",
|
||||
"required": true
|
||||
}
|
||||
},
|
||||
"description": "Generate AI-powered scenarios for a collection",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %> my-collection",
|
||||
"<%= config.bin %> <%= command.id %> my-collection --count 3",
|
||||
"<%= config.bin %> <%= command.id %> my-collection --skip-similarity-check"
|
||||
],
|
||||
"flags": {
|
||||
"count": {
|
||||
"char": "n",
|
||||
"description": "Number of scenarios to generate (1-10)",
|
||||
"name": "count",
|
||||
"default": 1,
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"skip-similarity-check": {
|
||||
"description": "Skip checking for similar existing scenarios",
|
||||
"name": "skip-similarity-check",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
"name": "json",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"verbose": {
|
||||
"char": "v",
|
||||
"description": "Show verbose output",
|
||||
"name": "verbose",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"hasDynamicHelp": false,
|
||||
"hiddenAliases": [],
|
||||
"id": "scenario:generate",
|
||||
"pluginAlias": "@tpmjs/cli",
|
||||
"pluginName": "@tpmjs/cli",
|
||||
"pluginType": "core",
|
||||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "scenario", "generate.js"]
|
||||
},
|
||||
"scenario:info": {
|
||||
"aliases": [],
|
||||
"args": {
|
||||
"scenarioId": {
|
||||
"description": "Scenario ID",
|
||||
"name": "scenarioId",
|
||||
"required": true
|
||||
}
|
||||
},
|
||||
"description": "Show detailed information about a scenario",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %> clu123abc456",
|
||||
"<%= config.bin %> <%= command.id %> clu123abc456 --runs 20",
|
||||
"<%= config.bin %> <%= command.id %> clu123abc456 --json"
|
||||
],
|
||||
"flags": {
|
||||
"runs": {
|
||||
"char": "r",
|
||||
"description": "Number of recent runs to show",
|
||||
"name": "runs",
|
||||
"default": 10,
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
"name": "json",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"verbose": {
|
||||
"char": "v",
|
||||
"description": "Show verbose output",
|
||||
"name": "verbose",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"hasDynamicHelp": false,
|
||||
"hiddenAliases": [],
|
||||
"id": "scenario:info",
|
||||
"pluginAlias": "@tpmjs/cli",
|
||||
"pluginName": "@tpmjs/cli",
|
||||
"pluginType": "core",
|
||||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "scenario", "info.js"]
|
||||
},
|
||||
"scenario:list": {
|
||||
"aliases": [],
|
||||
"args": {
|
||||
"collection": {
|
||||
"description": "Collection ID or slug (optional - shows all public scenarios if omitted)",
|
||||
"name": "collection",
|
||||
"required": false
|
||||
}
|
||||
},
|
||||
"description": "List scenarios for a collection or all public scenarios",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %>",
|
||||
"<%= config.bin %> <%= command.id %> my-collection",
|
||||
"<%= config.bin %> <%= command.id %> --limit 20 --json"
|
||||
],
|
||||
"flags": {
|
||||
"limit": {
|
||||
"char": "l",
|
||||
"description": "Maximum number of results",
|
||||
"name": "limit",
|
||||
"default": 20,
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"offset": {
|
||||
"char": "o",
|
||||
"description": "Offset for pagination",
|
||||
"name": "offset",
|
||||
"default": 0,
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"tags": {
|
||||
"char": "t",
|
||||
"description": "Filter by tags (comma-separated)",
|
||||
"name": "tags",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
"name": "json",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"verbose": {
|
||||
"char": "v",
|
||||
"description": "Show verbose output",
|
||||
"name": "verbose",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"hasDynamicHelp": false,
|
||||
"hiddenAliases": [],
|
||||
"id": "scenario:list",
|
||||
"pluginAlias": "@tpmjs/cli",
|
||||
"pluginName": "@tpmjs/cli",
|
||||
"pluginType": "core",
|
||||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "scenario", "list.js"]
|
||||
},
|
||||
"scenario:run": {
|
||||
"aliases": [],
|
||||
"args": {
|
||||
"collection": {
|
||||
"description": "Collection ID or slug",
|
||||
"name": "collection",
|
||||
"required": true
|
||||
}
|
||||
},
|
||||
"description": "Run all scenarios for a collection",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %> my-collection",
|
||||
"<%= config.bin %> <%= command.id %> my-collection --json",
|
||||
"<%= config.bin %> <%= command.id %> my-collection --verbose"
|
||||
],
|
||||
"flags": {
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
"name": "json",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"verbose": {
|
||||
"char": "v",
|
||||
"description": "Show verbose output",
|
||||
"name": "verbose",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"limit": {
|
||||
"char": "l",
|
||||
"description": "Maximum number of scenarios to run",
|
||||
"name": "limit",
|
||||
"default": 50,
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
}
|
||||
},
|
||||
"hasDynamicHelp": false,
|
||||
"hiddenAliases": [],
|
||||
"id": "scenario:run",
|
||||
"pluginAlias": "@tpmjs/cli",
|
||||
"pluginName": "@tpmjs/cli",
|
||||
"pluginType": "core",
|
||||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "scenario", "run.js"]
|
||||
},
|
||||
"scenario:test": {
|
||||
"aliases": [],
|
||||
"args": {
|
||||
"scenarioId": {
|
||||
"description": "Scenario ID to run",
|
||||
"name": "scenarioId",
|
||||
"required": true
|
||||
}
|
||||
},
|
||||
"description": "Run a single scenario by ID",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %> clu123abc456",
|
||||
"<%= config.bin %> <%= command.id %> clu123abc456 --json",
|
||||
"<%= config.bin %> <%= command.id %> clu123abc456 --verbose"
|
||||
],
|
||||
"flags": {
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
"name": "json",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"verbose": {
|
||||
"char": "v",
|
||||
"description": "Show verbose output including full reason",
|
||||
"name": "verbose",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"hasDynamicHelp": false,
|
||||
"hiddenAliases": [],
|
||||
"id": "scenario:test",
|
||||
"pluginAlias": "@tpmjs/cli",
|
||||
"pluginName": "@tpmjs/cli",
|
||||
"pluginType": "core",
|
||||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "scenario", "test.js"]
|
||||
},
|
||||
"tool:execute": {
|
||||
"aliases": [],
|
||||
|
|
@ -1379,12 +1502,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"tool",
|
||||
"execute.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "tool", "execute.js"]
|
||||
},
|
||||
"tool:info": {
|
||||
"aliases": [],
|
||||
|
|
@ -1429,12 +1547,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"tool",
|
||||
"info.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "tool", "info.js"]
|
||||
},
|
||||
"tool:init": {
|
||||
"aliases": [],
|
||||
|
|
@ -1459,10 +1572,7 @@
|
|||
"default": "minimal",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"options": [
|
||||
"minimal",
|
||||
"rich"
|
||||
],
|
||||
"options": ["minimal", "rich"],
|
||||
"type": "option"
|
||||
},
|
||||
"category": {
|
||||
|
|
@ -1519,12 +1629,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"tool",
|
||||
"init.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "tool", "init.js"]
|
||||
},
|
||||
"tool:search": {
|
||||
"aliases": [],
|
||||
|
|
@ -1591,12 +1696,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"tool",
|
||||
"search.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "tool", "search.js"]
|
||||
},
|
||||
"tool:trending": {
|
||||
"aliases": [],
|
||||
|
|
@ -1639,12 +1739,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"tool",
|
||||
"trending.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "tool", "trending.js"]
|
||||
},
|
||||
"tool:validate": {
|
||||
"aliases": [],
|
||||
|
|
@ -1687,13 +1782,8 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"tool",
|
||||
"validate.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "tool", "validate.js"]
|
||||
}
|
||||
},
|
||||
"version": "0.1.2"
|
||||
}
|
||||
"version": "0.1.3"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@tpmjs/cli",
|
||||
"version": "0.1.2",
|
||||
"version": "0.1.3",
|
||||
"description": "TPMJS command-line interface for AI tool discovery and execution",
|
||||
"author": "TPMJS",
|
||||
"license": "MIT",
|
||||
|
|
@ -55,6 +55,9 @@
|
|||
"collection": {
|
||||
"description": "Collection management"
|
||||
},
|
||||
"scenario": {
|
||||
"description": "Test scenario management and execution"
|
||||
},
|
||||
"auth": {
|
||||
"description": "Authentication"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import { Args, Command, Flags } from '@oclif/core';
|
||||
import open from 'open';
|
||||
import { createServer } from 'node:http';
|
||||
import { URL } from 'node:url';
|
||||
import { saveCredentials, getApiUrl } from '../../lib/config.js';
|
||||
import { Args, Command, Flags } from '@oclif/core';
|
||||
import open from 'open';
|
||||
import { TpmClient } from '../../lib/api-client.js';
|
||||
import { getApiUrl, saveCredentials } from '../../lib/config.js';
|
||||
import { createOutput } from '../../lib/output.js';
|
||||
|
||||
export default class Login extends Command {
|
||||
|
|
@ -62,7 +62,9 @@ export default class Login extends Command {
|
|||
output.listItem('tpm auth login --api-key <your-api-key>');
|
||||
output.listItem('tpm auth login --browser (opens browser for OAuth)');
|
||||
output.newLine();
|
||||
output.text(`Get your API key at: ${output.link('tpmjs.com/dashboard/settings/tpmjs-api-keys', 'https://tpmjs.com/dashboard/settings/tpmjs-api-keys')}`);
|
||||
output.text(
|
||||
`Get your API key at: ${output.link('tpmjs.com/dashboard/settings/tpmjs-api-keys', 'https://tpmjs.com/dashboard/settings/tpmjs-api-keys')}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -122,6 +124,13 @@ export default class Login extends Command {
|
|||
output.info('Opening browser for authentication...');
|
||||
|
||||
return new Promise((resolve) => {
|
||||
let timeoutId: NodeJS.Timeout;
|
||||
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeoutId);
|
||||
server.close();
|
||||
};
|
||||
|
||||
const server = createServer(async (req, res) => {
|
||||
const url = new URL(req.url ?? '/', `http://localhost:${port}`);
|
||||
|
||||
|
|
@ -132,8 +141,10 @@ export default class Login extends Command {
|
|||
|
||||
if (error) {
|
||||
res.writeHead(200, { 'Content-Type': 'text/html' });
|
||||
res.end('<html><body><h1>Authentication Failed</h1><p>You can close this window.</p></body></html>');
|
||||
server.close();
|
||||
res.end(
|
||||
'<html><body><h1>Authentication Failed</h1><p>You can close this window.</p></body></html>'
|
||||
);
|
||||
cleanup();
|
||||
output.error(`Authentication failed: ${error}`);
|
||||
resolve();
|
||||
return;
|
||||
|
|
@ -141,8 +152,10 @@ export default class Login extends Command {
|
|||
|
||||
if (receivedState !== state) {
|
||||
res.writeHead(200, { 'Content-Type': 'text/html' });
|
||||
res.end('<html><body><h1>Invalid State</h1><p>Authentication failed due to invalid state.</p></body></html>');
|
||||
server.close();
|
||||
res.end(
|
||||
'<html><body><h1>Invalid State</h1><p>Authentication failed due to invalid state.</p></body></html>'
|
||||
);
|
||||
cleanup();
|
||||
output.error('Authentication failed: Invalid state parameter');
|
||||
resolve();
|
||||
return;
|
||||
|
|
@ -152,8 +165,10 @@ export default class Login extends Command {
|
|||
saveCredentials({ apiKey });
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'text/html' });
|
||||
res.end('<html><body><h1>Success!</h1><p>You are now logged in. You can close this window.</p></body></html>');
|
||||
server.close();
|
||||
res.end(
|
||||
'<html><body><h1>Success!</h1><p>You are now logged in. You can close this window.</p></body></html>'
|
||||
);
|
||||
cleanup();
|
||||
|
||||
output.success('Logged in successfully via browser');
|
||||
|
||||
|
|
@ -165,7 +180,7 @@ export default class Login extends Command {
|
|||
} else {
|
||||
res.writeHead(200, { 'Content-Type': 'text/html' });
|
||||
res.end('<html><body><h1>Error</h1><p>No API key received.</p></body></html>');
|
||||
server.close();
|
||||
cleanup();
|
||||
output.error('No API key received from authentication');
|
||||
resolve();
|
||||
}
|
||||
|
|
@ -190,11 +205,14 @@ export default class Login extends Command {
|
|||
});
|
||||
|
||||
// Timeout after 5 minutes
|
||||
setTimeout(() => {
|
||||
server.close();
|
||||
output.error('Authentication timed out');
|
||||
resolve();
|
||||
}, 5 * 60 * 1000);
|
||||
timeoutId = setTimeout(
|
||||
() => {
|
||||
server.close();
|
||||
output.error('Authentication timed out');
|
||||
resolve();
|
||||
},
|
||||
5 * 60 * 1000
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
164
packages/cli/src/commands/scenario/generate.ts
Normal file
164
packages/cli/src/commands/scenario/generate.ts
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
import { Args, Command, Flags } from '@oclif/core';
|
||||
import { getClient } from '../../lib/api-client.js';
|
||||
import { createOutput } from '../../lib/output.js';
|
||||
|
||||
export default class ScenarioGenerate extends Command {
|
||||
static description = 'Generate AI-powered scenarios for a collection';
|
||||
|
||||
static examples = [
|
||||
'<%= config.bin %> <%= command.id %> my-collection',
|
||||
'<%= config.bin %> <%= command.id %> my-collection --count 3',
|
||||
'<%= config.bin %> <%= command.id %> my-collection --skip-similarity-check',
|
||||
];
|
||||
|
||||
static args = {
|
||||
collection: Args.string({
|
||||
description: 'Collection ID or slug',
|
||||
required: true,
|
||||
}),
|
||||
};
|
||||
|
||||
static flags = {
|
||||
count: Flags.integer({
|
||||
char: 'n',
|
||||
description: 'Number of scenarios to generate (1-10)',
|
||||
default: 1,
|
||||
min: 1,
|
||||
max: 10,
|
||||
}),
|
||||
'skip-similarity-check': Flags.boolean({
|
||||
description: 'Skip checking for similar existing scenarios',
|
||||
default: false,
|
||||
}),
|
||||
json: Flags.boolean({
|
||||
description: 'Output in JSON format',
|
||||
default: false,
|
||||
}),
|
||||
verbose: Flags.boolean({
|
||||
char: 'v',
|
||||
description: 'Show verbose output',
|
||||
default: false,
|
||||
}),
|
||||
};
|
||||
|
||||
async run(): Promise<void> {
|
||||
const { args, flags } = await this.parse(ScenarioGenerate);
|
||||
const output = createOutput(flags);
|
||||
const client = getClient();
|
||||
|
||||
if (!client.isAuthenticated()) {
|
||||
output.error('Not authenticated. Run `tpm auth login` first.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Find collection
|
||||
const collectionsSpinner = output.spinner('Finding collection...');
|
||||
let collectionId: string;
|
||||
let collectionName: string;
|
||||
|
||||
try {
|
||||
const collections = await client.listCollections({ limit: 100 });
|
||||
const collection = collections.data.find(
|
||||
(c) => c.id === args.collection || c.slug === args.collection
|
||||
);
|
||||
|
||||
if (!collection) {
|
||||
collectionsSpinner.fail('Collection not found');
|
||||
output.error(`No collection found with ID or slug: ${args.collection}`);
|
||||
return;
|
||||
}
|
||||
|
||||
collectionId = collection.id;
|
||||
collectionName = collection.name;
|
||||
collectionsSpinner.stop();
|
||||
} catch (error) {
|
||||
collectionsSpinner.fail('Failed to find collection');
|
||||
output.error(error instanceof Error ? error.message : 'Unknown error');
|
||||
return;
|
||||
}
|
||||
|
||||
// Generate scenarios
|
||||
const generateSpinner = output.spinner(
|
||||
`Generating ${flags.count} scenario${flags.count > 1 ? 's' : ''} for "${collectionName}"...`
|
||||
);
|
||||
|
||||
try {
|
||||
const result = await client.generateScenarios(collectionId, {
|
||||
count: flags.count,
|
||||
skipSimilarityCheck: flags['skip-similarity-check'],
|
||||
});
|
||||
|
||||
const scenarios =
|
||||
(
|
||||
result as unknown as {
|
||||
data: {
|
||||
scenarios: Array<{
|
||||
scenario: {
|
||||
id: string;
|
||||
name: string;
|
||||
prompt: string;
|
||||
tags: string[];
|
||||
};
|
||||
similarity?: {
|
||||
hasSimilar: boolean;
|
||||
maxSimilarity: number;
|
||||
similar: Array<{ name: string; similarity: number }>;
|
||||
};
|
||||
}>;
|
||||
};
|
||||
}
|
||||
).data?.scenarios || [];
|
||||
|
||||
generateSpinner.succeed(
|
||||
`Generated ${scenarios.length} scenario${scenarios.length > 1 ? 's' : ''}`
|
||||
);
|
||||
|
||||
if (flags.json) {
|
||||
output.json({ collection: collectionName, scenarios });
|
||||
return;
|
||||
}
|
||||
|
||||
output.newLine();
|
||||
|
||||
for (let i = 0; i < scenarios.length; i++) {
|
||||
const item = scenarios[i];
|
||||
if (!item) continue;
|
||||
const { scenario, similarity } = item;
|
||||
output.text(output.bold(`${i + 1}. ${scenario.name}`));
|
||||
output.text(` ID: ${scenario.id}`);
|
||||
|
||||
if (flags.verbose) {
|
||||
output.text(` Prompt: ${scenario.prompt}`);
|
||||
} else {
|
||||
output.text(` Prompt: ${scenario.prompt.slice(0, 80)}...`);
|
||||
}
|
||||
|
||||
if (scenario.tags.length > 0) {
|
||||
output.text(` Tags: ${scenario.tags.join(', ')}`);
|
||||
}
|
||||
|
||||
if (similarity?.hasSimilar) {
|
||||
output.text(
|
||||
output.yellow(` ⚠ Similar to existing: ${similarity.maxSimilarity}% match`)
|
||||
);
|
||||
if (flags.verbose && similarity.similar.length > 0) {
|
||||
for (const s of similarity.similar) {
|
||||
output.text(output.dim(` - "${s.name}" (${s.similarity}% similar)`));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
output.newLine();
|
||||
}
|
||||
|
||||
output.text(output.dim('Run these scenarios with:'));
|
||||
output.text(output.dim(` tpm scenario run ${args.collection}`));
|
||||
} catch (error) {
|
||||
generateSpinner.fail('Failed to generate scenarios');
|
||||
output.error(
|
||||
error instanceof Error ? error.message : 'Unknown error',
|
||||
flags.verbose ? String(error) : undefined
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
190
packages/cli/src/commands/scenario/info.ts
Normal file
190
packages/cli/src/commands/scenario/info.ts
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
import { Args, Command, Flags } from '@oclif/core';
|
||||
import { getClient } from '../../lib/api-client.js';
|
||||
import { createOutput } from '../../lib/output.js';
|
||||
|
||||
export default class ScenarioInfo extends Command {
|
||||
static description = 'Show detailed information about a scenario';
|
||||
|
||||
static examples = [
|
||||
'<%= config.bin %> <%= command.id %> clu123abc456',
|
||||
'<%= config.bin %> <%= command.id %> clu123abc456 --runs 20',
|
||||
'<%= config.bin %> <%= command.id %> clu123abc456 --json',
|
||||
];
|
||||
|
||||
static args = {
|
||||
scenarioId: Args.string({
|
||||
description: 'Scenario ID',
|
||||
required: true,
|
||||
}),
|
||||
};
|
||||
|
||||
static flags = {
|
||||
runs: Flags.integer({
|
||||
char: 'r',
|
||||
description: 'Number of recent runs to show',
|
||||
default: 10,
|
||||
}),
|
||||
json: Flags.boolean({
|
||||
description: 'Output in JSON format',
|
||||
default: false,
|
||||
}),
|
||||
verbose: Flags.boolean({
|
||||
char: 'v',
|
||||
description: 'Show verbose output',
|
||||
default: false,
|
||||
}),
|
||||
};
|
||||
|
||||
async run(): Promise<void> {
|
||||
const { args, flags } = await this.parse(ScenarioInfo);
|
||||
const output = createOutput(flags);
|
||||
const client = getClient();
|
||||
|
||||
const spinner = output.spinner('Fetching scenario...');
|
||||
|
||||
try {
|
||||
const response = await client.getScenario(args.scenarioId);
|
||||
const scenario = (
|
||||
response as unknown as {
|
||||
data: {
|
||||
id: string;
|
||||
name: string | null;
|
||||
prompt: string;
|
||||
description: string | null;
|
||||
tags: string[];
|
||||
qualityScore: number;
|
||||
consecutivePasses: number;
|
||||
consecutiveFails: number;
|
||||
totalRuns: number;
|
||||
lastRunAt: string | null;
|
||||
lastRunStatus: string | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
collection?: {
|
||||
id: string;
|
||||
name: string;
|
||||
slug: string | null;
|
||||
username: string | null;
|
||||
};
|
||||
recentRuns?: Array<{
|
||||
id: string;
|
||||
status: string;
|
||||
evaluatorVerdict: string | null;
|
||||
executionTimeMs: number | null;
|
||||
createdAt: string;
|
||||
}>;
|
||||
runCount?: number;
|
||||
};
|
||||
}
|
||||
).data;
|
||||
|
||||
spinner.stop();
|
||||
|
||||
if (flags.json) {
|
||||
output.json(scenario);
|
||||
return;
|
||||
}
|
||||
|
||||
// Header
|
||||
output.text(output.bold(scenario.name || 'Unnamed Scenario'));
|
||||
output.text(output.dim(`ID: ${scenario.id}`));
|
||||
output.newLine();
|
||||
|
||||
// Collection info
|
||||
if (scenario.collection) {
|
||||
output.text(output.bold('Collection'));
|
||||
output.text(` Name: ${scenario.collection.name}`);
|
||||
if (scenario.collection.slug) {
|
||||
output.text(` Slug: ${scenario.collection.slug}`);
|
||||
}
|
||||
if (scenario.collection.username) {
|
||||
output.text(` Owner: @${scenario.collection.username}`);
|
||||
}
|
||||
output.newLine();
|
||||
}
|
||||
|
||||
// Prompt
|
||||
output.text(output.bold('Prompt'));
|
||||
if (flags.verbose || scenario.prompt.length <= 200) {
|
||||
output.text(` ${scenario.prompt}`);
|
||||
} else {
|
||||
output.text(` ${scenario.prompt.slice(0, 200)}...`);
|
||||
output.text(output.dim(' (use --verbose to see full prompt)'));
|
||||
}
|
||||
output.newLine();
|
||||
|
||||
// Tags
|
||||
if (scenario.tags && scenario.tags.length > 0) {
|
||||
output.text(output.bold('Tags'));
|
||||
output.text(` ${scenario.tags.join(', ')}`);
|
||||
output.newLine();
|
||||
}
|
||||
|
||||
// Metrics
|
||||
output.text(output.bold('Metrics'));
|
||||
output.text(` Quality Score: ${(scenario.qualityScore * 100).toFixed(1)}%`);
|
||||
output.text(` Total Runs: ${scenario.totalRuns}`);
|
||||
output.text(` Consecutive Passes: ${scenario.consecutivePasses}`);
|
||||
output.text(` Consecutive Fails: ${scenario.consecutiveFails}`);
|
||||
if (scenario.lastRunStatus) {
|
||||
const statusColor =
|
||||
scenario.lastRunStatus === 'pass'
|
||||
? output.green
|
||||
: scenario.lastRunStatus === 'fail'
|
||||
? output.red
|
||||
: output.yellow;
|
||||
output.text(` Last Run Status: ${statusColor(scenario.lastRunStatus)}`);
|
||||
}
|
||||
if (scenario.lastRunAt) {
|
||||
output.text(` Last Run: ${new Date(scenario.lastRunAt).toLocaleString()}`);
|
||||
}
|
||||
output.newLine();
|
||||
|
||||
// Timestamps
|
||||
output.text(output.bold('Timestamps'));
|
||||
output.text(` Created: ${new Date(scenario.createdAt).toLocaleString()}`);
|
||||
output.text(` Updated: ${new Date(scenario.updatedAt).toLocaleString()}`);
|
||||
output.newLine();
|
||||
|
||||
// Recent runs
|
||||
if (scenario.recentRuns && scenario.recentRuns.length > 0) {
|
||||
output.text(
|
||||
output.bold(
|
||||
`Recent Runs (${scenario.recentRuns.length} of ${scenario.runCount ?? scenario.totalRuns})`
|
||||
)
|
||||
);
|
||||
output.table(
|
||||
scenario.recentRuns.map((run) => ({
|
||||
status:
|
||||
run.status === 'pass'
|
||||
? output.green('pass')
|
||||
: run.status === 'fail'
|
||||
? output.red('fail')
|
||||
: output.yellow(run.status),
|
||||
verdict: run.evaluatorVerdict || '-',
|
||||
time: run.executionTimeMs ? `${run.executionTimeMs}ms` : '-',
|
||||
date: new Date(run.createdAt).toLocaleString(),
|
||||
})),
|
||||
[
|
||||
{ key: 'status', header: 'Status', width: 10 },
|
||||
{ key: 'verdict', header: 'Verdict', width: 10 },
|
||||
{ key: 'time', header: 'Time', width: 12 },
|
||||
{ key: 'date', header: 'Date', width: 25 },
|
||||
]
|
||||
);
|
||||
} else {
|
||||
output.text(output.dim('No runs yet'));
|
||||
}
|
||||
|
||||
output.newLine();
|
||||
output.text(output.dim('Run this scenario with:'));
|
||||
output.text(output.dim(` tpm scenario test ${scenario.id}`));
|
||||
} catch (error) {
|
||||
spinner.fail('Failed to fetch scenario');
|
||||
output.error(
|
||||
error instanceof Error ? error.message : 'Unknown error',
|
||||
flags.verbose ? String(error) : undefined
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
181
packages/cli/src/commands/scenario/list.ts
Normal file
181
packages/cli/src/commands/scenario/list.ts
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
import { Args, Command, Flags } from '@oclif/core';
|
||||
import {
|
||||
type ApiResponse,
|
||||
getClient,
|
||||
type PaginatedResponse,
|
||||
type Scenario,
|
||||
} from '../../lib/api-client.js';
|
||||
import { createOutput } from '../../lib/output.js';
|
||||
|
||||
export default class ScenarioList extends Command {
|
||||
static description = 'List scenarios for a collection or all public scenarios';
|
||||
|
||||
static examples = [
|
||||
'<%= config.bin %> <%= command.id %>',
|
||||
'<%= config.bin %> <%= command.id %> my-collection',
|
||||
'<%= config.bin %> <%= command.id %> --limit 20 --json',
|
||||
];
|
||||
|
||||
static args = {
|
||||
collection: Args.string({
|
||||
description: 'Collection ID or slug (optional - shows all public scenarios if omitted)',
|
||||
required: false,
|
||||
}),
|
||||
};
|
||||
|
||||
static flags = {
|
||||
limit: Flags.integer({
|
||||
char: 'l',
|
||||
description: 'Maximum number of results',
|
||||
default: 20,
|
||||
}),
|
||||
offset: Flags.integer({
|
||||
char: 'o',
|
||||
description: 'Offset for pagination',
|
||||
default: 0,
|
||||
}),
|
||||
tags: Flags.string({
|
||||
char: 't',
|
||||
description: 'Filter by tags (comma-separated)',
|
||||
}),
|
||||
json: Flags.boolean({
|
||||
description: 'Output in JSON format',
|
||||
default: false,
|
||||
}),
|
||||
verbose: Flags.boolean({
|
||||
char: 'v',
|
||||
description: 'Show verbose output',
|
||||
default: false,
|
||||
}),
|
||||
};
|
||||
|
||||
async run(): Promise<void> {
|
||||
const { args, flags } = await this.parse(ScenarioList);
|
||||
const output = createOutput(flags);
|
||||
const client = getClient();
|
||||
|
||||
const spinner = output.spinner('Fetching scenarios...');
|
||||
|
||||
try {
|
||||
let response: PaginatedResponse<Scenario> | ApiResponse<{ scenarios: Scenario[] }>;
|
||||
|
||||
if (args.collection) {
|
||||
// Try to find collection by ID or slug
|
||||
const collections = await client.listCollections({ limit: 100 });
|
||||
const collection = collections.data.find(
|
||||
(c) => c.id === args.collection || c.slug === args.collection
|
||||
);
|
||||
|
||||
if (!collection) {
|
||||
spinner.fail('Collection not found');
|
||||
output.error(`No collection found with ID or slug: ${args.collection}`);
|
||||
return;
|
||||
}
|
||||
|
||||
response = await client.listCollectionScenarios(collection.id, {
|
||||
limit: flags.limit,
|
||||
offset: flags.offset,
|
||||
});
|
||||
|
||||
spinner.stop();
|
||||
|
||||
if (flags.json) {
|
||||
output.json(response);
|
||||
return;
|
||||
}
|
||||
|
||||
const scenarios =
|
||||
(
|
||||
response as unknown as {
|
||||
data: {
|
||||
scenarios: Array<{
|
||||
id: string;
|
||||
name: string | null;
|
||||
prompt: string;
|
||||
qualityScore: number;
|
||||
totalRuns: number;
|
||||
lastRunStatus: string | null;
|
||||
tags: string[];
|
||||
}>;
|
||||
};
|
||||
}
|
||||
).data?.scenarios || [];
|
||||
|
||||
if (scenarios.length === 0) {
|
||||
output.info(`No scenarios found for collection "${collection.name}"`);
|
||||
output.text(
|
||||
'Generate some with: tpm scenario generate ' + (collection.slug || collection.id)
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
output.text(output.bold(`Scenarios for ${collection.name}\n`));
|
||||
output.table(
|
||||
scenarios.map((s) => ({
|
||||
name: s.name || s.prompt.slice(0, 30) + '...',
|
||||
quality: (s.qualityScore * 100).toFixed(0) + '%',
|
||||
runs: s.totalRuns,
|
||||
status: s.lastRunStatus || '-',
|
||||
tags: s.tags.slice(0, 3).join(', ') || '-',
|
||||
})),
|
||||
[
|
||||
{ key: 'name', header: 'Name', width: 35 },
|
||||
{ key: 'quality', header: 'Quality', width: 10 },
|
||||
{ key: 'runs', header: 'Runs', width: 8 },
|
||||
{ key: 'status', header: 'Status', width: 8 },
|
||||
{ key: 'tags', header: 'Tags', width: 20 },
|
||||
]
|
||||
);
|
||||
} else {
|
||||
response = await client.listScenarios({
|
||||
limit: flags.limit,
|
||||
offset: flags.offset,
|
||||
tags: flags.tags,
|
||||
});
|
||||
|
||||
spinner.stop();
|
||||
|
||||
if (flags.json) {
|
||||
output.json(response);
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.data.length === 0) {
|
||||
output.info('No public scenarios found');
|
||||
return;
|
||||
}
|
||||
|
||||
output.table(
|
||||
response.data.map((s) => ({
|
||||
name: s.name || s.prompt.slice(0, 30) + '...',
|
||||
collection: s.collection?.name || '-',
|
||||
quality: (s.qualityScore * 100).toFixed(0) + '%',
|
||||
runs: s.totalRuns,
|
||||
status: s.lastRunStatus || '-',
|
||||
})),
|
||||
[
|
||||
{ key: 'name', header: 'Name', width: 35 },
|
||||
{ key: 'collection', header: 'Collection', width: 25 },
|
||||
{ key: 'quality', header: 'Quality', width: 10 },
|
||||
{ key: 'runs', header: 'Runs', width: 8 },
|
||||
{ key: 'status', header: 'Status', width: 8 },
|
||||
]
|
||||
);
|
||||
|
||||
output.newLine();
|
||||
output.text(
|
||||
output.dim(
|
||||
`Showing ${response.data.length} scenario(s)` +
|
||||
(response.pagination.hasMore ? ` (more available)` : '')
|
||||
)
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
spinner.fail('Failed to fetch scenarios');
|
||||
output.error(
|
||||
error instanceof Error ? error.message : 'Unknown error',
|
||||
flags.verbose ? String(error) : undefined
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
208
packages/cli/src/commands/scenario/run.ts
Normal file
208
packages/cli/src/commands/scenario/run.ts
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
import { Args, Command, Flags } from '@oclif/core';
|
||||
import { getClient } from '../../lib/api-client.js';
|
||||
import { createOutput } from '../../lib/output.js';
|
||||
|
||||
export default class ScenarioRun extends Command {
|
||||
static description = 'Run all scenarios for a collection';
|
||||
|
||||
static examples = [
|
||||
'<%= config.bin %> <%= command.id %> my-collection',
|
||||
'<%= config.bin %> <%= command.id %> my-collection --json',
|
||||
'<%= config.bin %> <%= command.id %> my-collection --verbose',
|
||||
];
|
||||
|
||||
static args = {
|
||||
collection: Args.string({
|
||||
description: 'Collection ID or slug',
|
||||
required: true,
|
||||
}),
|
||||
};
|
||||
|
||||
static flags = {
|
||||
json: Flags.boolean({
|
||||
description: 'Output in JSON format',
|
||||
default: false,
|
||||
}),
|
||||
verbose: Flags.boolean({
|
||||
char: 'v',
|
||||
description: 'Show verbose output',
|
||||
default: false,
|
||||
}),
|
||||
limit: Flags.integer({
|
||||
char: 'l',
|
||||
description: 'Maximum number of scenarios to run',
|
||||
default: 50,
|
||||
}),
|
||||
};
|
||||
|
||||
async run(): Promise<void> {
|
||||
const { args, flags } = await this.parse(ScenarioRun);
|
||||
const output = createOutput(flags);
|
||||
const client = getClient();
|
||||
|
||||
if (!client.isAuthenticated()) {
|
||||
output.error('Not authenticated. Run `tpm auth login` first.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Find collection
|
||||
const collectionsSpinner = output.spinner('Finding collection...');
|
||||
let collectionId: string;
|
||||
|
||||
try {
|
||||
const collections = await client.listCollections({ limit: 100 });
|
||||
const collection = collections.data.find(
|
||||
(c) => c.id === args.collection || c.slug === args.collection
|
||||
);
|
||||
|
||||
if (!collection) {
|
||||
collectionsSpinner.fail('Collection not found');
|
||||
output.error(`No collection found with ID or slug: ${args.collection}`);
|
||||
return;
|
||||
}
|
||||
|
||||
collectionId = collection.id;
|
||||
collectionsSpinner.stop();
|
||||
output.text(output.bold(`Running scenarios for: ${collection.name}\n`));
|
||||
} catch (error) {
|
||||
collectionsSpinner.fail('Failed to find collection');
|
||||
output.error(error instanceof Error ? error.message : 'Unknown error');
|
||||
return;
|
||||
}
|
||||
|
||||
// Fetch scenarios
|
||||
const scenariosSpinner = output.spinner('Fetching scenarios...');
|
||||
let scenarios: Array<{
|
||||
id: string;
|
||||
name: string | null;
|
||||
prompt: string;
|
||||
}>;
|
||||
|
||||
try {
|
||||
const response = await client.listCollectionScenarios(collectionId, { limit: flags.limit });
|
||||
scenarios =
|
||||
(
|
||||
response as unknown as {
|
||||
data: {
|
||||
scenarios: Array<{
|
||||
id: string;
|
||||
name: string | null;
|
||||
prompt: string;
|
||||
}>;
|
||||
};
|
||||
}
|
||||
).data?.scenarios || [];
|
||||
|
||||
if (scenarios.length === 0) {
|
||||
scenariosSpinner.fail('No scenarios found');
|
||||
output.info('This collection has no scenarios. Generate some with:');
|
||||
output.text(` tpm scenario generate ${args.collection}`);
|
||||
return;
|
||||
}
|
||||
|
||||
scenariosSpinner.stop();
|
||||
output.info(`Found ${scenarios.length} scenario(s) to run\n`);
|
||||
} catch (error) {
|
||||
scenariosSpinner.fail('Failed to fetch scenarios');
|
||||
output.error(error instanceof Error ? error.message : 'Unknown error');
|
||||
return;
|
||||
}
|
||||
|
||||
// Run each scenario
|
||||
const results: Array<{
|
||||
name: string;
|
||||
status: string;
|
||||
verdict: string | null;
|
||||
reason: string | null;
|
||||
timeMs: number | null;
|
||||
}> = [];
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
let errors = 0;
|
||||
|
||||
for (const scenario of scenarios) {
|
||||
const name = scenario.name || scenario.prompt.slice(0, 40) + '...';
|
||||
const runSpinner = output.spinner(`Running: ${name}`);
|
||||
|
||||
try {
|
||||
const result = await client.runScenario(scenario.id);
|
||||
const runData = (
|
||||
result as unknown as {
|
||||
data: {
|
||||
status: string;
|
||||
success: boolean;
|
||||
evaluator: { verdict: string | null; reason: string | null };
|
||||
usage: { executionTimeMs: number | null };
|
||||
};
|
||||
}
|
||||
).data;
|
||||
|
||||
if (runData.success) {
|
||||
passed++;
|
||||
runSpinner.succeed(`${output.green('✓')} ${name}`);
|
||||
} else if (runData.status === 'error') {
|
||||
errors++;
|
||||
runSpinner.fail(`${output.red('✗')} ${name} (error)`);
|
||||
} else {
|
||||
failed++;
|
||||
runSpinner.fail(`${output.red('✗')} ${name}`);
|
||||
}
|
||||
|
||||
if (flags.verbose && runData.evaluator?.reason) {
|
||||
output.text(output.dim(` → ${runData.evaluator.reason}`));
|
||||
}
|
||||
|
||||
results.push({
|
||||
name,
|
||||
status: runData.status,
|
||||
verdict: runData.evaluator?.verdict ?? null,
|
||||
reason: runData.evaluator?.reason ?? null,
|
||||
timeMs: runData.usage?.executionTimeMs ?? null,
|
||||
});
|
||||
} catch (error) {
|
||||
errors++;
|
||||
runSpinner.fail(`${output.red('✗')} ${name} (error)`);
|
||||
results.push({
|
||||
name,
|
||||
status: 'error',
|
||||
verdict: null,
|
||||
reason: error instanceof Error ? error.message : 'Unknown error',
|
||||
timeMs: null,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Output summary
|
||||
output.newLine();
|
||||
output.text(output.bold('─'.repeat(50)));
|
||||
output.text(output.bold('Summary'));
|
||||
output.text(` ${output.green('Passed:')} ${passed}`);
|
||||
output.text(` ${output.red('Failed:')} ${failed}`);
|
||||
if (errors > 0) {
|
||||
output.text(` ${output.yellow('Errors:')} ${errors}`);
|
||||
}
|
||||
output.text(` ${output.dim('Total:')} ${scenarios.length}`);
|
||||
|
||||
const passRate = scenarios.length > 0 ? (passed / scenarios.length) * 100 : 0;
|
||||
output.newLine();
|
||||
output.text(`Pass rate: ${passRate.toFixed(1)}%`);
|
||||
|
||||
if (flags.json) {
|
||||
output.newLine();
|
||||
output.json({
|
||||
collection: args.collection,
|
||||
total: scenarios.length,
|
||||
passed,
|
||||
failed,
|
||||
errors,
|
||||
passRate: passRate.toFixed(1),
|
||||
results,
|
||||
});
|
||||
}
|
||||
|
||||
// Exit with error code if any failures
|
||||
if (failed > 0 || errors > 0) {
|
||||
this.exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
168
packages/cli/src/commands/scenario/test.ts
Normal file
168
packages/cli/src/commands/scenario/test.ts
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
import { Args, Command, Flags } from '@oclif/core';
|
||||
import { getClient } from '../../lib/api-client.js';
|
||||
import { createOutput } from '../../lib/output.js';
|
||||
|
||||
export default class ScenarioTest extends Command {
|
||||
static description = 'Run a single scenario by ID';
|
||||
|
||||
static examples = [
|
||||
'<%= config.bin %> <%= command.id %> clu123abc456',
|
||||
'<%= config.bin %> <%= command.id %> clu123abc456 --json',
|
||||
'<%= config.bin %> <%= command.id %> clu123abc456 --verbose',
|
||||
];
|
||||
|
||||
static args = {
|
||||
scenarioId: Args.string({
|
||||
description: 'Scenario ID to run',
|
||||
required: true,
|
||||
}),
|
||||
};
|
||||
|
||||
static flags = {
|
||||
json: Flags.boolean({
|
||||
description: 'Output in JSON format',
|
||||
default: false,
|
||||
}),
|
||||
verbose: Flags.boolean({
|
||||
char: 'v',
|
||||
description: 'Show verbose output including full reason',
|
||||
default: false,
|
||||
}),
|
||||
};
|
||||
|
||||
async run(): Promise<void> {
|
||||
const { args, flags } = await this.parse(ScenarioTest);
|
||||
const output = createOutput(flags);
|
||||
const client = getClient();
|
||||
|
||||
if (!client.isAuthenticated()) {
|
||||
output.error('Not authenticated. Run `tpm auth login` first.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Fetch scenario info first
|
||||
const infoSpinner = output.spinner('Fetching scenario...');
|
||||
let scenarioName: string;
|
||||
|
||||
try {
|
||||
const scenarioResponse = await client.getScenario(args.scenarioId);
|
||||
const scenario = (
|
||||
scenarioResponse as unknown as {
|
||||
data: {
|
||||
name: string | null;
|
||||
prompt: string;
|
||||
collection?: { name: string };
|
||||
};
|
||||
}
|
||||
).data;
|
||||
|
||||
scenarioName = scenario.name || scenario.prompt.slice(0, 50) + '...';
|
||||
infoSpinner.stop();
|
||||
|
||||
output.text(output.bold(`Scenario: ${scenarioName}`));
|
||||
if (scenario.collection) {
|
||||
output.text(output.dim(`Collection: ${scenario.collection.name}`));
|
||||
}
|
||||
output.newLine();
|
||||
} catch (error) {
|
||||
infoSpinner.fail('Scenario not found');
|
||||
output.error(
|
||||
error instanceof Error ? error.message : 'Unknown error',
|
||||
flags.verbose ? String(error) : undefined
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Run the scenario
|
||||
const runSpinner = output.spinner('Executing scenario...');
|
||||
|
||||
try {
|
||||
const result = await client.runScenario(args.scenarioId);
|
||||
const runData = (
|
||||
result as unknown as {
|
||||
data: {
|
||||
runId: string;
|
||||
status: string;
|
||||
success: boolean;
|
||||
evaluator: {
|
||||
model: string | null;
|
||||
verdict: string | null;
|
||||
reason: string | null;
|
||||
};
|
||||
usage: {
|
||||
inputTokens: number | null;
|
||||
outputTokens: number | null;
|
||||
totalTokens: number | null;
|
||||
executionTimeMs: number | null;
|
||||
};
|
||||
timestamps: {
|
||||
startedAt: string | null;
|
||||
completedAt: string | null;
|
||||
createdAt: string;
|
||||
};
|
||||
quotaRemaining: number;
|
||||
};
|
||||
}
|
||||
).data;
|
||||
|
||||
if (runData.success) {
|
||||
runSpinner.succeed(output.green('Scenario PASSED'));
|
||||
} else if (runData.status === 'error') {
|
||||
runSpinner.fail(output.red('Scenario ERROR'));
|
||||
} else {
|
||||
runSpinner.fail(output.red('Scenario FAILED'));
|
||||
}
|
||||
|
||||
output.newLine();
|
||||
|
||||
if (flags.json) {
|
||||
output.json(runData);
|
||||
return;
|
||||
}
|
||||
|
||||
// Display results
|
||||
output.text(output.bold('Results'));
|
||||
output.text(` Status: ${runData.status}`);
|
||||
output.text(` Verdict: ${runData.evaluator?.verdict || 'N/A'}`);
|
||||
|
||||
if (runData.evaluator?.reason) {
|
||||
if (flags.verbose) {
|
||||
output.text(` Reason: ${runData.evaluator.reason}`);
|
||||
} else {
|
||||
const truncatedReason =
|
||||
runData.evaluator.reason.length > 80
|
||||
? runData.evaluator.reason.slice(0, 80) + '...'
|
||||
: runData.evaluator.reason;
|
||||
output.text(` Reason: ${truncatedReason}`);
|
||||
}
|
||||
}
|
||||
|
||||
output.newLine();
|
||||
output.text(output.bold('Usage'));
|
||||
if (runData.usage.executionTimeMs) {
|
||||
output.text(` Duration: ${runData.usage.executionTimeMs}ms`);
|
||||
}
|
||||
if (runData.usage.totalTokens) {
|
||||
output.text(
|
||||
` Tokens: ${runData.usage.totalTokens} (in: ${runData.usage.inputTokens}, out: ${runData.usage.outputTokens})`
|
||||
);
|
||||
}
|
||||
|
||||
output.newLine();
|
||||
output.text(output.dim(`Run ID: ${runData.runId}`));
|
||||
output.text(output.dim(`Quota remaining: ${runData.quotaRemaining} runs/day`));
|
||||
|
||||
// Exit with error code if failed
|
||||
if (!runData.success) {
|
||||
this.exit(1);
|
||||
}
|
||||
} catch (error) {
|
||||
runSpinner.fail('Failed to run scenario');
|
||||
output.error(
|
||||
error instanceof Error ? error.message : 'Unknown error',
|
||||
flags.verbose ? String(error) : undefined
|
||||
);
|
||||
this.exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -149,6 +149,73 @@ export interface ApiKey {
|
|||
createdAt: string;
|
||||
}
|
||||
|
||||
// Scenario types
|
||||
export interface Scenario {
|
||||
id: string;
|
||||
collectionId: string | null;
|
||||
prompt: string;
|
||||
name: string | null;
|
||||
description: string | null;
|
||||
tags: string[];
|
||||
qualityScore: number;
|
||||
totalRuns: number;
|
||||
lastRunAt: string | null;
|
||||
lastRunStatus: string | null;
|
||||
consecutivePasses: number;
|
||||
consecutiveFails: number;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
collection?: {
|
||||
id: string;
|
||||
name: string;
|
||||
slug: string | null;
|
||||
username: string | null;
|
||||
} | null;
|
||||
}
|
||||
|
||||
export interface ScenarioRun {
|
||||
id: string;
|
||||
status: string;
|
||||
success: boolean;
|
||||
evaluator: {
|
||||
model: string | null;
|
||||
verdict: string | null;
|
||||
reason: string | null;
|
||||
};
|
||||
assertions: unknown;
|
||||
usage: {
|
||||
inputTokens: number | null;
|
||||
outputTokens: number | null;
|
||||
totalTokens: number | null;
|
||||
executionTimeMs: number | null;
|
||||
};
|
||||
timestamps: {
|
||||
startedAt: string | null;
|
||||
completedAt: string | null;
|
||||
createdAt: string;
|
||||
};
|
||||
quotaRemaining?: number;
|
||||
}
|
||||
|
||||
export interface ScenarioListOptions extends PaginationOptions {
|
||||
collectionId?: string;
|
||||
tags?: string;
|
||||
sortBy?: 'qualityScore' | 'totalRuns' | 'createdAt' | 'lastRunAt';
|
||||
}
|
||||
|
||||
export interface CreateScenarioInput {
|
||||
collectionId: string;
|
||||
prompt: string;
|
||||
name?: string;
|
||||
description?: string;
|
||||
tags?: string[];
|
||||
}
|
||||
|
||||
export interface GenerateScenariosInput {
|
||||
count?: number;
|
||||
skipSimilarityCheck?: boolean;
|
||||
}
|
||||
|
||||
// Stats types
|
||||
export interface Stats {
|
||||
tools: {
|
||||
|
|
@ -175,10 +242,7 @@ export class TpmClient {
|
|||
this.timeout = options.timeout ?? 30000;
|
||||
}
|
||||
|
||||
private async request<T>(
|
||||
endpoint: string,
|
||||
options: RequestInit = {}
|
||||
): Promise<T> {
|
||||
private async request<T>(endpoint: string, options: RequestInit = {}): Promise<T> {
|
||||
const url = `${this.baseUrl}${endpoint}`;
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
|
|
@ -199,7 +263,7 @@ export class TpmClient {
|
|||
signal: controller.signal,
|
||||
});
|
||||
|
||||
const data = await response.json() as T & { message?: string; error?: string };
|
||||
const data = (await response.json()) as T & { message?: string; error?: string };
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ApiError(
|
||||
|
|
@ -240,14 +304,16 @@ export class TpmClient {
|
|||
}
|
||||
|
||||
async getTool(packageName: string, toolName: string): Promise<ApiResponse<Tool>> {
|
||||
return this.request(`/tools/${encodeURIComponent(packageName)}/${encodeURIComponent(toolName)}`);
|
||||
return this.request(
|
||||
`/tools/${encodeURIComponent(packageName)}/${encodeURIComponent(toolName)}`
|
||||
);
|
||||
}
|
||||
|
||||
async getToolBySlug(slug: string): Promise<ApiResponse<Tool>> {
|
||||
// Search for the tool by slug
|
||||
const searchResult = await this.searchTools({ query: slug, limit: 1 });
|
||||
if (searchResult.data && searchResult.data.length > 0) {
|
||||
const tool = searchResult.data.find(t => t.slug === slug) || searchResult.data[0];
|
||||
const tool = searchResult.data.find((t) => t.slug === slug) || searchResult.data[0];
|
||||
return { success: true, data: tool };
|
||||
}
|
||||
return { success: false, error: 'Tool not found' };
|
||||
|
|
@ -264,7 +330,9 @@ export class TpmClient {
|
|||
return this.request<PaginatedResponse<Tool>>(endpoint);
|
||||
}
|
||||
|
||||
async validateTpmjsField(field: unknown): Promise<ApiResponse<{ valid: boolean; tier: string | null; errors?: unknown[] }>> {
|
||||
async validateTpmjsField(
|
||||
field: unknown
|
||||
): Promise<ApiResponse<{ valid: boolean; tier: string | null; errors?: unknown[] }>> {
|
||||
return this.request('/tools/validate', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(field),
|
||||
|
|
@ -285,7 +353,7 @@ export class TpmClient {
|
|||
const url = `${this.baseUrl}/tools/${encodeURIComponent(slug)}/execute`;
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'text/event-stream',
|
||||
Accept: 'text/event-stream',
|
||||
};
|
||||
|
||||
if (this.apiKey) {
|
||||
|
|
@ -403,7 +471,10 @@ export class TpmClient {
|
|||
});
|
||||
}
|
||||
|
||||
async updateCollection(id: string, input: UpdateCollectionInput): Promise<ApiResponse<Collection>> {
|
||||
async updateCollection(
|
||||
id: string,
|
||||
input: UpdateCollectionInput
|
||||
): Promise<ApiResponse<Collection>> {
|
||||
return this.request(`/collections/${id}`, {
|
||||
method: 'PATCH',
|
||||
body: JSON.stringify(input),
|
||||
|
|
@ -441,6 +512,80 @@ export class TpmClient {
|
|||
return this.request('/user/tpmjs-api-keys');
|
||||
}
|
||||
|
||||
// Scenarios
|
||||
async listScenarios(options: ScenarioListOptions = {}): Promise<PaginatedResponse<Scenario>> {
|
||||
const params = new URLSearchParams();
|
||||
if (options.limit) params.set('limit', String(options.limit));
|
||||
if (options.offset) params.set('offset', String(options.offset));
|
||||
if (options.collectionId) params.set('collectionId', options.collectionId);
|
||||
if (options.tags) params.set('tags', options.tags);
|
||||
if (options.sortBy) params.set('sortBy', options.sortBy);
|
||||
|
||||
const queryString = params.toString();
|
||||
const endpoint = queryString ? `/scenarios?${queryString}` : '/scenarios';
|
||||
|
||||
return this.request<PaginatedResponse<Scenario>>(endpoint);
|
||||
}
|
||||
|
||||
async listCollectionScenarios(
|
||||
collectionId: string,
|
||||
options: PaginationOptions = {}
|
||||
): Promise<ApiResponse<{ scenarios: Scenario[] }>> {
|
||||
const params = new URLSearchParams();
|
||||
if (options.limit) params.set('limit', String(options.limit));
|
||||
if (options.offset) params.set('offset', String(options.offset));
|
||||
|
||||
const queryString = params.toString();
|
||||
const endpoint = queryString
|
||||
? `/collections/${collectionId}/scenarios?${queryString}`
|
||||
: `/collections/${collectionId}/scenarios`;
|
||||
|
||||
return this.request<ApiResponse<{ scenarios: Scenario[] }>>(endpoint);
|
||||
}
|
||||
|
||||
async getScenario(id: string): Promise<ApiResponse<Scenario>> {
|
||||
return this.request(`/scenarios/${id}`);
|
||||
}
|
||||
|
||||
async createScenario(input: CreateScenarioInput): Promise<ApiResponse<Scenario>> {
|
||||
return this.request('/scenarios', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(input),
|
||||
});
|
||||
}
|
||||
|
||||
async generateScenarios(
|
||||
collectionId: string,
|
||||
input: GenerateScenariosInput = {}
|
||||
): Promise<ApiResponse<{ scenarios: { scenario: Scenario; similarity?: unknown }[] }>> {
|
||||
return this.request(`/collections/${collectionId}/scenarios/generate`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(input),
|
||||
});
|
||||
}
|
||||
|
||||
async runScenario(scenarioId: string): Promise<ApiResponse<ScenarioRun>> {
|
||||
return this.request(`/scenarios/${scenarioId}/run`, {
|
||||
method: 'POST',
|
||||
});
|
||||
}
|
||||
|
||||
async getScenarioRuns(
|
||||
scenarioId: string,
|
||||
options: PaginationOptions = {}
|
||||
): Promise<PaginatedResponse<ScenarioRun>> {
|
||||
const params = new URLSearchParams();
|
||||
if (options.limit) params.set('limit', String(options.limit));
|
||||
if (options.offset) params.set('offset', String(options.offset));
|
||||
|
||||
const queryString = params.toString();
|
||||
const endpoint = queryString
|
||||
? `/scenarios/${scenarioId}/runs?${queryString}`
|
||||
: `/scenarios/${scenarioId}/runs`;
|
||||
|
||||
return this.request<PaginatedResponse<ScenarioRun>>(endpoint);
|
||||
}
|
||||
|
||||
// Check if authenticated
|
||||
isAuthenticated(): boolean {
|
||||
return !!this.apiKey;
|
||||
|
|
|
|||
|
|
@ -173,6 +173,27 @@ export class OutputFormatter {
|
|||
// OSC 8 hyperlink support for modern terminals
|
||||
return `\x1b]8;;${url}\x07${pc.underline(pc.blue(text))}\x1b]8;;\x07`;
|
||||
}
|
||||
|
||||
// Color helpers
|
||||
green(text: string): string {
|
||||
return pc.green(text);
|
||||
}
|
||||
|
||||
red(text: string): string {
|
||||
return pc.red(text);
|
||||
}
|
||||
|
||||
yellow(text: string): string {
|
||||
return pc.yellow(text);
|
||||
}
|
||||
|
||||
blue(text: string): string {
|
||||
return pc.blue(text);
|
||||
}
|
||||
|
||||
cyan(text: string): string {
|
||||
return pc.cyan(text);
|
||||
}
|
||||
}
|
||||
|
||||
// Convenience function to create formatter from command flags
|
||||
|
|
|
|||
|
|
@ -467,6 +467,7 @@ model Collection {
|
|||
agents AgentCollection[]
|
||||
likes CollectionLike[]
|
||||
bridgeTools CollectionBridgeTool[]
|
||||
scenarios Scenario[]
|
||||
|
||||
// Unique constraint: user can't have duplicate collection slugs
|
||||
@@unique([userId, slug])
|
||||
|
|
@ -1118,3 +1119,136 @@ model ApiUsageSummary {
|
|||
@@index([periodType, periodStart])
|
||||
@@map("api_usage_summaries")
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Scenario Models (Integration Testing for Collections)
|
||||
// ============================================================================
|
||||
|
||||
/// Scenario - AI-generated test scenarios for collections
|
||||
model Scenario {
|
||||
id String @id @default(cuid())
|
||||
|
||||
// Collection relationship (nullable for orphaned scenarios)
|
||||
collectionId String? @map("collection_id")
|
||||
collection Collection? @relation(fields: [collectionId], references: [id], onDelete: SetNull)
|
||||
|
||||
// Content
|
||||
prompt String @db.Text // AI-generated free-form prompt
|
||||
name String? @db.VarChar(200) // Optional human-readable name
|
||||
description String? @db.Text
|
||||
|
||||
// Validation (optional assertions)
|
||||
assertions Json? @db.JsonB // { regex?: string[], schema?: object }
|
||||
|
||||
// AI-generated metadata
|
||||
tags String[] @default([]) @db.Text
|
||||
|
||||
// Quality metrics (streak-based scoring)
|
||||
qualityScore Float @default(0) @map("quality_score")
|
||||
consecutivePasses Int @default(0) @map("consecutive_passes")
|
||||
consecutiveFails Int @default(0) @map("consecutive_fails")
|
||||
totalRuns Int @default(0) @map("total_runs")
|
||||
lastRunAt DateTime? @map("last_run_at")
|
||||
lastRunStatus String? @map("last_run_status") @db.VarChar(20) // 'pass' | 'fail' | 'error'
|
||||
|
||||
// Timestamps
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
|
||||
// Relations
|
||||
runs ScenarioRun[]
|
||||
embedding ScenarioEmbedding?
|
||||
|
||||
@@index([collectionId])
|
||||
@@index([qualityScore])
|
||||
@@index([createdAt])
|
||||
@@index([lastRunStatus])
|
||||
@@map("scenarios")
|
||||
}
|
||||
|
||||
/// ScenarioEmbedding - vector embeddings for scenario similarity detection
|
||||
model ScenarioEmbedding {
|
||||
id String @id @default(cuid())
|
||||
|
||||
// Scenario relationship
|
||||
scenarioId String @unique @map("scenario_id")
|
||||
scenario Scenario @relation(fields: [scenarioId], references: [id], onDelete: Cascade)
|
||||
|
||||
// Embedding data
|
||||
embedding Json @db.JsonB // Array of floats (1536 dims for text-embedding-3-small)
|
||||
model String @default("text-embedding-3-small") @db.VarChar(50)
|
||||
|
||||
// Timestamps
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
|
||||
@@index([scenarioId])
|
||||
@@map("scenario_embeddings")
|
||||
}
|
||||
|
||||
/// ScenarioRun - individual execution records for scenarios
|
||||
model ScenarioRun {
|
||||
id String @id @default(cuid())
|
||||
|
||||
// Scenario relationship
|
||||
scenarioId String @map("scenario_id")
|
||||
scenario Scenario @relation(fields: [scenarioId], references: [id], onDelete: Cascade)
|
||||
|
||||
// Execution context
|
||||
userId String @map("user_id") // Who triggered the run
|
||||
agentId String? @map("agent_id") // Ephemeral agent ID (for debugging)
|
||||
|
||||
// Status
|
||||
status String @db.VarChar(20) // 'pending' | 'running' | 'pass' | 'fail' | 'error'
|
||||
retryCount Int @default(0) @map("retry_count")
|
||||
|
||||
// Results
|
||||
conversation Json? @db.JsonB // Full message history
|
||||
output String? @db.Text // Final output from agent
|
||||
errorLog String? @map("error_log") @db.Text // Full error logs (private to owner)
|
||||
|
||||
// LLM Evaluation
|
||||
evaluatorModel String? @map("evaluator_model") @db.VarChar(50) // e.g., "claude-3.5-sonnet"
|
||||
evaluatorVerdict String? @map("evaluator_verdict") @db.VarChar(10) // 'pass' | 'fail'
|
||||
evaluatorReason String? @map("evaluator_reason") @db.Text // Explanation
|
||||
|
||||
// Assertions
|
||||
assertionResults Json? @map("assertion_results") @db.JsonB // { passed: string[], failed: string[] }
|
||||
|
||||
// Cost tracking
|
||||
inputTokens Int? @map("input_tokens")
|
||||
outputTokens Int? @map("output_tokens")
|
||||
totalTokens Int? @map("total_tokens")
|
||||
executionTimeMs Int? @map("execution_time_ms")
|
||||
estimatedCost Decimal? @map("estimated_cost") @db.Decimal(10, 6)
|
||||
|
||||
// Timestamps
|
||||
startedAt DateTime? @map("started_at")
|
||||
completedAt DateTime? @map("completed_at")
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
|
||||
@@index([scenarioId])
|
||||
@@index([userId])
|
||||
@@index([status])
|
||||
@@index([createdAt])
|
||||
@@map("scenario_runs")
|
||||
}
|
||||
|
||||
/// ScenarioQuota - daily usage quotas for scenario runs
|
||||
model ScenarioQuota {
|
||||
id String @id @default(cuid())
|
||||
|
||||
// User relationship
|
||||
userId String @unique @map("user_id")
|
||||
|
||||
// Quota configuration
|
||||
dailyLimit Int @default(50) @map("daily_limit") // Runs per day
|
||||
dailyUsed Int @default(0) @map("daily_used")
|
||||
lastResetAt DateTime @default(now()) @map("last_reset_at")
|
||||
|
||||
// Timestamps
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
|
||||
@@index([userId])
|
||||
@@map("scenario_quotas")
|
||||
}
|
||||
|
|
|
|||
2681
pnpm-lock.yaml
generated
2681
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load diff
184
scripts/migrate-use-cases-to-scenarios.ts
Normal file
184
scripts/migrate-use-cases-to-scenarios.ts
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
/**
|
||||
* Migration script: Convert existing Collection.useCases (JSON) to Scenario records
|
||||
*
|
||||
* This script:
|
||||
* 1. Finds all collections with useCases
|
||||
* 2. Creates a Scenario for each use case
|
||||
* 3. Generates embeddings for similarity detection
|
||||
* 4. Generates AI tags for categorization
|
||||
*
|
||||
* Run with: npx tsx scripts/migrate-use-cases-to-scenarios.ts
|
||||
*/
|
||||
|
||||
// Direct import since this script runs standalone
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
|
||||
// Use case structure from the existing generator
|
||||
interface ToolStep {
|
||||
toolName: string;
|
||||
packageName: string;
|
||||
purpose: string;
|
||||
order: number;
|
||||
}
|
||||
|
||||
interface UseCase {
|
||||
id: string;
|
||||
userPrompt: string;
|
||||
description: string;
|
||||
toolSequence: ToolStep[];
|
||||
}
|
||||
|
||||
async function computeEmbedding(text: string): Promise<number[] | null> {
|
||||
// Skip embedding generation in migration - we'll generate them lazily later
|
||||
// This keeps the migration fast and doesn't require API keys
|
||||
console.log(` [skip] Embedding generation deferred for: "${text.slice(0, 50)}..."`);
|
||||
return null;
|
||||
}
|
||||
|
||||
async function generateTags(prompt: string, description: string): Promise<string[]> {
|
||||
// Generate basic tags from the prompt/description
|
||||
// Real AI-based tag generation will happen when scenarios are viewed
|
||||
const words = `${prompt} ${description}`.toLowerCase();
|
||||
const tags: string[] = [];
|
||||
|
||||
// Simple keyword extraction
|
||||
if (words.includes('scrape') || words.includes('crawl') || words.includes('fetch')) {
|
||||
tags.push('web-scraping');
|
||||
}
|
||||
if (words.includes('api') || words.includes('endpoint')) {
|
||||
tags.push('api');
|
||||
}
|
||||
if (words.includes('data') || words.includes('extract')) {
|
||||
tags.push('data-extraction');
|
||||
}
|
||||
if (words.includes('search') || words.includes('find')) {
|
||||
tags.push('search');
|
||||
}
|
||||
if (words.includes('code') || words.includes('debug') || words.includes('fix')) {
|
||||
tags.push('development');
|
||||
}
|
||||
if (words.includes('file') || words.includes('document')) {
|
||||
tags.push('files');
|
||||
}
|
||||
if (words.includes('image') || words.includes('screenshot')) {
|
||||
tags.push('media');
|
||||
}
|
||||
if (words.includes('email') || words.includes('message')) {
|
||||
tags.push('communication');
|
||||
}
|
||||
if (words.includes('monitor') || words.includes('track')) {
|
||||
tags.push('monitoring');
|
||||
}
|
||||
if (words.includes('automate') || words.includes('workflow')) {
|
||||
tags.push('automation');
|
||||
}
|
||||
|
||||
return tags.length > 0 ? tags : ['general'];
|
||||
}
|
||||
|
||||
async function migrateUseCases() {
|
||||
console.log('Starting use cases to scenarios migration...\n');
|
||||
|
||||
// Find all collections with useCases
|
||||
const collections = await prisma.collection.findMany({
|
||||
where: {
|
||||
useCases: { not: null },
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
useCases: true,
|
||||
},
|
||||
});
|
||||
|
||||
console.log(`Found ${collections.length} collections with use cases to migrate.\n`);
|
||||
|
||||
let totalMigrated = 0;
|
||||
let totalSkipped = 0;
|
||||
let totalErrors = 0;
|
||||
|
||||
for (const collection of collections) {
|
||||
console.log(`Processing collection: "${collection.name}" (${collection.id})`);
|
||||
|
||||
const useCases = collection.useCases as { useCases: UseCase[] } | null;
|
||||
if (!useCases || !useCases.useCases || !Array.isArray(useCases.useCases)) {
|
||||
console.log(` [skip] No valid useCases array found\n`);
|
||||
totalSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const useCase of useCases.useCases) {
|
||||
try {
|
||||
// Check if scenario already exists for this prompt
|
||||
const existing = await prisma.scenario.findFirst({
|
||||
where: {
|
||||
collectionId: collection.id,
|
||||
prompt: useCase.userPrompt,
|
||||
},
|
||||
});
|
||||
|
||||
if (existing) {
|
||||
console.log(
|
||||
` [skip] Scenario already exists for: "${useCase.userPrompt.slice(0, 40)}..."`
|
||||
);
|
||||
totalSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Generate tags
|
||||
const tags = await generateTags(useCase.userPrompt, useCase.description);
|
||||
|
||||
// Create the scenario
|
||||
const scenario = await prisma.scenario.create({
|
||||
data: {
|
||||
collectionId: collection.id,
|
||||
prompt: useCase.userPrompt,
|
||||
name: useCase.description,
|
||||
description: `Migrated from legacy use case: ${useCase.id}. Tool sequence: ${useCase.toolSequence.map((t) => t.toolName).join(' → ')}`,
|
||||
tags,
|
||||
},
|
||||
});
|
||||
|
||||
console.log(` [created] Scenario: "${useCase.description.slice(0, 50)}..."`);
|
||||
totalMigrated++;
|
||||
|
||||
// Optionally generate embedding (deferred for now)
|
||||
const embedding = await computeEmbedding(useCase.userPrompt);
|
||||
if (embedding) {
|
||||
await prisma.scenarioEmbedding.create({
|
||||
data: {
|
||||
scenarioId: scenario.id,
|
||||
embedding: embedding as unknown as object,
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(` [error] Failed to migrate use case "${useCase.id}":`, error);
|
||||
totalErrors++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('');
|
||||
}
|
||||
|
||||
console.log('Migration complete!');
|
||||
console.log(` Total migrated: ${totalMigrated}`);
|
||||
console.log(` Total skipped: ${totalSkipped}`);
|
||||
console.log(` Total errors: ${totalErrors}`);
|
||||
}
|
||||
|
||||
// Run the migration
|
||||
migrateUseCases()
|
||||
.then(() => {
|
||||
console.log('\nDone.');
|
||||
process.exit(0);
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error('Migration failed:', error);
|
||||
process.exit(1);
|
||||
})
|
||||
.finally(async () => {
|
||||
await prisma.$disconnect();
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue