feat: add scenarios system for collection testing
Implements the complete scenarios feature for TPMJS: API endpoints: - GET/POST /api/scenarios - list/create scenarios - GET/PATCH/DELETE /api/scenarios/:id - scenario CRUD - POST /api/scenarios/:id/run - execute scenario - GET /api/scenarios/:id/runs - run history - POST /api/scenarios/check-similarity - vector similarity check - GET /api/scenarios/featured - featured scenarios - POST /api/collections/:id/scenarios/generate - AI scenario generation - GET /api/collections/:id/scenarios - list collection scenarios Services: - generate-prompt.ts - AI prompt generation using GPT-4o-mini - similarity.ts - vector embedding and cosine similarity - evaluate.ts - LLM-based success evaluation - execute.ts - scenario execution orchestration CLI commands: - tpm scenario list [collection] - list scenarios - tpm scenario run <collection> - run all scenarios - tpm scenario test <id> - run single scenario - tpm scenario generate <collection> - generate scenarios - tpm scenario info <id> - scenario details Database: - Scenario, ScenarioEmbedding, ScenarioRun, ScenarioQuota models - Streak-based quality scoring - Daily quota management Migration script included for converting existing useCases.
This commit is contained in:
parent
154f000505
commit
ea742c9386
26 changed files with 6922 additions and 282 deletions
184
scripts/migrate-use-cases-to-scenarios.ts
Normal file
184
scripts/migrate-use-cases-to-scenarios.ts
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
/**
|
||||
* Migration script: Convert existing Collection.useCases (JSON) to Scenario records
|
||||
*
|
||||
* This script:
|
||||
* 1. Finds all collections with useCases
|
||||
* 2. Creates a Scenario for each use case
|
||||
* 3. Generates embeddings for similarity detection
|
||||
* 4. Generates AI tags for categorization
|
||||
*
|
||||
* Run with: npx tsx scripts/migrate-use-cases-to-scenarios.ts
|
||||
*/
|
||||
|
||||
// Direct import since this script runs standalone
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
|
||||
// Use case structure from the existing generator
|
||||
interface ToolStep {
|
||||
toolName: string;
|
||||
packageName: string;
|
||||
purpose: string;
|
||||
order: number;
|
||||
}
|
||||
|
||||
interface UseCase {
|
||||
id: string;
|
||||
userPrompt: string;
|
||||
description: string;
|
||||
toolSequence: ToolStep[];
|
||||
}
|
||||
|
||||
async function computeEmbedding(text: string): Promise<number[] | null> {
|
||||
// Skip embedding generation in migration - we'll generate them lazily later
|
||||
// This keeps the migration fast and doesn't require API keys
|
||||
console.log(` [skip] Embedding generation deferred for: "${text.slice(0, 50)}..."`);
|
||||
return null;
|
||||
}
|
||||
|
||||
async function generateTags(prompt: string, description: string): Promise<string[]> {
|
||||
// Generate basic tags from the prompt/description
|
||||
// Real AI-based tag generation will happen when scenarios are viewed
|
||||
const words = `${prompt} ${description}`.toLowerCase();
|
||||
const tags: string[] = [];
|
||||
|
||||
// Simple keyword extraction
|
||||
if (words.includes('scrape') || words.includes('crawl') || words.includes('fetch')) {
|
||||
tags.push('web-scraping');
|
||||
}
|
||||
if (words.includes('api') || words.includes('endpoint')) {
|
||||
tags.push('api');
|
||||
}
|
||||
if (words.includes('data') || words.includes('extract')) {
|
||||
tags.push('data-extraction');
|
||||
}
|
||||
if (words.includes('search') || words.includes('find')) {
|
||||
tags.push('search');
|
||||
}
|
||||
if (words.includes('code') || words.includes('debug') || words.includes('fix')) {
|
||||
tags.push('development');
|
||||
}
|
||||
if (words.includes('file') || words.includes('document')) {
|
||||
tags.push('files');
|
||||
}
|
||||
if (words.includes('image') || words.includes('screenshot')) {
|
||||
tags.push('media');
|
||||
}
|
||||
if (words.includes('email') || words.includes('message')) {
|
||||
tags.push('communication');
|
||||
}
|
||||
if (words.includes('monitor') || words.includes('track')) {
|
||||
tags.push('monitoring');
|
||||
}
|
||||
if (words.includes('automate') || words.includes('workflow')) {
|
||||
tags.push('automation');
|
||||
}
|
||||
|
||||
return tags.length > 0 ? tags : ['general'];
|
||||
}
|
||||
|
||||
async function migrateUseCases() {
|
||||
console.log('Starting use cases to scenarios migration...\n');
|
||||
|
||||
// Find all collections with useCases
|
||||
const collections = await prisma.collection.findMany({
|
||||
where: {
|
||||
useCases: { not: null },
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
useCases: true,
|
||||
},
|
||||
});
|
||||
|
||||
console.log(`Found ${collections.length} collections with use cases to migrate.\n`);
|
||||
|
||||
let totalMigrated = 0;
|
||||
let totalSkipped = 0;
|
||||
let totalErrors = 0;
|
||||
|
||||
for (const collection of collections) {
|
||||
console.log(`Processing collection: "${collection.name}" (${collection.id})`);
|
||||
|
||||
const useCases = collection.useCases as { useCases: UseCase[] } | null;
|
||||
if (!useCases || !useCases.useCases || !Array.isArray(useCases.useCases)) {
|
||||
console.log(` [skip] No valid useCases array found\n`);
|
||||
totalSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const useCase of useCases.useCases) {
|
||||
try {
|
||||
// Check if scenario already exists for this prompt
|
||||
const existing = await prisma.scenario.findFirst({
|
||||
where: {
|
||||
collectionId: collection.id,
|
||||
prompt: useCase.userPrompt,
|
||||
},
|
||||
});
|
||||
|
||||
if (existing) {
|
||||
console.log(
|
||||
` [skip] Scenario already exists for: "${useCase.userPrompt.slice(0, 40)}..."`
|
||||
);
|
||||
totalSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Generate tags
|
||||
const tags = await generateTags(useCase.userPrompt, useCase.description);
|
||||
|
||||
// Create the scenario
|
||||
const scenario = await prisma.scenario.create({
|
||||
data: {
|
||||
collectionId: collection.id,
|
||||
prompt: useCase.userPrompt,
|
||||
name: useCase.description,
|
||||
description: `Migrated from legacy use case: ${useCase.id}. Tool sequence: ${useCase.toolSequence.map((t) => t.toolName).join(' → ')}`,
|
||||
tags,
|
||||
},
|
||||
});
|
||||
|
||||
console.log(` [created] Scenario: "${useCase.description.slice(0, 50)}..."`);
|
||||
totalMigrated++;
|
||||
|
||||
// Optionally generate embedding (deferred for now)
|
||||
const embedding = await computeEmbedding(useCase.userPrompt);
|
||||
if (embedding) {
|
||||
await prisma.scenarioEmbedding.create({
|
||||
data: {
|
||||
scenarioId: scenario.id,
|
||||
embedding: embedding as unknown as object,
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(` [error] Failed to migrate use case "${useCase.id}":`, error);
|
||||
totalErrors++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('');
|
||||
}
|
||||
|
||||
console.log('Migration complete!');
|
||||
console.log(` Total migrated: ${totalMigrated}`);
|
||||
console.log(` Total skipped: ${totalSkipped}`);
|
||||
console.log(` Total errors: ${totalErrors}`);
|
||||
}
|
||||
|
||||
// Run the migration
|
||||
migrateUseCases()
|
||||
.then(() => {
|
||||
console.log('\nDone.');
|
||||
process.exit(0);
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error('Migration failed:', error);
|
||||
process.exit(1);
|
||||
})
|
||||
.finally(async () => {
|
||||
await prisma.$disconnect();
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue