feat: add scenarios system for collection testing
Implements the complete scenarios feature for TPMJS: API endpoints: - GET/POST /api/scenarios - list/create scenarios - GET/PATCH/DELETE /api/scenarios/:id - scenario CRUD - POST /api/scenarios/:id/run - execute scenario - GET /api/scenarios/:id/runs - run history - POST /api/scenarios/check-similarity - vector similarity check - GET /api/scenarios/featured - featured scenarios - POST /api/collections/:id/scenarios/generate - AI scenario generation - GET /api/collections/:id/scenarios - list collection scenarios Services: - generate-prompt.ts - AI prompt generation using GPT-4o-mini - similarity.ts - vector embedding and cosine similarity - evaluate.ts - LLM-based success evaluation - execute.ts - scenario execution orchestration CLI commands: - tpm scenario list [collection] - list scenarios - tpm scenario run <collection> - run all scenarios - tpm scenario test <id> - run single scenario - tpm scenario generate <collection> - generate scenarios - tpm scenario info <id> - scenario details Database: - Scenario, ScenarioEmbedding, ScenarioRun, ScenarioQuota models - Streak-based quality scoring - Daily quota management Migration script included for converting existing useCases.
This commit is contained in:
Ajax Davis
parent
154f000505
commit
ea742c9386
26 changed files with 6922 additions and 282 deletions
|
|
@ -4,9 +4,7 @@
|
|||
"aliases": [],
|
||||
"args": {},
|
||||
"description": "Run diagnostic checks for TPMJS CLI",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %>"
|
||||
],
|
||||
"examples": ["<%= config.bin %> <%= command.id %>"],
|
||||
"flags": {
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
|
|
@ -31,11 +29,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"doctor.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "doctor.js"]
|
||||
},
|
||||
"playground": {
|
||||
"aliases": [],
|
||||
|
|
@ -79,19 +73,13 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"playground.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "playground.js"]
|
||||
},
|
||||
"update": {
|
||||
"aliases": [],
|
||||
"args": {},
|
||||
"description": "Update the TPMJS CLI to the latest version",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %>"
|
||||
],
|
||||
"examples": ["<%= config.bin %> <%= command.id %>"],
|
||||
"flags": {
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
|
|
@ -122,11 +110,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"update.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "update.js"]
|
||||
},
|
||||
"agent:chat": {
|
||||
"aliases": [],
|
||||
|
|
@ -186,12 +170,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"agent",
|
||||
"chat.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "agent", "chat.js"]
|
||||
},
|
||||
"agent:create": {
|
||||
"aliases": [],
|
||||
|
|
@ -233,13 +212,7 @@
|
|||
"required": true,
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"options": [
|
||||
"ANTHROPIC",
|
||||
"OPENAI",
|
||||
"GOOGLE",
|
||||
"GROQ",
|
||||
"MISTRAL"
|
||||
],
|
||||
"options": ["ANTHROPIC", "OPENAI", "GOOGLE", "GROQ", "MISTRAL"],
|
||||
"type": "option"
|
||||
},
|
||||
"model": {
|
||||
|
|
@ -297,12 +270,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"agent",
|
||||
"create.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "agent", "create.js"]
|
||||
},
|
||||
"agent:delete": {
|
||||
"aliases": [],
|
||||
|
|
@ -349,12 +317,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"agent",
|
||||
"delete.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "agent", "delete.js"]
|
||||
},
|
||||
"agent:list": {
|
||||
"aliases": [],
|
||||
|
|
@ -406,12 +369,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"agent",
|
||||
"list.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "agent", "list.js"]
|
||||
},
|
||||
"agent:update": {
|
||||
"aliases": [],
|
||||
|
|
@ -457,13 +415,7 @@
|
|||
"name": "provider",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"options": [
|
||||
"ANTHROPIC",
|
||||
"OPENAI",
|
||||
"GOOGLE",
|
||||
"GROQ",
|
||||
"MISTRAL"
|
||||
],
|
||||
"options": ["ANTHROPIC", "OPENAI", "GOOGLE", "GROQ", "MISTRAL"],
|
||||
"type": "option"
|
||||
},
|
||||
"model": {
|
||||
|
|
@ -519,12 +471,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"agent",
|
||||
"update.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "agent", "update.js"]
|
||||
},
|
||||
"auth:login": {
|
||||
"aliases": [],
|
||||
|
|
@ -579,20 +526,13 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"auth",
|
||||
"login.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "auth", "login.js"]
|
||||
},
|
||||
"auth:logout": {
|
||||
"aliases": [],
|
||||
"args": {},
|
||||
"description": "Log out from TPMJS",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %>"
|
||||
],
|
||||
"examples": ["<%= config.bin %> <%= command.id %>"],
|
||||
"flags": {
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
|
|
@ -610,20 +550,13 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"auth",
|
||||
"logout.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "auth", "logout.js"]
|
||||
},
|
||||
"auth:status": {
|
||||
"aliases": [],
|
||||
"args": {},
|
||||
"description": "Show authentication status",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %>"
|
||||
],
|
||||
"examples": ["<%= config.bin %> <%= command.id %>"],
|
||||
"flags": {
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
|
|
@ -648,20 +581,13 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"auth",
|
||||
"status.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "auth", "status.js"]
|
||||
},
|
||||
"auth:whoami": {
|
||||
"aliases": [],
|
||||
"args": {},
|
||||
"description": "Show current user information",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %>"
|
||||
],
|
||||
"examples": ["<%= config.bin %> <%= command.id %>"],
|
||||
"flags": {
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
|
|
@ -686,12 +612,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"auth",
|
||||
"whoami.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "auth", "whoami.js"]
|
||||
},
|
||||
"collection:add": {
|
||||
"aliases": [],
|
||||
|
|
@ -731,12 +652,7 @@
|
|||
"strict": false,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"collection",
|
||||
"add.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "collection", "add.js"]
|
||||
},
|
||||
"collection:create": {
|
||||
"aliases": [],
|
||||
|
|
@ -793,12 +709,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"collection",
|
||||
"create.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "collection", "create.js"]
|
||||
},
|
||||
"collection:delete": {
|
||||
"aliases": [],
|
||||
|
|
@ -845,12 +756,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"collection",
|
||||
"delete.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "collection", "delete.js"]
|
||||
},
|
||||
"collection:import": {
|
||||
"aliases": [],
|
||||
|
|
@ -899,12 +805,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"collection",
|
||||
"import.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "collection", "import.js"]
|
||||
},
|
||||
"collection:list": {
|
||||
"aliases": [],
|
||||
|
|
@ -956,12 +857,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"collection",
|
||||
"list.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "collection", "list.js"]
|
||||
},
|
||||
"collection:remove": {
|
||||
"aliases": [],
|
||||
|
|
@ -978,9 +874,7 @@
|
|||
}
|
||||
},
|
||||
"description": "Remove a tool from a collection",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %> my-collection tool-id-1"
|
||||
],
|
||||
"examples": ["<%= config.bin %> <%= command.id %> my-collection tool-id-1"],
|
||||
"flags": {
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
|
|
@ -1005,12 +899,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"collection",
|
||||
"remove.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "collection", "remove.js"]
|
||||
},
|
||||
"collection:update": {
|
||||
"aliases": [],
|
||||
|
|
@ -1072,12 +961,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"collection",
|
||||
"update.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "collection", "update.js"]
|
||||
},
|
||||
"mcp:config": {
|
||||
"aliases": [],
|
||||
|
|
@ -1102,12 +986,7 @@
|
|||
"default": "claude",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"options": [
|
||||
"claude",
|
||||
"cursor",
|
||||
"windsurf",
|
||||
"generic"
|
||||
],
|
||||
"options": ["claude", "cursor", "windsurf", "generic"],
|
||||
"type": "option"
|
||||
},
|
||||
"output": {
|
||||
|
|
@ -1142,12 +1021,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"mcp",
|
||||
"config.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "mcp", "config.js"]
|
||||
},
|
||||
"mcp:serve": {
|
||||
"aliases": [],
|
||||
|
|
@ -1208,12 +1082,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"mcp",
|
||||
"serve.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "mcp", "serve.js"]
|
||||
},
|
||||
"publish:check": {
|
||||
"aliases": [],
|
||||
|
|
@ -1253,12 +1122,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"publish",
|
||||
"check.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "publish", "check.js"]
|
||||
},
|
||||
"publish:preview": {
|
||||
"aliases": [],
|
||||
|
|
@ -1301,12 +1165,271 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"publish",
|
||||
"preview.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "publish", "preview.js"]
|
||||
},
|
||||
"scenario:generate": {
|
||||
"aliases": [],
|
||||
"args": {
|
||||
"collection": {
|
||||
"description": "Collection ID or slug",
|
||||
"name": "collection",
|
||||
"required": true
|
||||
}
|
||||
},
|
||||
"description": "Generate AI-powered scenarios for a collection",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %> my-collection",
|
||||
"<%= config.bin %> <%= command.id %> my-collection --count 3",
|
||||
"<%= config.bin %> <%= command.id %> my-collection --skip-similarity-check"
|
||||
],
|
||||
"flags": {
|
||||
"count": {
|
||||
"char": "n",
|
||||
"description": "Number of scenarios to generate (1-10)",
|
||||
"name": "count",
|
||||
"default": 1,
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"skip-similarity-check": {
|
||||
"description": "Skip checking for similar existing scenarios",
|
||||
"name": "skip-similarity-check",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
"name": "json",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"verbose": {
|
||||
"char": "v",
|
||||
"description": "Show verbose output",
|
||||
"name": "verbose",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"hasDynamicHelp": false,
|
||||
"hiddenAliases": [],
|
||||
"id": "scenario:generate",
|
||||
"pluginAlias": "@tpmjs/cli",
|
||||
"pluginName": "@tpmjs/cli",
|
||||
"pluginType": "core",
|
||||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "scenario", "generate.js"]
|
||||
},
|
||||
"scenario:info": {
|
||||
"aliases": [],
|
||||
"args": {
|
||||
"scenarioId": {
|
||||
"description": "Scenario ID",
|
||||
"name": "scenarioId",
|
||||
"required": true
|
||||
}
|
||||
},
|
||||
"description": "Show detailed information about a scenario",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %> clu123abc456",
|
||||
"<%= config.bin %> <%= command.id %> clu123abc456 --runs 20",
|
||||
"<%= config.bin %> <%= command.id %> clu123abc456 --json"
|
||||
],
|
||||
"flags": {
|
||||
"runs": {
|
||||
"char": "r",
|
||||
"description": "Number of recent runs to show",
|
||||
"name": "runs",
|
||||
"default": 10,
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
"name": "json",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"verbose": {
|
||||
"char": "v",
|
||||
"description": "Show verbose output",
|
||||
"name": "verbose",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"hasDynamicHelp": false,
|
||||
"hiddenAliases": [],
|
||||
"id": "scenario:info",
|
||||
"pluginAlias": "@tpmjs/cli",
|
||||
"pluginName": "@tpmjs/cli",
|
||||
"pluginType": "core",
|
||||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "scenario", "info.js"]
|
||||
},
|
||||
"scenario:list": {
|
||||
"aliases": [],
|
||||
"args": {
|
||||
"collection": {
|
||||
"description": "Collection ID or slug (optional - shows all public scenarios if omitted)",
|
||||
"name": "collection",
|
||||
"required": false
|
||||
}
|
||||
},
|
||||
"description": "List scenarios for a collection or all public scenarios",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %>",
|
||||
"<%= config.bin %> <%= command.id %> my-collection",
|
||||
"<%= config.bin %> <%= command.id %> --limit 20 --json"
|
||||
],
|
||||
"flags": {
|
||||
"limit": {
|
||||
"char": "l",
|
||||
"description": "Maximum number of results",
|
||||
"name": "limit",
|
||||
"default": 20,
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"offset": {
|
||||
"char": "o",
|
||||
"description": "Offset for pagination",
|
||||
"name": "offset",
|
||||
"default": 0,
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"tags": {
|
||||
"char": "t",
|
||||
"description": "Filter by tags (comma-separated)",
|
||||
"name": "tags",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
"name": "json",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"verbose": {
|
||||
"char": "v",
|
||||
"description": "Show verbose output",
|
||||
"name": "verbose",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"hasDynamicHelp": false,
|
||||
"hiddenAliases": [],
|
||||
"id": "scenario:list",
|
||||
"pluginAlias": "@tpmjs/cli",
|
||||
"pluginName": "@tpmjs/cli",
|
||||
"pluginType": "core",
|
||||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "scenario", "list.js"]
|
||||
},
|
||||
"scenario:run": {
|
||||
"aliases": [],
|
||||
"args": {
|
||||
"collection": {
|
||||
"description": "Collection ID or slug",
|
||||
"name": "collection",
|
||||
"required": true
|
||||
}
|
||||
},
|
||||
"description": "Run all scenarios for a collection",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %> my-collection",
|
||||
"<%= config.bin %> <%= command.id %> my-collection --json",
|
||||
"<%= config.bin %> <%= command.id %> my-collection --verbose"
|
||||
],
|
||||
"flags": {
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
"name": "json",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"verbose": {
|
||||
"char": "v",
|
||||
"description": "Show verbose output",
|
||||
"name": "verbose",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"limit": {
|
||||
"char": "l",
|
||||
"description": "Maximum number of scenarios to run",
|
||||
"name": "limit",
|
||||
"default": 50,
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
}
|
||||
},
|
||||
"hasDynamicHelp": false,
|
||||
"hiddenAliases": [],
|
||||
"id": "scenario:run",
|
||||
"pluginAlias": "@tpmjs/cli",
|
||||
"pluginName": "@tpmjs/cli",
|
||||
"pluginType": "core",
|
||||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "scenario", "run.js"]
|
||||
},
|
||||
"scenario:test": {
|
||||
"aliases": [],
|
||||
"args": {
|
||||
"scenarioId": {
|
||||
"description": "Scenario ID to run",
|
||||
"name": "scenarioId",
|
||||
"required": true
|
||||
}
|
||||
},
|
||||
"description": "Run a single scenario by ID",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %> clu123abc456",
|
||||
"<%= config.bin %> <%= command.id %> clu123abc456 --json",
|
||||
"<%= config.bin %> <%= command.id %> clu123abc456 --verbose"
|
||||
],
|
||||
"flags": {
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
"name": "json",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"verbose": {
|
||||
"char": "v",
|
||||
"description": "Show verbose output including full reason",
|
||||
"name": "verbose",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"hasDynamicHelp": false,
|
||||
"hiddenAliases": [],
|
||||
"id": "scenario:test",
|
||||
"pluginAlias": "@tpmjs/cli",
|
||||
"pluginName": "@tpmjs/cli",
|
||||
"pluginType": "core",
|
||||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "scenario", "test.js"]
|
||||
},
|
||||
"tool:execute": {
|
||||
"aliases": [],
|
||||
|
|
@ -1379,12 +1502,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"tool",
|
||||
"execute.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "tool", "execute.js"]
|
||||
},
|
||||
"tool:info": {
|
||||
"aliases": [],
|
||||
|
|
@ -1429,12 +1547,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"tool",
|
||||
"info.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "tool", "info.js"]
|
||||
},
|
||||
"tool:init": {
|
||||
"aliases": [],
|
||||
|
|
@ -1459,10 +1572,7 @@
|
|||
"default": "minimal",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"options": [
|
||||
"minimal",
|
||||
"rich"
|
||||
],
|
||||
"options": ["minimal", "rich"],
|
||||
"type": "option"
|
||||
},
|
||||
"category": {
|
||||
|
|
@ -1519,12 +1629,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"tool",
|
||||
"init.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "tool", "init.js"]
|
||||
},
|
||||
"tool:search": {
|
||||
"aliases": [],
|
||||
|
|
@ -1591,12 +1696,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"tool",
|
||||
"search.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "tool", "search.js"]
|
||||
},
|
||||
"tool:trending": {
|
||||
"aliases": [],
|
||||
|
|
@ -1639,12 +1739,7 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"tool",
|
||||
"trending.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "tool", "trending.js"]
|
||||
},
|
||||
"tool:validate": {
|
||||
"aliases": [],
|
||||
|
|
@ -1687,13 +1782,8 @@
|
|||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": [
|
||||
"dist",
|
||||
"commands",
|
||||
"tool",
|
||||
"validate.js"
|
||||
]
|
||||
"relativePath": ["dist", "commands", "tool", "validate.js"]
|
||||
}
|
||||
},
|
||||
"version": "0.1.2"
|
||||
}
|
||||
"version": "0.1.3"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@tpmjs/cli",
|
||||
"version": "0.1.2",
|
||||
"version": "0.1.3",
|
||||
"description": "TPMJS command-line interface for AI tool discovery and execution",
|
||||
"author": "TPMJS",
|
||||
"license": "MIT",
|
||||
|
|
@ -55,6 +55,9 @@
|
|||
"collection": {
|
||||
"description": "Collection management"
|
||||
},
|
||||
"scenario": {
|
||||
"description": "Test scenario management and execution"
|
||||
},
|
||||
"auth": {
|
||||
"description": "Authentication"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import { Args, Command, Flags } from '@oclif/core';
|
||||
import open from 'open';
|
||||
import { createServer } from 'node:http';
|
||||
import { URL } from 'node:url';
|
||||
import { saveCredentials, getApiUrl } from '../../lib/config.js';
|
||||
import { Args, Command, Flags } from '@oclif/core';
|
||||
import open from 'open';
|
||||
import { TpmClient } from '../../lib/api-client.js';
|
||||
import { getApiUrl, saveCredentials } from '../../lib/config.js';
|
||||
import { createOutput } from '../../lib/output.js';
|
||||
|
||||
export default class Login extends Command {
|
||||
|
|
@ -62,7 +62,9 @@ export default class Login extends Command {
|
|||
output.listItem('tpm auth login --api-key <your-api-key>');
|
||||
output.listItem('tpm auth login --browser (opens browser for OAuth)');
|
||||
output.newLine();
|
||||
output.text(`Get your API key at: ${output.link('tpmjs.com/dashboard/settings/tpmjs-api-keys', 'https://tpmjs.com/dashboard/settings/tpmjs-api-keys')}`);
|
||||
output.text(
|
||||
`Get your API key at: ${output.link('tpmjs.com/dashboard/settings/tpmjs-api-keys', 'https://tpmjs.com/dashboard/settings/tpmjs-api-keys')}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -122,6 +124,13 @@ export default class Login extends Command {
|
|||
output.info('Opening browser for authentication...');
|
||||
|
||||
return new Promise((resolve) => {
|
||||
let timeoutId: NodeJS.Timeout;
|
||||
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeoutId);
|
||||
server.close();
|
||||
};
|
||||
|
||||
const server = createServer(async (req, res) => {
|
||||
const url = new URL(req.url ?? '/', `http://localhost:${port}`);
|
||||
|
||||
|
|
@ -132,8 +141,10 @@ export default class Login extends Command {
|
|||
|
||||
if (error) {
|
||||
res.writeHead(200, { 'Content-Type': 'text/html' });
|
||||
res.end('<html><body><h1>Authentication Failed</h1><p>You can close this window.</p></body></html>');
|
||||
server.close();
|
||||
res.end(
|
||||
'<html><body><h1>Authentication Failed</h1><p>You can close this window.</p></body></html>'
|
||||
);
|
||||
cleanup();
|
||||
output.error(`Authentication failed: ${error}`);
|
||||
resolve();
|
||||
return;
|
||||
|
|
@ -141,8 +152,10 @@ export default class Login extends Command {
|
|||
|
||||
if (receivedState !== state) {
|
||||
res.writeHead(200, { 'Content-Type': 'text/html' });
|
||||
res.end('<html><body><h1>Invalid State</h1><p>Authentication failed due to invalid state.</p></body></html>');
|
||||
server.close();
|
||||
res.end(
|
||||
'<html><body><h1>Invalid State</h1><p>Authentication failed due to invalid state.</p></body></html>'
|
||||
);
|
||||
cleanup();
|
||||
output.error('Authentication failed: Invalid state parameter');
|
||||
resolve();
|
||||
return;
|
||||
|
|
@ -152,8 +165,10 @@ export default class Login extends Command {
|
|||
saveCredentials({ apiKey });
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'text/html' });
|
||||
res.end('<html><body><h1>Success!</h1><p>You are now logged in. You can close this window.</p></body></html>');
|
||||
server.close();
|
||||
res.end(
|
||||
'<html><body><h1>Success!</h1><p>You are now logged in. You can close this window.</p></body></html>'
|
||||
);
|
||||
cleanup();
|
||||
|
||||
output.success('Logged in successfully via browser');
|
||||
|
||||
|
|
@ -165,7 +180,7 @@ export default class Login extends Command {
|
|||
} else {
|
||||
res.writeHead(200, { 'Content-Type': 'text/html' });
|
||||
res.end('<html><body><h1>Error</h1><p>No API key received.</p></body></html>');
|
||||
server.close();
|
||||
cleanup();
|
||||
output.error('No API key received from authentication');
|
||||
resolve();
|
||||
}
|
||||
|
|
@ -190,11 +205,14 @@ export default class Login extends Command {
|
|||
});
|
||||
|
||||
// Timeout after 5 minutes
|
||||
setTimeout(() => {
|
||||
server.close();
|
||||
output.error('Authentication timed out');
|
||||
resolve();
|
||||
}, 5 * 60 * 1000);
|
||||
timeoutId = setTimeout(
|
||||
() => {
|
||||
server.close();
|
||||
output.error('Authentication timed out');
|
||||
resolve();
|
||||
},
|
||||
5 * 60 * 1000
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
164
packages/cli/src/commands/scenario/generate.ts
Normal file
164
packages/cli/src/commands/scenario/generate.ts
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
import { Args, Command, Flags } from '@oclif/core';
|
||||
import { getClient } from '../../lib/api-client.js';
|
||||
import { createOutput } from '../../lib/output.js';
|
||||
|
||||
export default class ScenarioGenerate extends Command {
|
||||
static description = 'Generate AI-powered scenarios for a collection';
|
||||
|
||||
static examples = [
|
||||
'<%= config.bin %> <%= command.id %> my-collection',
|
||||
'<%= config.bin %> <%= command.id %> my-collection --count 3',
|
||||
'<%= config.bin %> <%= command.id %> my-collection --skip-similarity-check',
|
||||
];
|
||||
|
||||
static args = {
|
||||
collection: Args.string({
|
||||
description: 'Collection ID or slug',
|
||||
required: true,
|
||||
}),
|
||||
};
|
||||
|
||||
static flags = {
|
||||
count: Flags.integer({
|
||||
char: 'n',
|
||||
description: 'Number of scenarios to generate (1-10)',
|
||||
default: 1,
|
||||
min: 1,
|
||||
max: 10,
|
||||
}),
|
||||
'skip-similarity-check': Flags.boolean({
|
||||
description: 'Skip checking for similar existing scenarios',
|
||||
default: false,
|
||||
}),
|
||||
json: Flags.boolean({
|
||||
description: 'Output in JSON format',
|
||||
default: false,
|
||||
}),
|
||||
verbose: Flags.boolean({
|
||||
char: 'v',
|
||||
description: 'Show verbose output',
|
||||
default: false,
|
||||
}),
|
||||
};
|
||||
|
||||
async run(): Promise<void> {
|
||||
const { args, flags } = await this.parse(ScenarioGenerate);
|
||||
const output = createOutput(flags);
|
||||
const client = getClient();
|
||||
|
||||
if (!client.isAuthenticated()) {
|
||||
output.error('Not authenticated. Run `tpm auth login` first.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Find collection
|
||||
const collectionsSpinner = output.spinner('Finding collection...');
|
||||
let collectionId: string;
|
||||
let collectionName: string;
|
||||
|
||||
try {
|
||||
const collections = await client.listCollections({ limit: 100 });
|
||||
const collection = collections.data.find(
|
||||
(c) => c.id === args.collection || c.slug === args.collection
|
||||
);
|
||||
|
||||
if (!collection) {
|
||||
collectionsSpinner.fail('Collection not found');
|
||||
output.error(`No collection found with ID or slug: ${args.collection}`);
|
||||
return;
|
||||
}
|
||||
|
||||
collectionId = collection.id;
|
||||
collectionName = collection.name;
|
||||
collectionsSpinner.stop();
|
||||
} catch (error) {
|
||||
collectionsSpinner.fail('Failed to find collection');
|
||||
output.error(error instanceof Error ? error.message : 'Unknown error');
|
||||
return;
|
||||
}
|
||||
|
||||
// Generate scenarios
|
||||
const generateSpinner = output.spinner(
|
||||
`Generating ${flags.count} scenario${flags.count > 1 ? 's' : ''} for "${collectionName}"...`
|
||||
);
|
||||
|
||||
try {
|
||||
const result = await client.generateScenarios(collectionId, {
|
||||
count: flags.count,
|
||||
skipSimilarityCheck: flags['skip-similarity-check'],
|
||||
});
|
||||
|
||||
const scenarios =
|
||||
(
|
||||
result as unknown as {
|
||||
data: {
|
||||
scenarios: Array<{
|
||||
scenario: {
|
||||
id: string;
|
||||
name: string;
|
||||
prompt: string;
|
||||
tags: string[];
|
||||
};
|
||||
similarity?: {
|
||||
hasSimilar: boolean;
|
||||
maxSimilarity: number;
|
||||
similar: Array<{ name: string; similarity: number }>;
|
||||
};
|
||||
}>;
|
||||
};
|
||||
}
|
||||
).data?.scenarios || [];
|
||||
|
||||
generateSpinner.succeed(
|
||||
`Generated ${scenarios.length} scenario${scenarios.length > 1 ? 's' : ''}`
|
||||
);
|
||||
|
||||
if (flags.json) {
|
||||
output.json({ collection: collectionName, scenarios });
|
||||
return;
|
||||
}
|
||||
|
||||
output.newLine();
|
||||
|
||||
for (let i = 0; i < scenarios.length; i++) {
|
||||
const item = scenarios[i];
|
||||
if (!item) continue;
|
||||
const { scenario, similarity } = item;
|
||||
output.text(output.bold(`${i + 1}. ${scenario.name}`));
|
||||
output.text(` ID: ${scenario.id}`);
|
||||
|
||||
if (flags.verbose) {
|
||||
output.text(` Prompt: ${scenario.prompt}`);
|
||||
} else {
|
||||
output.text(` Prompt: ${scenario.prompt.slice(0, 80)}...`);
|
||||
}
|
||||
|
||||
if (scenario.tags.length > 0) {
|
||||
output.text(` Tags: ${scenario.tags.join(', ')}`);
|
||||
}
|
||||
|
||||
if (similarity?.hasSimilar) {
|
||||
output.text(
|
||||
output.yellow(` ⚠ Similar to existing: ${similarity.maxSimilarity}% match`)
|
||||
);
|
||||
if (flags.verbose && similarity.similar.length > 0) {
|
||||
for (const s of similarity.similar) {
|
||||
output.text(output.dim(` - "${s.name}" (${s.similarity}% similar)`));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
output.newLine();
|
||||
}
|
||||
|
||||
output.text(output.dim('Run these scenarios with:'));
|
||||
output.text(output.dim(` tpm scenario run ${args.collection}`));
|
||||
} catch (error) {
|
||||
generateSpinner.fail('Failed to generate scenarios');
|
||||
output.error(
|
||||
error instanceof Error ? error.message : 'Unknown error',
|
||||
flags.verbose ? String(error) : undefined
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
190
packages/cli/src/commands/scenario/info.ts
Normal file
190
packages/cli/src/commands/scenario/info.ts
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
import { Args, Command, Flags } from '@oclif/core';
|
||||
import { getClient } from '../../lib/api-client.js';
|
||||
import { createOutput } from '../../lib/output.js';
|
||||
|
||||
export default class ScenarioInfo extends Command {
|
||||
static description = 'Show detailed information about a scenario';
|
||||
|
||||
static examples = [
|
||||
'<%= config.bin %> <%= command.id %> clu123abc456',
|
||||
'<%= config.bin %> <%= command.id %> clu123abc456 --runs 20',
|
||||
'<%= config.bin %> <%= command.id %> clu123abc456 --json',
|
||||
];
|
||||
|
||||
static args = {
|
||||
scenarioId: Args.string({
|
||||
description: 'Scenario ID',
|
||||
required: true,
|
||||
}),
|
||||
};
|
||||
|
||||
static flags = {
|
||||
runs: Flags.integer({
|
||||
char: 'r',
|
||||
description: 'Number of recent runs to show',
|
||||
default: 10,
|
||||
}),
|
||||
json: Flags.boolean({
|
||||
description: 'Output in JSON format',
|
||||
default: false,
|
||||
}),
|
||||
verbose: Flags.boolean({
|
||||
char: 'v',
|
||||
description: 'Show verbose output',
|
||||
default: false,
|
||||
}),
|
||||
};
|
||||
|
||||
async run(): Promise<void> {
|
||||
const { args, flags } = await this.parse(ScenarioInfo);
|
||||
const output = createOutput(flags);
|
||||
const client = getClient();
|
||||
|
||||
const spinner = output.spinner('Fetching scenario...');
|
||||
|
||||
try {
|
||||
const response = await client.getScenario(args.scenarioId);
|
||||
const scenario = (
|
||||
response as unknown as {
|
||||
data: {
|
||||
id: string;
|
||||
name: string | null;
|
||||
prompt: string;
|
||||
description: string | null;
|
||||
tags: string[];
|
||||
qualityScore: number;
|
||||
consecutivePasses: number;
|
||||
consecutiveFails: number;
|
||||
totalRuns: number;
|
||||
lastRunAt: string | null;
|
||||
lastRunStatus: string | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
collection?: {
|
||||
id: string;
|
||||
name: string;
|
||||
slug: string | null;
|
||||
username: string | null;
|
||||
};
|
||||
recentRuns?: Array<{
|
||||
id: string;
|
||||
status: string;
|
||||
evaluatorVerdict: string | null;
|
||||
executionTimeMs: number | null;
|
||||
createdAt: string;
|
||||
}>;
|
||||
runCount?: number;
|
||||
};
|
||||
}
|
||||
).data;
|
||||
|
||||
spinner.stop();
|
||||
|
||||
if (flags.json) {
|
||||
output.json(scenario);
|
||||
return;
|
||||
}
|
||||
|
||||
// Header
|
||||
output.text(output.bold(scenario.name || 'Unnamed Scenario'));
|
||||
output.text(output.dim(`ID: ${scenario.id}`));
|
||||
output.newLine();
|
||||
|
||||
// Collection info
|
||||
if (scenario.collection) {
|
||||
output.text(output.bold('Collection'));
|
||||
output.text(` Name: ${scenario.collection.name}`);
|
||||
if (scenario.collection.slug) {
|
||||
output.text(` Slug: ${scenario.collection.slug}`);
|
||||
}
|
||||
if (scenario.collection.username) {
|
||||
output.text(` Owner: @${scenario.collection.username}`);
|
||||
}
|
||||
output.newLine();
|
||||
}
|
||||
|
||||
// Prompt
|
||||
output.text(output.bold('Prompt'));
|
||||
if (flags.verbose || scenario.prompt.length <= 200) {
|
||||
output.text(` ${scenario.prompt}`);
|
||||
} else {
|
||||
output.text(` ${scenario.prompt.slice(0, 200)}...`);
|
||||
output.text(output.dim(' (use --verbose to see full prompt)'));
|
||||
}
|
||||
output.newLine();
|
||||
|
||||
// Tags
|
||||
if (scenario.tags && scenario.tags.length > 0) {
|
||||
output.text(output.bold('Tags'));
|
||||
output.text(` ${scenario.tags.join(', ')}`);
|
||||
output.newLine();
|
||||
}
|
||||
|
||||
// Metrics
|
||||
output.text(output.bold('Metrics'));
|
||||
output.text(` Quality Score: ${(scenario.qualityScore * 100).toFixed(1)}%`);
|
||||
output.text(` Total Runs: ${scenario.totalRuns}`);
|
||||
output.text(` Consecutive Passes: ${scenario.consecutivePasses}`);
|
||||
output.text(` Consecutive Fails: ${scenario.consecutiveFails}`);
|
||||
if (scenario.lastRunStatus) {
|
||||
const statusColor =
|
||||
scenario.lastRunStatus === 'pass'
|
||||
? output.green
|
||||
: scenario.lastRunStatus === 'fail'
|
||||
? output.red
|
||||
: output.yellow;
|
||||
output.text(` Last Run Status: ${statusColor(scenario.lastRunStatus)}`);
|
||||
}
|
||||
if (scenario.lastRunAt) {
|
||||
output.text(` Last Run: ${new Date(scenario.lastRunAt).toLocaleString()}`);
|
||||
}
|
||||
output.newLine();
|
||||
|
||||
// Timestamps
|
||||
output.text(output.bold('Timestamps'));
|
||||
output.text(` Created: ${new Date(scenario.createdAt).toLocaleString()}`);
|
||||
output.text(` Updated: ${new Date(scenario.updatedAt).toLocaleString()}`);
|
||||
output.newLine();
|
||||
|
||||
// Recent runs
|
||||
if (scenario.recentRuns && scenario.recentRuns.length > 0) {
|
||||
output.text(
|
||||
output.bold(
|
||||
`Recent Runs (${scenario.recentRuns.length} of ${scenario.runCount ?? scenario.totalRuns})`
|
||||
)
|
||||
);
|
||||
output.table(
|
||||
scenario.recentRuns.map((run) => ({
|
||||
status:
|
||||
run.status === 'pass'
|
||||
? output.green('pass')
|
||||
: run.status === 'fail'
|
||||
? output.red('fail')
|
||||
: output.yellow(run.status),
|
||||
verdict: run.evaluatorVerdict || '-',
|
||||
time: run.executionTimeMs ? `${run.executionTimeMs}ms` : '-',
|
||||
date: new Date(run.createdAt).toLocaleString(),
|
||||
})),
|
||||
[
|
||||
{ key: 'status', header: 'Status', width: 10 },
|
||||
{ key: 'verdict', header: 'Verdict', width: 10 },
|
||||
{ key: 'time', header: 'Time', width: 12 },
|
||||
{ key: 'date', header: 'Date', width: 25 },
|
||||
]
|
||||
);
|
||||
} else {
|
||||
output.text(output.dim('No runs yet'));
|
||||
}
|
||||
|
||||
output.newLine();
|
||||
output.text(output.dim('Run this scenario with:'));
|
||||
output.text(output.dim(` tpm scenario test ${scenario.id}`));
|
||||
} catch (error) {
|
||||
spinner.fail('Failed to fetch scenario');
|
||||
output.error(
|
||||
error instanceof Error ? error.message : 'Unknown error',
|
||||
flags.verbose ? String(error) : undefined
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
181
packages/cli/src/commands/scenario/list.ts
Normal file
181
packages/cli/src/commands/scenario/list.ts
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
import { Args, Command, Flags } from '@oclif/core';
|
||||
import {
|
||||
type ApiResponse,
|
||||
getClient,
|
||||
type PaginatedResponse,
|
||||
type Scenario,
|
||||
} from '../../lib/api-client.js';
|
||||
import { createOutput } from '../../lib/output.js';
|
||||
|
||||
export default class ScenarioList extends Command {
|
||||
static description = 'List scenarios for a collection or all public scenarios';
|
||||
|
||||
static examples = [
|
||||
'<%= config.bin %> <%= command.id %>',
|
||||
'<%= config.bin %> <%= command.id %> my-collection',
|
||||
'<%= config.bin %> <%= command.id %> --limit 20 --json',
|
||||
];
|
||||
|
||||
static args = {
|
||||
collection: Args.string({
|
||||
description: 'Collection ID or slug (optional - shows all public scenarios if omitted)',
|
||||
required: false,
|
||||
}),
|
||||
};
|
||||
|
||||
static flags = {
|
||||
limit: Flags.integer({
|
||||
char: 'l',
|
||||
description: 'Maximum number of results',
|
||||
default: 20,
|
||||
}),
|
||||
offset: Flags.integer({
|
||||
char: 'o',
|
||||
description: 'Offset for pagination',
|
||||
default: 0,
|
||||
}),
|
||||
tags: Flags.string({
|
||||
char: 't',
|
||||
description: 'Filter by tags (comma-separated)',
|
||||
}),
|
||||
json: Flags.boolean({
|
||||
description: 'Output in JSON format',
|
||||
default: false,
|
||||
}),
|
||||
verbose: Flags.boolean({
|
||||
char: 'v',
|
||||
description: 'Show verbose output',
|
||||
default: false,
|
||||
}),
|
||||
};
|
||||
|
||||
async run(): Promise<void> {
|
||||
const { args, flags } = await this.parse(ScenarioList);
|
||||
const output = createOutput(flags);
|
||||
const client = getClient();
|
||||
|
||||
const spinner = output.spinner('Fetching scenarios...');
|
||||
|
||||
try {
|
||||
let response: PaginatedResponse<Scenario> | ApiResponse<{ scenarios: Scenario[] }>;
|
||||
|
||||
if (args.collection) {
|
||||
// Try to find collection by ID or slug
|
||||
const collections = await client.listCollections({ limit: 100 });
|
||||
const collection = collections.data.find(
|
||||
(c) => c.id === args.collection || c.slug === args.collection
|
||||
);
|
||||
|
||||
if (!collection) {
|
||||
spinner.fail('Collection not found');
|
||||
output.error(`No collection found with ID or slug: ${args.collection}`);
|
||||
return;
|
||||
}
|
||||
|
||||
response = await client.listCollectionScenarios(collection.id, {
|
||||
limit: flags.limit,
|
||||
offset: flags.offset,
|
||||
});
|
||||
|
||||
spinner.stop();
|
||||
|
||||
if (flags.json) {
|
||||
output.json(response);
|
||||
return;
|
||||
}
|
||||
|
||||
const scenarios =
|
||||
(
|
||||
response as unknown as {
|
||||
data: {
|
||||
scenarios: Array<{
|
||||
id: string;
|
||||
name: string | null;
|
||||
prompt: string;
|
||||
qualityScore: number;
|
||||
totalRuns: number;
|
||||
lastRunStatus: string | null;
|
||||
tags: string[];
|
||||
}>;
|
||||
};
|
||||
}
|
||||
).data?.scenarios || [];
|
||||
|
||||
if (scenarios.length === 0) {
|
||||
output.info(`No scenarios found for collection "${collection.name}"`);
|
||||
output.text(
|
||||
'Generate some with: tpm scenario generate ' + (collection.slug || collection.id)
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
output.text(output.bold(`Scenarios for ${collection.name}\n`));
|
||||
output.table(
|
||||
scenarios.map((s) => ({
|
||||
name: s.name || s.prompt.slice(0, 30) + '...',
|
||||
quality: (s.qualityScore * 100).toFixed(0) + '%',
|
||||
runs: s.totalRuns,
|
||||
status: s.lastRunStatus || '-',
|
||||
tags: s.tags.slice(0, 3).join(', ') || '-',
|
||||
})),
|
||||
[
|
||||
{ key: 'name', header: 'Name', width: 35 },
|
||||
{ key: 'quality', header: 'Quality', width: 10 },
|
||||
{ key: 'runs', header: 'Runs', width: 8 },
|
||||
{ key: 'status', header: 'Status', width: 8 },
|
||||
{ key: 'tags', header: 'Tags', width: 20 },
|
||||
]
|
||||
);
|
||||
} else {
|
||||
response = await client.listScenarios({
|
||||
limit: flags.limit,
|
||||
offset: flags.offset,
|
||||
tags: flags.tags,
|
||||
});
|
||||
|
||||
spinner.stop();
|
||||
|
||||
if (flags.json) {
|
||||
output.json(response);
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.data.length === 0) {
|
||||
output.info('No public scenarios found');
|
||||
return;
|
||||
}
|
||||
|
||||
output.table(
|
||||
response.data.map((s) => ({
|
||||
name: s.name || s.prompt.slice(0, 30) + '...',
|
||||
collection: s.collection?.name || '-',
|
||||
quality: (s.qualityScore * 100).toFixed(0) + '%',
|
||||
runs: s.totalRuns,
|
||||
status: s.lastRunStatus || '-',
|
||||
})),
|
||||
[
|
||||
{ key: 'name', header: 'Name', width: 35 },
|
||||
{ key: 'collection', header: 'Collection', width: 25 },
|
||||
{ key: 'quality', header: 'Quality', width: 10 },
|
||||
{ key: 'runs', header: 'Runs', width: 8 },
|
||||
{ key: 'status', header: 'Status', width: 8 },
|
||||
]
|
||||
);
|
||||
|
||||
output.newLine();
|
||||
output.text(
|
||||
output.dim(
|
||||
`Showing ${response.data.length} scenario(s)` +
|
||||
(response.pagination.hasMore ? ` (more available)` : '')
|
||||
)
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
spinner.fail('Failed to fetch scenarios');
|
||||
output.error(
|
||||
error instanceof Error ? error.message : 'Unknown error',
|
||||
flags.verbose ? String(error) : undefined
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
208
packages/cli/src/commands/scenario/run.ts
Normal file
208
packages/cli/src/commands/scenario/run.ts
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
import { Args, Command, Flags } from '@oclif/core';
|
||||
import { getClient } from '../../lib/api-client.js';
|
||||
import { createOutput } from '../../lib/output.js';
|
||||
|
||||
export default class ScenarioRun extends Command {
|
||||
static description = 'Run all scenarios for a collection';
|
||||
|
||||
static examples = [
|
||||
'<%= config.bin %> <%= command.id %> my-collection',
|
||||
'<%= config.bin %> <%= command.id %> my-collection --json',
|
||||
'<%= config.bin %> <%= command.id %> my-collection --verbose',
|
||||
];
|
||||
|
||||
static args = {
|
||||
collection: Args.string({
|
||||
description: 'Collection ID or slug',
|
||||
required: true,
|
||||
}),
|
||||
};
|
||||
|
||||
static flags = {
|
||||
json: Flags.boolean({
|
||||
description: 'Output in JSON format',
|
||||
default: false,
|
||||
}),
|
||||
verbose: Flags.boolean({
|
||||
char: 'v',
|
||||
description: 'Show verbose output',
|
||||
default: false,
|
||||
}),
|
||||
limit: Flags.integer({
|
||||
char: 'l',
|
||||
description: 'Maximum number of scenarios to run',
|
||||
default: 50,
|
||||
}),
|
||||
};
|
||||
|
||||
async run(): Promise<void> {
|
||||
const { args, flags } = await this.parse(ScenarioRun);
|
||||
const output = createOutput(flags);
|
||||
const client = getClient();
|
||||
|
||||
if (!client.isAuthenticated()) {
|
||||
output.error('Not authenticated. Run `tpm auth login` first.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Find collection
|
||||
const collectionsSpinner = output.spinner('Finding collection...');
|
||||
let collectionId: string;
|
||||
|
||||
try {
|
||||
const collections = await client.listCollections({ limit: 100 });
|
||||
const collection = collections.data.find(
|
||||
(c) => c.id === args.collection || c.slug === args.collection
|
||||
);
|
||||
|
||||
if (!collection) {
|
||||
collectionsSpinner.fail('Collection not found');
|
||||
output.error(`No collection found with ID or slug: ${args.collection}`);
|
||||
return;
|
||||
}
|
||||
|
||||
collectionId = collection.id;
|
||||
collectionsSpinner.stop();
|
||||
output.text(output.bold(`Running scenarios for: ${collection.name}\n`));
|
||||
} catch (error) {
|
||||
collectionsSpinner.fail('Failed to find collection');
|
||||
output.error(error instanceof Error ? error.message : 'Unknown error');
|
||||
return;
|
||||
}
|
||||
|
||||
// Fetch scenarios
|
||||
const scenariosSpinner = output.spinner('Fetching scenarios...');
|
||||
let scenarios: Array<{
|
||||
id: string;
|
||||
name: string | null;
|
||||
prompt: string;
|
||||
}>;
|
||||
|
||||
try {
|
||||
const response = await client.listCollectionScenarios(collectionId, { limit: flags.limit });
|
||||
scenarios =
|
||||
(
|
||||
response as unknown as {
|
||||
data: {
|
||||
scenarios: Array<{
|
||||
id: string;
|
||||
name: string | null;
|
||||
prompt: string;
|
||||
}>;
|
||||
};
|
||||
}
|
||||
).data?.scenarios || [];
|
||||
|
||||
if (scenarios.length === 0) {
|
||||
scenariosSpinner.fail('No scenarios found');
|
||||
output.info('This collection has no scenarios. Generate some with:');
|
||||
output.text(` tpm scenario generate ${args.collection}`);
|
||||
return;
|
||||
}
|
||||
|
||||
scenariosSpinner.stop();
|
||||
output.info(`Found ${scenarios.length} scenario(s) to run\n`);
|
||||
} catch (error) {
|
||||
scenariosSpinner.fail('Failed to fetch scenarios');
|
||||
output.error(error instanceof Error ? error.message : 'Unknown error');
|
||||
return;
|
||||
}
|
||||
|
||||
// Run each scenario
|
||||
const results: Array<{
|
||||
name: string;
|
||||
status: string;
|
||||
verdict: string | null;
|
||||
reason: string | null;
|
||||
timeMs: number | null;
|
||||
}> = [];
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
let errors = 0;
|
||||
|
||||
for (const scenario of scenarios) {
|
||||
const name = scenario.name || scenario.prompt.slice(0, 40) + '...';
|
||||
const runSpinner = output.spinner(`Running: ${name}`);
|
||||
|
||||
try {
|
||||
const result = await client.runScenario(scenario.id);
|
||||
const runData = (
|
||||
result as unknown as {
|
||||
data: {
|
||||
status: string;
|
||||
success: boolean;
|
||||
evaluator: { verdict: string | null; reason: string | null };
|
||||
usage: { executionTimeMs: number | null };
|
||||
};
|
||||
}
|
||||
).data;
|
||||
|
||||
if (runData.success) {
|
||||
passed++;
|
||||
runSpinner.succeed(`${output.green('✓')} ${name}`);
|
||||
} else if (runData.status === 'error') {
|
||||
errors++;
|
||||
runSpinner.fail(`${output.red('✗')} ${name} (error)`);
|
||||
} else {
|
||||
failed++;
|
||||
runSpinner.fail(`${output.red('✗')} ${name}`);
|
||||
}
|
||||
|
||||
if (flags.verbose && runData.evaluator?.reason) {
|
||||
output.text(output.dim(` → ${runData.evaluator.reason}`));
|
||||
}
|
||||
|
||||
results.push({
|
||||
name,
|
||||
status: runData.status,
|
||||
verdict: runData.evaluator?.verdict ?? null,
|
||||
reason: runData.evaluator?.reason ?? null,
|
||||
timeMs: runData.usage?.executionTimeMs ?? null,
|
||||
});
|
||||
} catch (error) {
|
||||
errors++;
|
||||
runSpinner.fail(`${output.red('✗')} ${name} (error)`);
|
||||
results.push({
|
||||
name,
|
||||
status: 'error',
|
||||
verdict: null,
|
||||
reason: error instanceof Error ? error.message : 'Unknown error',
|
||||
timeMs: null,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Output summary
|
||||
output.newLine();
|
||||
output.text(output.bold('─'.repeat(50)));
|
||||
output.text(output.bold('Summary'));
|
||||
output.text(` ${output.green('Passed:')} ${passed}`);
|
||||
output.text(` ${output.red('Failed:')} ${failed}`);
|
||||
if (errors > 0) {
|
||||
output.text(` ${output.yellow('Errors:')} ${errors}`);
|
||||
}
|
||||
output.text(` ${output.dim('Total:')} ${scenarios.length}`);
|
||||
|
||||
const passRate = scenarios.length > 0 ? (passed / scenarios.length) * 100 : 0;
|
||||
output.newLine();
|
||||
output.text(`Pass rate: ${passRate.toFixed(1)}%`);
|
||||
|
||||
if (flags.json) {
|
||||
output.newLine();
|
||||
output.json({
|
||||
collection: args.collection,
|
||||
total: scenarios.length,
|
||||
passed,
|
||||
failed,
|
||||
errors,
|
||||
passRate: passRate.toFixed(1),
|
||||
results,
|
||||
});
|
||||
}
|
||||
|
||||
// Exit with error code if any failures
|
||||
if (failed > 0 || errors > 0) {
|
||||
this.exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
168
packages/cli/src/commands/scenario/test.ts
Normal file
168
packages/cli/src/commands/scenario/test.ts
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
import { Args, Command, Flags } from '@oclif/core';
|
||||
import { getClient } from '../../lib/api-client.js';
|
||||
import { createOutput } from '../../lib/output.js';
|
||||
|
||||
export default class ScenarioTest extends Command {
|
||||
static description = 'Run a single scenario by ID';
|
||||
|
||||
static examples = [
|
||||
'<%= config.bin %> <%= command.id %> clu123abc456',
|
||||
'<%= config.bin %> <%= command.id %> clu123abc456 --json',
|
||||
'<%= config.bin %> <%= command.id %> clu123abc456 --verbose',
|
||||
];
|
||||
|
||||
static args = {
|
||||
scenarioId: Args.string({
|
||||
description: 'Scenario ID to run',
|
||||
required: true,
|
||||
}),
|
||||
};
|
||||
|
||||
static flags = {
|
||||
json: Flags.boolean({
|
||||
description: 'Output in JSON format',
|
||||
default: false,
|
||||
}),
|
||||
verbose: Flags.boolean({
|
||||
char: 'v',
|
||||
description: 'Show verbose output including full reason',
|
||||
default: false,
|
||||
}),
|
||||
};
|
||||
|
||||
async run(): Promise<void> {
|
||||
const { args, flags } = await this.parse(ScenarioTest);
|
||||
const output = createOutput(flags);
|
||||
const client = getClient();
|
||||
|
||||
if (!client.isAuthenticated()) {
|
||||
output.error('Not authenticated. Run `tpm auth login` first.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Fetch scenario info first
|
||||
const infoSpinner = output.spinner('Fetching scenario...');
|
||||
let scenarioName: string;
|
||||
|
||||
try {
|
||||
const scenarioResponse = await client.getScenario(args.scenarioId);
|
||||
const scenario = (
|
||||
scenarioResponse as unknown as {
|
||||
data: {
|
||||
name: string | null;
|
||||
prompt: string;
|
||||
collection?: { name: string };
|
||||
};
|
||||
}
|
||||
).data;
|
||||
|
||||
scenarioName = scenario.name || scenario.prompt.slice(0, 50) + '...';
|
||||
infoSpinner.stop();
|
||||
|
||||
output.text(output.bold(`Scenario: ${scenarioName}`));
|
||||
if (scenario.collection) {
|
||||
output.text(output.dim(`Collection: ${scenario.collection.name}`));
|
||||
}
|
||||
output.newLine();
|
||||
} catch (error) {
|
||||
infoSpinner.fail('Scenario not found');
|
||||
output.error(
|
||||
error instanceof Error ? error.message : 'Unknown error',
|
||||
flags.verbose ? String(error) : undefined
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Run the scenario
|
||||
const runSpinner = output.spinner('Executing scenario...');
|
||||
|
||||
try {
|
||||
const result = await client.runScenario(args.scenarioId);
|
||||
const runData = (
|
||||
result as unknown as {
|
||||
data: {
|
||||
runId: string;
|
||||
status: string;
|
||||
success: boolean;
|
||||
evaluator: {
|
||||
model: string | null;
|
||||
verdict: string | null;
|
||||
reason: string | null;
|
||||
};
|
||||
usage: {
|
||||
inputTokens: number | null;
|
||||
outputTokens: number | null;
|
||||
totalTokens: number | null;
|
||||
executionTimeMs: number | null;
|
||||
};
|
||||
timestamps: {
|
||||
startedAt: string | null;
|
||||
completedAt: string | null;
|
||||
createdAt: string;
|
||||
};
|
||||
quotaRemaining: number;
|
||||
};
|
||||
}
|
||||
).data;
|
||||
|
||||
if (runData.success) {
|
||||
runSpinner.succeed(output.green('Scenario PASSED'));
|
||||
} else if (runData.status === 'error') {
|
||||
runSpinner.fail(output.red('Scenario ERROR'));
|
||||
} else {
|
||||
runSpinner.fail(output.red('Scenario FAILED'));
|
||||
}
|
||||
|
||||
output.newLine();
|
||||
|
||||
if (flags.json) {
|
||||
output.json(runData);
|
||||
return;
|
||||
}
|
||||
|
||||
// Display results
|
||||
output.text(output.bold('Results'));
|
||||
output.text(` Status: ${runData.status}`);
|
||||
output.text(` Verdict: ${runData.evaluator?.verdict || 'N/A'}`);
|
||||
|
||||
if (runData.evaluator?.reason) {
|
||||
if (flags.verbose) {
|
||||
output.text(` Reason: ${runData.evaluator.reason}`);
|
||||
} else {
|
||||
const truncatedReason =
|
||||
runData.evaluator.reason.length > 80
|
||||
? runData.evaluator.reason.slice(0, 80) + '...'
|
||||
: runData.evaluator.reason;
|
||||
output.text(` Reason: ${truncatedReason}`);
|
||||
}
|
||||
}
|
||||
|
||||
output.newLine();
|
||||
output.text(output.bold('Usage'));
|
||||
if (runData.usage.executionTimeMs) {
|
||||
output.text(` Duration: ${runData.usage.executionTimeMs}ms`);
|
||||
}
|
||||
if (runData.usage.totalTokens) {
|
||||
output.text(
|
||||
` Tokens: ${runData.usage.totalTokens} (in: ${runData.usage.inputTokens}, out: ${runData.usage.outputTokens})`
|
||||
);
|
||||
}
|
||||
|
||||
output.newLine();
|
||||
output.text(output.dim(`Run ID: ${runData.runId}`));
|
||||
output.text(output.dim(`Quota remaining: ${runData.quotaRemaining} runs/day`));
|
||||
|
||||
// Exit with error code if failed
|
||||
if (!runData.success) {
|
||||
this.exit(1);
|
||||
}
|
||||
} catch (error) {
|
||||
runSpinner.fail('Failed to run scenario');
|
||||
output.error(
|
||||
error instanceof Error ? error.message : 'Unknown error',
|
||||
flags.verbose ? String(error) : undefined
|
||||
);
|
||||
this.exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -149,6 +149,73 @@ export interface ApiKey {
|
|||
createdAt: string;
|
||||
}
|
||||
|
||||
// Scenario types
|
||||
export interface Scenario {
|
||||
id: string;
|
||||
collectionId: string | null;
|
||||
prompt: string;
|
||||
name: string | null;
|
||||
description: string | null;
|
||||
tags: string[];
|
||||
qualityScore: number;
|
||||
totalRuns: number;
|
||||
lastRunAt: string | null;
|
||||
lastRunStatus: string | null;
|
||||
consecutivePasses: number;
|
||||
consecutiveFails: number;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
collection?: {
|
||||
id: string;
|
||||
name: string;
|
||||
slug: string | null;
|
||||
username: string | null;
|
||||
} | null;
|
||||
}
|
||||
|
||||
export interface ScenarioRun {
|
||||
id: string;
|
||||
status: string;
|
||||
success: boolean;
|
||||
evaluator: {
|
||||
model: string | null;
|
||||
verdict: string | null;
|
||||
reason: string | null;
|
||||
};
|
||||
assertions: unknown;
|
||||
usage: {
|
||||
inputTokens: number | null;
|
||||
outputTokens: number | null;
|
||||
totalTokens: number | null;
|
||||
executionTimeMs: number | null;
|
||||
};
|
||||
timestamps: {
|
||||
startedAt: string | null;
|
||||
completedAt: string | null;
|
||||
createdAt: string;
|
||||
};
|
||||
quotaRemaining?: number;
|
||||
}
|
||||
|
||||
export interface ScenarioListOptions extends PaginationOptions {
|
||||
collectionId?: string;
|
||||
tags?: string;
|
||||
sortBy?: 'qualityScore' | 'totalRuns' | 'createdAt' | 'lastRunAt';
|
||||
}
|
||||
|
||||
export interface CreateScenarioInput {
|
||||
collectionId: string;
|
||||
prompt: string;
|
||||
name?: string;
|
||||
description?: string;
|
||||
tags?: string[];
|
||||
}
|
||||
|
||||
export interface GenerateScenariosInput {
|
||||
count?: number;
|
||||
skipSimilarityCheck?: boolean;
|
||||
}
|
||||
|
||||
// Stats types
|
||||
export interface Stats {
|
||||
tools: {
|
||||
|
|
@ -175,10 +242,7 @@ export class TpmClient {
|
|||
this.timeout = options.timeout ?? 30000;
|
||||
}
|
||||
|
||||
private async request<T>(
|
||||
endpoint: string,
|
||||
options: RequestInit = {}
|
||||
): Promise<T> {
|
||||
private async request<T>(endpoint: string, options: RequestInit = {}): Promise<T> {
|
||||
const url = `${this.baseUrl}${endpoint}`;
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
|
|
@ -199,7 +263,7 @@ export class TpmClient {
|
|||
signal: controller.signal,
|
||||
});
|
||||
|
||||
const data = await response.json() as T & { message?: string; error?: string };
|
||||
const data = (await response.json()) as T & { message?: string; error?: string };
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ApiError(
|
||||
|
|
@ -240,14 +304,16 @@ export class TpmClient {
|
|||
}
|
||||
|
||||
async getTool(packageName: string, toolName: string): Promise<ApiResponse<Tool>> {
|
||||
return this.request(`/tools/${encodeURIComponent(packageName)}/${encodeURIComponent(toolName)}`);
|
||||
return this.request(
|
||||
`/tools/${encodeURIComponent(packageName)}/${encodeURIComponent(toolName)}`
|
||||
);
|
||||
}
|
||||
|
||||
async getToolBySlug(slug: string): Promise<ApiResponse<Tool>> {
|
||||
// Search for the tool by slug
|
||||
const searchResult = await this.searchTools({ query: slug, limit: 1 });
|
||||
if (searchResult.data && searchResult.data.length > 0) {
|
||||
const tool = searchResult.data.find(t => t.slug === slug) || searchResult.data[0];
|
||||
const tool = searchResult.data.find((t) => t.slug === slug) || searchResult.data[0];
|
||||
return { success: true, data: tool };
|
||||
}
|
||||
return { success: false, error: 'Tool not found' };
|
||||
|
|
@ -264,7 +330,9 @@ export class TpmClient {
|
|||
return this.request<PaginatedResponse<Tool>>(endpoint);
|
||||
}
|
||||
|
||||
async validateTpmjsField(field: unknown): Promise<ApiResponse<{ valid: boolean; tier: string | null; errors?: unknown[] }>> {
|
||||
async validateTpmjsField(
|
||||
field: unknown
|
||||
): Promise<ApiResponse<{ valid: boolean; tier: string | null; errors?: unknown[] }>> {
|
||||
return this.request('/tools/validate', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(field),
|
||||
|
|
@ -285,7 +353,7 @@ export class TpmClient {
|
|||
const url = `${this.baseUrl}/tools/${encodeURIComponent(slug)}/execute`;
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'text/event-stream',
|
||||
Accept: 'text/event-stream',
|
||||
};
|
||||
|
||||
if (this.apiKey) {
|
||||
|
|
@ -403,7 +471,10 @@ export class TpmClient {
|
|||
});
|
||||
}
|
||||
|
||||
async updateCollection(id: string, input: UpdateCollectionInput): Promise<ApiResponse<Collection>> {
|
||||
async updateCollection(
|
||||
id: string,
|
||||
input: UpdateCollectionInput
|
||||
): Promise<ApiResponse<Collection>> {
|
||||
return this.request(`/collections/${id}`, {
|
||||
method: 'PATCH',
|
||||
body: JSON.stringify(input),
|
||||
|
|
@ -441,6 +512,80 @@ export class TpmClient {
|
|||
return this.request('/user/tpmjs-api-keys');
|
||||
}
|
||||
|
||||
// Scenarios
|
||||
async listScenarios(options: ScenarioListOptions = {}): Promise<PaginatedResponse<Scenario>> {
|
||||
const params = new URLSearchParams();
|
||||
if (options.limit) params.set('limit', String(options.limit));
|
||||
if (options.offset) params.set('offset', String(options.offset));
|
||||
if (options.collectionId) params.set('collectionId', options.collectionId);
|
||||
if (options.tags) params.set('tags', options.tags);
|
||||
if (options.sortBy) params.set('sortBy', options.sortBy);
|
||||
|
||||
const queryString = params.toString();
|
||||
const endpoint = queryString ? `/scenarios?${queryString}` : '/scenarios';
|
||||
|
||||
return this.request<PaginatedResponse<Scenario>>(endpoint);
|
||||
}
|
||||
|
||||
async listCollectionScenarios(
|
||||
collectionId: string,
|
||||
options: PaginationOptions = {}
|
||||
): Promise<ApiResponse<{ scenarios: Scenario[] }>> {
|
||||
const params = new URLSearchParams();
|
||||
if (options.limit) params.set('limit', String(options.limit));
|
||||
if (options.offset) params.set('offset', String(options.offset));
|
||||
|
||||
const queryString = params.toString();
|
||||
const endpoint = queryString
|
||||
? `/collections/${collectionId}/scenarios?${queryString}`
|
||||
: `/collections/${collectionId}/scenarios`;
|
||||
|
||||
return this.request<ApiResponse<{ scenarios: Scenario[] }>>(endpoint);
|
||||
}
|
||||
|
||||
async getScenario(id: string): Promise<ApiResponse<Scenario>> {
|
||||
return this.request(`/scenarios/${id}`);
|
||||
}
|
||||
|
||||
async createScenario(input: CreateScenarioInput): Promise<ApiResponse<Scenario>> {
|
||||
return this.request('/scenarios', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(input),
|
||||
});
|
||||
}
|
||||
|
||||
async generateScenarios(
|
||||
collectionId: string,
|
||||
input: GenerateScenariosInput = {}
|
||||
): Promise<ApiResponse<{ scenarios: { scenario: Scenario; similarity?: unknown }[] }>> {
|
||||
return this.request(`/collections/${collectionId}/scenarios/generate`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(input),
|
||||
});
|
||||
}
|
||||
|
||||
async runScenario(scenarioId: string): Promise<ApiResponse<ScenarioRun>> {
|
||||
return this.request(`/scenarios/${scenarioId}/run`, {
|
||||
method: 'POST',
|
||||
});
|
||||
}
|
||||
|
||||
async getScenarioRuns(
|
||||
scenarioId: string,
|
||||
options: PaginationOptions = {}
|
||||
): Promise<PaginatedResponse<ScenarioRun>> {
|
||||
const params = new URLSearchParams();
|
||||
if (options.limit) params.set('limit', String(options.limit));
|
||||
if (options.offset) params.set('offset', String(options.offset));
|
||||
|
||||
const queryString = params.toString();
|
||||
const endpoint = queryString
|
||||
? `/scenarios/${scenarioId}/runs?${queryString}`
|
||||
: `/scenarios/${scenarioId}/runs`;
|
||||
|
||||
return this.request<PaginatedResponse<ScenarioRun>>(endpoint);
|
||||
}
|
||||
|
||||
// Check if authenticated
|
||||
isAuthenticated(): boolean {
|
||||
return !!this.apiKey;
|
||||
|
|
|
|||
|
|
@ -173,6 +173,27 @@ export class OutputFormatter {
|
|||
// OSC 8 hyperlink support for modern terminals
|
||||
return `\x1b]8;;${url}\x07${pc.underline(pc.blue(text))}\x1b]8;;\x07`;
|
||||
}
|
||||
|
||||
// Color helpers
|
||||
green(text: string): string {
|
||||
return pc.green(text);
|
||||
}
|
||||
|
||||
red(text: string): string {
|
||||
return pc.red(text);
|
||||
}
|
||||
|
||||
yellow(text: string): string {
|
||||
return pc.yellow(text);
|
||||
}
|
||||
|
||||
blue(text: string): string {
|
||||
return pc.blue(text);
|
||||
}
|
||||
|
||||
cyan(text: string): string {
|
||||
return pc.cyan(text);
|
||||
}
|
||||
}
|
||||
|
||||
// Convenience function to create formatter from command flags
|
||||
|
|
|
|||
|
|
@ -467,6 +467,7 @@ model Collection {
|
|||
agents AgentCollection[]
|
||||
likes CollectionLike[]
|
||||
bridgeTools CollectionBridgeTool[]
|
||||
scenarios Scenario[]
|
||||
|
||||
// Unique constraint: user can't have duplicate collection slugs
|
||||
@@unique([userId, slug])
|
||||
|
|
@ -1118,3 +1119,136 @@ model ApiUsageSummary {
|
|||
@@index([periodType, periodStart])
|
||||
@@map("api_usage_summaries")
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Scenario Models (Integration Testing for Collections)
|
||||
// ============================================================================
|
||||
|
||||
/// Scenario - AI-generated test scenarios for collections
|
||||
model Scenario {
|
||||
id String @id @default(cuid())
|
||||
|
||||
// Collection relationship (nullable for orphaned scenarios)
|
||||
collectionId String? @map("collection_id")
|
||||
collection Collection? @relation(fields: [collectionId], references: [id], onDelete: SetNull)
|
||||
|
||||
// Content
|
||||
prompt String @db.Text // AI-generated free-form prompt
|
||||
name String? @db.VarChar(200) // Optional human-readable name
|
||||
description String? @db.Text
|
||||
|
||||
// Validation (optional assertions)
|
||||
assertions Json? @db.JsonB // { regex?: string[], schema?: object }
|
||||
|
||||
// AI-generated metadata
|
||||
tags String[] @default([]) @db.Text
|
||||
|
||||
// Quality metrics (streak-based scoring)
|
||||
qualityScore Float @default(0) @map("quality_score")
|
||||
consecutivePasses Int @default(0) @map("consecutive_passes")
|
||||
consecutiveFails Int @default(0) @map("consecutive_fails")
|
||||
totalRuns Int @default(0) @map("total_runs")
|
||||
lastRunAt DateTime? @map("last_run_at")
|
||||
lastRunStatus String? @map("last_run_status") @db.VarChar(20) // 'pass' | 'fail' | 'error'
|
||||
|
||||
// Timestamps
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
|
||||
// Relations
|
||||
runs ScenarioRun[]
|
||||
embedding ScenarioEmbedding?
|
||||
|
||||
@@index([collectionId])
|
||||
@@index([qualityScore])
|
||||
@@index([createdAt])
|
||||
@@index([lastRunStatus])
|
||||
@@map("scenarios")
|
||||
}
|
||||
|
||||
/// ScenarioEmbedding - vector embeddings for scenario similarity detection
|
||||
model ScenarioEmbedding {
|
||||
id String @id @default(cuid())
|
||||
|
||||
// Scenario relationship
|
||||
scenarioId String @unique @map("scenario_id")
|
||||
scenario Scenario @relation(fields: [scenarioId], references: [id], onDelete: Cascade)
|
||||
|
||||
// Embedding data
|
||||
embedding Json @db.JsonB // Array of floats (1536 dims for text-embedding-3-small)
|
||||
model String @default("text-embedding-3-small") @db.VarChar(50)
|
||||
|
||||
// Timestamps
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
|
||||
@@index([scenarioId])
|
||||
@@map("scenario_embeddings")
|
||||
}
|
||||
|
||||
/// ScenarioRun - individual execution records for scenarios
|
||||
model ScenarioRun {
|
||||
id String @id @default(cuid())
|
||||
|
||||
// Scenario relationship
|
||||
scenarioId String @map("scenario_id")
|
||||
scenario Scenario @relation(fields: [scenarioId], references: [id], onDelete: Cascade)
|
||||
|
||||
// Execution context
|
||||
userId String @map("user_id") // Who triggered the run
|
||||
agentId String? @map("agent_id") // Ephemeral agent ID (for debugging)
|
||||
|
||||
// Status
|
||||
status String @db.VarChar(20) // 'pending' | 'running' | 'pass' | 'fail' | 'error'
|
||||
retryCount Int @default(0) @map("retry_count")
|
||||
|
||||
// Results
|
||||
conversation Json? @db.JsonB // Full message history
|
||||
output String? @db.Text // Final output from agent
|
||||
errorLog String? @map("error_log") @db.Text // Full error logs (private to owner)
|
||||
|
||||
// LLM Evaluation
|
||||
evaluatorModel String? @map("evaluator_model") @db.VarChar(50) // e.g., "claude-3.5-sonnet"
|
||||
evaluatorVerdict String? @map("evaluator_verdict") @db.VarChar(10) // 'pass' | 'fail'
|
||||
evaluatorReason String? @map("evaluator_reason") @db.Text // Explanation
|
||||
|
||||
// Assertions
|
||||
assertionResults Json? @map("assertion_results") @db.JsonB // { passed: string[], failed: string[] }
|
||||
|
||||
// Cost tracking
|
||||
inputTokens Int? @map("input_tokens")
|
||||
outputTokens Int? @map("output_tokens")
|
||||
totalTokens Int? @map("total_tokens")
|
||||
executionTimeMs Int? @map("execution_time_ms")
|
||||
estimatedCost Decimal? @map("estimated_cost") @db.Decimal(10, 6)
|
||||
|
||||
// Timestamps
|
||||
startedAt DateTime? @map("started_at")
|
||||
completedAt DateTime? @map("completed_at")
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
|
||||
@@index([scenarioId])
|
||||
@@index([userId])
|
||||
@@index([status])
|
||||
@@index([createdAt])
|
||||
@@map("scenario_runs")
|
||||
}
|
||||
|
||||
/// ScenarioQuota - daily usage quotas for scenario runs
|
||||
model ScenarioQuota {
|
||||
id String @id @default(cuid())
|
||||
|
||||
// User relationship
|
||||
userId String @unique @map("user_id")
|
||||
|
||||
// Quota configuration
|
||||
dailyLimit Int @default(50) @map("daily_limit") // Runs per day
|
||||
dailyUsed Int @default(0) @map("daily_used")
|
||||
lastResetAt DateTime @default(now()) @map("last_reset_at")
|
||||
|
||||
// Timestamps
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
|
||||
@@index([userId])
|
||||
@@map("scenario_quotas")
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue