From c52e6eaf24efb4b3263b4b5c52dcc3d7171f6044 Mon Sep 17 00:00:00 2001 From: Ajax Davis Date: Sun, 18 Jan 2026 09:53:36 +1000 Subject: [PATCH] docs: add scenarios user guide and API reference - Add /docs/scenarios page with comprehensive user guide - Overview of what scenarios are - Common archetypes for different tool types - CLI commands: generate, list, run, test, info - Quality scoring explanation - CI/CD integration examples - Rate limits documentation - Add /docs/api/scenarios page with API reference - List scenarios endpoint - Get scenario details - List collection scenarios - Create scenario - Generate scenarios with AI - Run scenario - Get run history - Check prompt similarity - Featured scenarios - Error responses --- apps/web/src/app/docs/api/scenarios/page.tsx | 679 +++++++++++++++++++ apps/web/src/app/docs/scenarios/page.tsx | 614 +++++++++++++++++ 2 files changed, 1293 insertions(+) create mode 100644 apps/web/src/app/docs/api/scenarios/page.tsx create mode 100644 apps/web/src/app/docs/scenarios/page.tsx diff --git a/apps/web/src/app/docs/api/scenarios/page.tsx b/apps/web/src/app/docs/api/scenarios/page.tsx new file mode 100644 index 0000000..e98f0bc --- /dev/null +++ b/apps/web/src/app/docs/api/scenarios/page.tsx @@ -0,0 +1,679 @@ +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@tpmjs/ui/Card/Card'; +import { CodeBlock } from '@tpmjs/ui/CodeBlock/CodeBlock'; +import type { Metadata } from 'next'; + +export const metadata: Metadata = { + title: 'Scenarios API | TPMJS Docs', + description: 'API reference for TPMJS scenarios endpoints', +}; + +export default function ScenariosApiPage(): React.ReactElement { + return ( +
+
+

Scenarios API

+

+ The Scenarios API allows you to create, run, and manage test scenarios for your tool + collections. +

+
+ + {/* List Scenarios */} + + + List Scenarios + GET /api/scenarios + + +

+ Retrieve a paginated list of public scenarios with optional filtering. +

+ +

Query Parameters

+
    +
  • + limit - Number of results (default: 20, max: 50) +
    • +
  • + offset - Pagination offset +
    • +
  • + collectionId - Filter by collection ID +
    • +
  • + tags - Filter by tags (comma-separated) +
    • +
  • + sortBy - Sort field: qualityScore, totalRuns,{' '} + createdAt, lastRunAt +
    • +
+ +

Example Request

+ + +

Example Response

+ + + + + {/* Get Scenario */} + + + Get Scenario + GET /api/scenarios/:id + + +

+ Retrieve detailed information about a specific scenario. +

+ +

Path Parameters

+
    +
  • + id - Scenario ID +
    • +
+ +

Example Request

+ + +

Example Response

+ + + + + {/* List Collection Scenarios */} + + + List Collection Scenarios + GET /api/collections/:id/scenarios + + +

+ Retrieve all scenarios for a specific collection. +

+ +

Path Parameters

+
    +
  • + id - Collection ID +
    • +
+ +

Query Parameters

+
    +
  • + limit - Number of results (default: 20, max: 50) +
    • +
  • + offset - Pagination offset +
    • +
+ +

Example Request

+ + +

Example Response

+ + + + + {/* Create Scenario */} + + + Create Scenario + POST /api/scenarios + + +

+ Create a new scenario manually. Requires authentication and collection ownership. +

+ +

Request Body

+ + +

TypeScript Example

+ + + + + {/* Generate Scenarios */} + + + Generate Scenarios + POST /api/collections/:id/scenarios/generate + + +

+ AI-generate scenarios based on the collection's tools. Requires authentication and + collection ownership. +

+ +

Path Parameters

+
    +
  • + id - Collection ID +
    • +
+ +

Request Body

+ + +

Example Request

+ + +

Example Response

+ + +
+

Similarity Warning

+

+ When similarity.maxSimilarity exceeds 0.7, the scenario is similar to an + existing one. The mostSimilar field shows which scenario it overlaps + with. +

+
+ + + + {/* Run Scenario */} + + + Run Scenario + POST /api/scenarios/:id/run + + +

+ Execute a scenario and return the results. Requires authentication. Consumes one run + from your daily quota. +

+ +

Path Parameters

+
    +
  • + id - Scenario ID +
    • +
+ +

Example Request

+ + +

Example Response

+ + +

Failure Response

+ + + + + {/* Get Scenario Runs */} + + + Get Scenario Run History + GET /api/scenarios/:id/runs + + +

+ Retrieve the run history for a scenario. Requires authentication if the scenario belongs + to a private collection. +

+ +

Path Parameters

+
    +
  • + id - Scenario ID +
    • +
+ +

Query Parameters

+
    +
  • + limit - Number of results (default: 10, max: 50) +
    • +
  • + offset - Pagination offset +
    • +
+ +

Example Request

+ + +

Example Response

+ + + + + {/* Check Similarity */} + + + Check Prompt Similarity + POST /api/scenarios/check-similarity + + +

+ Check if a prompt is similar to existing scenarios in a collection. Useful before + creating manual scenarios to avoid duplicates. +

+ +

Request Body

+ + +

Example Response

+ + +
+

Similarity Threshold

+

+ A maxSimilarity above 0.7 indicates significant overlap with existing + scenarios. Consider modifying your prompt to test different aspects of your tools. +

+
+ + + + {/* Featured Scenarios */} + + + Get Featured Scenarios + GET /api/scenarios/featured + + +

+ Retrieve featured scenarios for the homepage showcase. Returns a mix of high-quality, + diverse, and recently successful scenarios. +

+ +

Query Parameters

+
    +
  • + limit - Number of results (default: 10, max: 20) +
    • +
+ +

Example Request

+ + +

Example Response

+ + + + + {/* Error Responses */} + + + Error Responses + Common error codes and responses + + +

401 Unauthorized

+ + +

403 Forbidden

+ + +

404 Not Found

+ + +

429 Rate Limited

+ + + +
+ ); +} diff --git a/apps/web/src/app/docs/scenarios/page.tsx b/apps/web/src/app/docs/scenarios/page.tsx new file mode 100644 index 0000000..482eab7 --- /dev/null +++ b/apps/web/src/app/docs/scenarios/page.tsx @@ -0,0 +1,614 @@ +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@tpmjs/ui/Card/Card'; +import { CodeBlock } from '@tpmjs/ui/CodeBlock/CodeBlock'; +import type { Metadata } from 'next'; +import Link from 'next/link'; + +export const metadata: Metadata = { + title: 'Scenarios Guide | TPMJS Docs', + description: 'Test your tool collections with AI-generated scenarios and automated evaluation', +}; + +export default function ScenariosPage(): React.ReactElement { + return ( +
+
+

Scenarios

+

+ Scenarios are AI-generated test cases for your tool collections. They automatically verify + that your tools work correctly and provide quality metrics over time. +

+
+ + {/* Overview */} + + + What are Scenarios? + Automated testing for tool collections + + +

+ A Scenario is a test case that exercises your tool collection with a realistic user + prompt. When you run a scenario: +

+
    +
  1. An ephemeral agent is created with your collection's tools
    2. +
  2. The agent executes the scenario's prompt
    3. +
  3. An LLM evaluates whether the task was completed successfully
    4. +
  4. Results are recorded and quality scores are updated
    5. +
+

+ This enables continuous testing of your tool collections, similar to unit tests for + code. +

+ + + + {/* Use Cases */} + + + Common Archetypes + Typical scenarios for different tool types + + +
+

Web Scraping Tools

+

+ Test data extraction from various webpage structures: +

+ +
+ +
+

Search Tools

+

+ Verify search accuracy and relevance: +

+ +
+ +
+

Data Processing Tools

+

+ Test transformations and calculations: +

+ +
+ +
+

File/Document Tools

+

+ Verify file operations and content extraction: +

+ +
+ + + + {/* CLI Installation */} + + + Getting Started + Install the CLI and authenticate + + +

Install the TPMJS CLI

+ + +

Authenticate

+

+ You need a TPMJS API key to run scenarios. Get one from your{' '} + + dashboard settings + + . +

+ + + + + {/* Generate Scenarios */} + + + Generate Scenarios + tpm scenario generate + + +

+ AI-generate test scenarios based on your collection's tools. The generator analyzes + your tools and creates realistic prompts. +

+ +

Basic Usage

+ + +

Example Output

+ + +
+

Similarity Detection

+

+ Generated scenarios are checked against existing ones using vector similarity. If a + scenario is >70% similar to an existing one, you'll see a warning. This helps + maintain diverse test coverage. +

+
+ + + + {/* List Scenarios */} + + + List Scenarios + tpm scenario list + + +

+ View all scenarios for a collection or browse public scenarios. +

+ +

Usage

+ + +

Example Output

+ + + + + {/* Run Scenarios */} + + + Run All Scenarios + tpm scenario run + + +

+ Execute all scenarios for a collection. This is ideal for CI/CD pipelines or batch + testing. +

+ +

Usage

+ + +

Example Output

+ + +

Exit Codes

+
    +
  • + 0 - All scenarios passed +
    • +
  • + 1 - One or more scenarios failed +
    • +
+ + + + {/* Test Single Scenario */} + + + Test Single Scenario + tpm scenario test + + +

+ Run a single scenario by its ID. Useful for debugging or re-testing specific failures. +

+ +

Usage

+ + +

Example Output

+ + + + + {/* Scenario Info */} + + + View Scenario Details + tpm scenario info + + +

+ View detailed information about a scenario including its run history and quality + metrics. +

+ +

Usage

+ + +

Example Output

+ + + + + {/* Quality Scoring */} + + + Quality Scoring + How scenario quality is calculated + + +

+ Quality scores help identify reliable scenarios and track improvement over time. Scores + range from 0% to 100%. +

+ +

Streak-Based Scoring

+

+ Scenarios earn bonus points for consecutive passes and lose points for consecutive + failures: +

+ +
+
+
+ On Pass +
+
    +
  • +5% base score
    • +
  • +1% per consecutive pass
    • +
  • Maximum score: 100%
    • +
+
+
+
+ On Failure +
+
    +
  • -10% base penalty
    • +
  • -2% per consecutive fail
    • +
  • Minimum score: 0%
    • +
+
+
+ +
+

Example

+

+ A scenario with 5 consecutive passes would have a quality score of approximately 85% + (5% × 5 + 1% × (1+2+3+4+5) = 25% + 15% = 40%, plus base score). High-quality scenarios + are featured on the TPMJS homepage showcase. +

+
+ + + + {/* CI/CD Integration */} + + + CI/CD Integration + Automate scenario testing in your pipeline + + +

+ Integrate scenario testing into your CI/CD pipeline to catch regressions early. +

+ +

GitHub Actions Example

+ ~/.config/tpmjs/config.json + + - name: Run Scenarios + run: tpm scenario run my-collection --json > results.json + + - name: Check Results + run: | + FAILED=$(jq '.failed' results.json) + if [ "$FAILED" -gt 0 ]; then + echo "❌ $FAILED scenario(s) failed" + exit 1 + fi + echo "✅ All scenarios passed"`} + language="yaml" + showCopy={true} + /> + +

JSON Output Format

+ + + + + {/* Rate Limits */} + + + Rate Limits + Daily quota and usage tracking + + +

+ Scenario execution is subject to daily rate limits to ensure fair usage: +

+ +
    +
  • + Free tier: 50 scenario runs per day +
    • +
  • + Pro tier: 500 scenario runs per day +
    • +
  • Quotas reset at midnight UTC
    • +
  • Failed runs count toward the quota
    • +
+ +

+ The remaining quota is shown after each scenario run. Plan your CI/CD schedules + accordingly to stay within limits. +

+ + + + {/* Next Steps */} + + + Next Steps + Continue learning about TPMJS + + +
    +
  • + + Scenarios API Reference → + +

    + REST API endpoints for programmatic scenario management +

    +
    • +
  • + + Collections API → + +

    + Create and manage tool collections for your scenarios +

    +
    • +
  • + + Agents Documentation → + +

    + Learn how scenarios use agents for execution +

    +
    • +
  • + + Browse Tool Registry → + +

    + Discover tools to add to your collections +

    +
    • +
+ + +
+ ); +}