diff --git a/apps/web/src/app/docs/developers/guide/page.tsx b/apps/web/src/app/docs/developers/guide/page.tsx new file mode 100644 index 0000000..c0d4454 --- /dev/null +++ b/apps/web/src/app/docs/developers/guide/page.tsx @@ -0,0 +1,690 @@ +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@tpmjs/ui/Card/Card'; +import { CodeBlock } from '@tpmjs/ui/CodeBlock/CodeBlock'; +import type { Metadata } from 'next'; +import Link from 'next/link'; + +export const metadata: Metadata = { + title: 'Developers Guide | TPMJS Docs', + description: + 'Understand Scenarios for developers - AI-powered testing, evaluation, and continuous integration for tool collections', +}; + +export default function DevelopersGuidePage(): React.ReactElement { + return ( +
+ {/* Introduction */} +
+

Developers Guide: Scenarios

+

+ Scenarios are TPMJS's AI-powered testing and evaluation system for tool collections. + They provide automated testing, quality tracking, and regression prevention for your + tools. +

+
+ + {/* What are Scenarios? */} + + + What Are Scenarios? + + Automated test cases for tool collections with LLM evaluation + + + +

+ A Scenario is a test case that simulates how a real user would interact with your tool + collection. Unlike traditional unit tests that verify individual functions, scenarios + exercise your tools end-to-end with realistic prompts and assertions. +

+ +
+

Key Components:

+
    +
  • + AI Agent Execution: An ephemeral agent is created with your + collection's tools +
    • +
  • + Realistic Prompt: The agent executes your scenario's prompt + against the tools +
    • +
  • + LLM Evaluation: An LLM analyzes whether the task was completed + successfully +
    • +
  • + Result Recording: All results are stored for quality tracking and + historical analysis +
    • +
+
+ +

+ Think of scenarios as integration tests with AI intelligenceโ€”instead of hard-coded + assertions, scenarios use natural language evaluation to verify your tools work as + intended. +

+ + + + {/* Why Use Scenarios? */} + + + Why Use Scenarios? + Benefits over traditional testing approaches + + +
+
+

๐Ÿงช Continuous Testing

+

+ Run scenarios automatically on every code change to catch regressions early. +

+
+
+

๐ŸŽฏ AI-Powered Validation

+

+ Use LLM evaluation to verify your tools actually solve real problems, not just pass + code tests. +

+
+
+

๐Ÿ“Š Quality Metrics

+

+ Track quality scores over time to identify reliable scenarios and areas for + improvement. +

+
+
+

๐Ÿ”„ CI/CD Ready

+

+ Integrate seamlessly into your pipeline with JSON output and exit codes. +

+
+
+ +
+

When to Use Scenarios

+
    +
  • + Multi-Tool Integration: Testing how multiple tools work together in + complex workflows +
    • +
  • + API-First Tools: Validating tools that make HTTP requests or parse + unstructured data +
    • +
  • + Quality-Critical Collections: When tool reliability impacts user + experience +
    • +
  • + Regression Prevention: Before deploying changes that might break + existing functionality +
    • +
  • + Documentation-Driven Testing: Ensuring tools work as described in + their public documentation +
    • +
+
+ + + + {/* How Scenarios Work */} + + + How Scenarios Work + Technical implementation and execution flow + + +
+

1. Scenario Definition

+

+ Scenarios are defined as test cases with a prompt, expected outputs, and optional + assertions. They can be created manually or AI-generated. +

+ +
+ +
+

2. Agent Execution

+

+ When a scenario runs, an ephemeral AI agent is created with your tool collection. The + agent executes the scenario prompt using only the tools in that collectionโ€”no external + access, no additional context. +

+ +
+ +
+

3. LLM Evaluation

+

+ After execution, a powerful LLM evaluates the results. This isn't just checking + for errorsโ€”it uses natural language understanding to verify the task was completed + correctly. +

+ +
+ +
+

4. Quality Scoring

+

+ Quality scores (0-100%) track scenario reliability over time using a streak-based + system: +

+
+
+
+ ๐ŸŽ‰ On Pass +
+
    +
  • +50% base score
    • +
  • +5% bonus per consecutive pass
    • +
  • Maximum: 100%
    • +
+
+
+
โš ๏ธ On Failure
+
    +
  • -20% base penalty
    • +
  • -5% penalty per consecutive fail
    • +
  • Minimum: 0%
    • +
+
+
+

+ Example: A scenario that passes 5 times consecutively earns 25% bonus + (50% + 5ร—5) for a total score of ~75%. High-quality scenarios (75%+) are featured on + the TPMJS homepage showcase. +

+
+ +
+

5. Conversation History

+

+ Every scenario execution captures the full conversation history between the agent, + your tools, and any outputs. This enables: +

+
    +
  • + Full Transparency: See exactly what the agent asked and how each + tool responded +
    • +
  • + Debugging: Inspect tool inputs, outputs, and errors in detail +
    • +
  • + Raw JSON Export: Copy the entire conversation for analysis or + automation +
    • +
  • + Usage Metrics: Track token usage, execution time, and costs per + scenario run +
    • +
+
+ + + + {/* Developer Use Cases */} + + + Developer Use Cases + + Practical scenarios for integrating scenario testing into your workflow + + + +
+

๐Ÿš€ CI/CD Integration

+

+ Automate scenario testing in your deployment pipeline to catch regressions before they + reach production. +

+ ~/.config/tpmjs/config.json + + - name: Run All Scenarios + run: tpm scenario run my-collection --json + + - name: Verify Results + run: | + FAILED=$(jq '.failed' results.json) + if [ "$FAILED" -gt 0 ]; then + echo "โŒ Scenario testing failed" + exit 1 + fi + echo "โœ… All scenarios passed"}`} + language="yaml" + showCopy={true} + /> +

+ Benefits: Block deploys on failures, generate test reports, and track + quality metrics over time. +

+
+ +
+

๐Ÿ” Local Development Testing

+

+ Test scenarios locally during development to verify tool behavior without consuming + quota. +

+ analysis.json`} + language="bash" + showCopy={true} + /> +

+ Benefits: Fast feedback loop, no quota consumption, and detailed + execution traces for debugging. +

+
+ +
+

๐Ÿ“Š Quality Monitoring Dashboard

+

+ Use scenario results to build dashboards showing tool reliability, success rates, and + performance trends. +

+ metrics.json + +# Analyze with scripts +node analyze-metrics.js \\ + --input metrics.json \\ + --output report.html \\ + --chart-type=timeline + +# Integrate with external monitoring +curl -X POST https://your-monitoring.com/webhook \\ + -H "Content-Type: application/json" \\ + -d @metrics.json`} + language="bash" + showCopy={true} + /> +

+ Benefits: Track improvements over time, identify flaky tools, and + demonstrate reliability to users. +

+
+ +
+

๐ŸŽ“ Tool Library Development

+

+ Ensure your tools work correctly with scenarios before publishing them. High-quality + scenarios increase tool visibility on the TPMJS homepage. +

+ 75% get featured in homepage showcase +Quality metrics attract more users and increase adoption.`} + language="bash" + showCopy={false} + /> +
+ + + + {/* Comparison Table */} + + + Testing Approaches Comparison + Scenarios vs Traditional Testing + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FeatureUnit Tests + Integration Tests + Scenarios
+ Execution Model + + Code runs individual functions + + Full system with external services + AI agent + your tools
+ Assertions + + Hard-coded conditions (a === b) + + Integration with test database + LLM natural language evaluation
+ Coverage + + Function-level code coverage + + End-to-end workflow coverage + Realistic user prompt coverage
+ Maintenance + + Brittle with implementation changes + Better with refactoringSelf-healing with AI prompts
+ Debugging + Stack traces, breakpoints + Network logs, service logs + + Full conversation history with tool I/O +
+ Best For + + Pure algorithms, mathematical functions + + Business logic, workflows, APIs + + AI tools, LLM interaction, realistic prompts +
+
+ + + + {/* Future of Scenarios */} + + + The Future of Scenarios + Roadmap and upcoming capabilities + + +
+

๐Ÿค– Self-Healing Scenarios

+

+ Scenarios that analyze their own failures and automatically improve their prompts. + Failed scenarios can generate fixes or suggest alternative approaches. +

+
+ +
+

๐Ÿ”„ Continuous Improvement Loop

+

+ Each scenario run contributes to quality metrics, creating a feedback loop that + continuously improves tool quality and scenario design over time. +

+
+ +
+

๐Ÿš€ CI/CD Integration

+

+ Automate scenario testing in your deployment pipeline to catch regressions before they + reach production. +

+ +

+ Benefits: Block deploys on failures, generate test reports, and track + quality metrics over time. +

+
+ +
+

๐Ÿ“ˆ Advanced Evaluation Metrics

+

+ Beyond simple pass/fail, future scenarios will measure token efficiency, response + quality, semantic correctness, and hallucination rates. +

+
+ +
+

๐Ÿ”Œ Scenario Marketplace

+

+ Share high-quality scenarios across organizations, enabling collaborative testing and + faster scenario coverage for popular tools. +

+
+ +
+

Vision

+

+ Scenarios will become the de facto standard for tool quality assurance on TPMJS. + Before users install a tool, they'll see its scenario history, quality scores, + and real-world performance metrics. +

+
+ + + + {/* Getting Started */} + + + Quick Start + Get started with scenarios in 5 minutes + + +
+

1. Install TPMJS CLI

+ +
+ +
+

2. Authenticate

+

+ Get a TPMJS API key from your{' '} + + dashboard settings + +

+ +
+ +
+

3. Create a Collection

+

+ Add tools to a collection if you don't have one yet. +

+
+ +
+

4. Generate Your First Scenario

+ +
+ +
+

5. Run Your Scenarios

+ +
+ + + + {/* Next Steps */} + + + Next Steps + Continue your journey + + +
    +
  • + + Scenarios API Reference โ†’ + +

    + REST API endpoints for programmatic scenario management +

    +
    • +
  • + + Scenarios CLI Guide โ†’ + +

    + Complete CLI commands for scenario generation and execution +

    +
    • +
  • + + Agents Documentation โ†’ + +

    + Learn how scenarios use agents internally for execution +

    +
    • +
  • + + My Collections โ†’ + +

    + Create and manage collections with your tools +

    +
    • +
+ + +
+ ); +} diff --git a/apps/web/src/components/AppHeader.tsx b/apps/web/src/components/AppHeader.tsx index 62d75d2..fc89189 100644 --- a/apps/web/src/components/AppHeader.tsx +++ b/apps/web/src/components/AppHeader.tsx @@ -159,6 +159,11 @@ function NavDropdown({ label, items }: NavDropdownProps): React.ReactElement { const developerItems: DropdownItem[] = [ { href: '/docs', label: 'Documentation', description: 'Guides and tutorials' }, + { + href: '/docs/developers/guide', + label: 'Developers Guide', + description: 'Scenarios, testing & quality', + }, { href: '/docs/api', label: 'API Reference', description: 'REST & MCP endpoints' }, { href: '/docs/executors', label: 'Custom Executors', description: 'Deploy your own' }, { href: '/sdk', label: 'SDK', description: 'Build with our SDK' },