From 5fafc90e3e6a559cb556aef8bbd5aa48e7a02eaa Mon Sep 17 00:00:00 2001 From: Ajax Davis Date: Tue, 20 Jan 2026 14:15:16 +1000 Subject: [PATCH] fix: reorder collection page sections and add layout to docs page - Move use cases section above scenarios section on collection detail pages - Add AppHeader and AppFooter to /docs/developers/guide page for consistent layout --- .../[slug]/CollectionDetailClient.tsx | 20 +- .../src/app/docs/developers/guide/page.tsx | 1106 +++++++++-------- 2 files changed, 574 insertions(+), 552 deletions(-) diff --git a/apps/web/src/app/(profile)/[username]/collections/[slug]/CollectionDetailClient.tsx b/apps/web/src/app/(profile)/[username]/collections/[slug]/CollectionDetailClient.tsx index d5c1ef2..afca7ee 100644 --- a/apps/web/src/app/(profile)/[username]/collections/[slug]/CollectionDetailClient.tsx +++ b/apps/web/src/app/(profile)/[username]/collections/[slug]/CollectionDetailClient.tsx @@ -393,6 +393,16 @@ export function CollectionDetailClient({ )} + {/* Use Cases Section */} + {collection.tools.length > 0 && ( + + )} + {/* Scenarios Section */} {collection.tools.length > 0 && ( )} - - {/* Use Cases Section - at the bottom */} - {collection.tools.length > 0 && ( - - )} diff --git a/apps/web/src/app/docs/developers/guide/page.tsx b/apps/web/src/app/docs/developers/guide/page.tsx index c0d4454..c09287a 100644 --- a/apps/web/src/app/docs/developers/guide/page.tsx +++ b/apps/web/src/app/docs/developers/guide/page.tsx @@ -3,6 +3,9 @@ import { CodeBlock } from '@tpmjs/ui/CodeBlock/CodeBlock'; import type { Metadata } from 'next'; import Link from 'next/link'; +import { AppFooter } from '~/components/AppFooter'; +import { AppHeader } from '~/components/AppHeader'; + export const metadata: Metadata = { title: 'Developers Guide | TPMJS Docs', description: @@ -11,141 +14,145 @@ export const metadata: Metadata = { export default function DevelopersGuidePage(): React.ReactElement { return ( -
- {/* Introduction */} -
-

Developers Guide: Scenarios

-

- Scenarios are TPMJS's AI-powered testing and evaluation system for tool collections. - They provide automated testing, quality tracking, and regression prevention for your - tools. -

-
+
+ - {/* What are Scenarios? */} - - - What Are Scenarios? - - Automated test cases for tool collections with LLM evaluation - - - -

- A Scenario is a test case that simulates how a real user would interact with your tool - collection. Unlike traditional unit tests that verify individual functions, scenarios - exercise your tools end-to-end with realistic prompts and assertions. -

- -
-

Key Components:

-
    -
  • - AI Agent Execution: An ephemeral agent is created with your - collection's tools -
    • -
  • - Realistic Prompt: The agent executes your scenario's prompt - against the tools -
    • -
  • - LLM Evaluation: An LLM analyzes whether the task was completed - successfully -
    • -
  • - Result Recording: All results are stored for quality tracking and - historical analysis -
    • -
-
- -

- Think of scenarios as integration tests with AI intelligenceโ€”instead of hard-coded - assertions, scenarios use natural language evaluation to verify your tools work as - intended. -

- - - - {/* Why Use Scenarios? */} - - - Why Use Scenarios? - Benefits over traditional testing approaches - - -
-
-

๐Ÿงช Continuous Testing

-

- Run scenarios automatically on every code change to catch regressions early. -

-
-
-

๐ŸŽฏ AI-Powered Validation

-

- Use LLM evaluation to verify your tools actually solve real problems, not just pass - code tests. -

-
-
-

๐Ÿ“Š Quality Metrics

-

- Track quality scores over time to identify reliable scenarios and areas for - improvement. -

-
-
-

๐Ÿ”„ CI/CD Ready

-

- Integrate seamlessly into your pipeline with JSON output and exit codes. -

-
-
- -
-

When to Use Scenarios

-
    -
  • - Multi-Tool Integration: Testing how multiple tools work together in - complex workflows -
    • -
  • - API-First Tools: Validating tools that make HTTP requests or parse - unstructured data -
    • -
  • - Quality-Critical Collections: When tool reliability impacts user - experience -
    • -
  • - Regression Prevention: Before deploying changes that might break - existing functionality -
    • -
  • - Documentation-Driven Testing: Ensuring tools work as described in - their public documentation -
    • -
-
- - - - {/* How Scenarios Work */} - - - How Scenarios Work - Technical implementation and execution flow - - +
+
+ {/* Introduction */}
-

1. Scenario Definition

-

- Scenarios are defined as test cases with a prompt, expected outputs, and optional - assertions. They can be created manually or AI-generated. +

Developers Guide: Scenarios

+

+ Scenarios are TPMJS's AI-powered testing and evaluation system for tool + collections. They provide automated testing, quality tracking, and regression + prevention for your tools.

- + + {/* What are Scenarios? */} + + + What Are Scenarios? + + Automated test cases for tool collections with LLM evaluation + + + +

+ A Scenario is a test case that simulates how a real user would interact with your + tool collection. Unlike traditional unit tests that verify individual functions, + scenarios exercise your tools end-to-end with realistic prompts and assertions. +

+ +
+

Key Components:

+
    +
  • + AI Agent Execution: An ephemeral agent is created with your + collection's tools +
    • +
  • + Realistic Prompt: The agent executes your scenario's + prompt against the tools +
    • +
  • + LLM Evaluation: An LLM analyzes whether the task was completed + successfully +
    • +
  • + Result Recording: All results are stored for quality tracking + and historical analysis +
    • +
+
+ +

+ Think of scenarios as integration tests with AI intelligenceโ€”instead of hard-coded + assertions, scenarios use natural language evaluation to verify your tools work as + intended. +

+ + + + {/* Why Use Scenarios? */} + + + Why Use Scenarios? + Benefits over traditional testing approaches + + +
+
+

๐Ÿงช Continuous Testing

+

+ Run scenarios automatically on every code change to catch regressions early. +

+
+
+

๐ŸŽฏ AI-Powered Validation

+

+ Use LLM evaluation to verify your tools actually solve real problems, not just + pass code tests. +

+
+
+

๐Ÿ“Š Quality Metrics

+

+ Track quality scores over time to identify reliable scenarios and areas for + improvement. +

+
+
+

๐Ÿ”„ CI/CD Ready

+

+ Integrate seamlessly into your pipeline with JSON output and exit codes. +

+
+
+ +
+

When to Use Scenarios

+
    +
  • + Multi-Tool Integration: Testing how multiple tools work + together in complex workflows +
    • +
  • + API-First Tools: Validating tools that make HTTP requests or + parse unstructured data +
    • +
  • + Quality-Critical Collections: When tool reliability impacts + user experience +
    • +
  • + Regression Prevention: Before deploying changes that might + break existing functionality +
    • +
  • + Documentation-Driven Testing: Ensuring tools work as described + in their public documentation +
    • +
+
+ + + + {/* How Scenarios Work */} + + + How Scenarios Work + Technical implementation and execution flow + + +
+

1. Scenario Definition

+

+ Scenarios are defined as test cases with a prompt, expected outputs, and optional + assertions. They can be created manually or AI-generated. +

+ -
+ language="json" + showCopy={false} + /> +
-
-

2. Agent Execution

-

- When a scenario runs, an ephemeral AI agent is created with your tool collection. The - agent executes the scenario prompt using only the tools in that collectionโ€”no external - access, no additional context. -

- +

2. Agent Execution

+

+ When a scenario runs, an ephemeral AI agent is created with your tool collection. + The agent executes the scenario prompt using only the tools in that collectionโ€”no + external access, no additional context. +

+ -
+ language="text" + showCopy={false} + /> +
-
-

3. LLM Evaluation

-

- After execution, a powerful LLM evaluates the results. This isn't just checking - for errorsโ€”it uses natural language understanding to verify the task was completed - correctly. -

- +

3. LLM Evaluation

+

+ After execution, a powerful LLM evaluates the results. This isn't just + checking for errorsโ€”it uses natural language understanding to verify the task was + completed correctly. +

+ -
+ language="text" + showCopy={false} + /> +
-
-

4. Quality Scoring

-

- Quality scores (0-100%) track scenario reliability over time using a streak-based - system: -

-
-
-
- ๐ŸŽ‰ On Pass -
-
    -
  • +50% base score
    • -
  • +5% bonus per consecutive pass
    • -
  • Maximum: 100%
    • +
    +

    4. Quality Scoring

    +

    + Quality scores (0-100%) track scenario reliability over time using a streak-based + system: +

    +
    +
    +
    + ๐ŸŽ‰ On Pass +
    +
      +
    • +50% base score
      • +
    • +5% bonus per consecutive pass
      • +
    • Maximum: 100%
      • +
    +
    +
    +
    + โš ๏ธ On Failure +
    +
      +
    • -20% base penalty
      • +
    • -5% penalty per consecutive fail
      • +
    • Minimum: 0%
      • +
    +
    +
    +

    + Example: A scenario that passes 5 times consecutively earns 25% + bonus (50% + 5ร—5) for a total score of ~75%. High-quality scenarios (75%+) are + featured on the TPMJS homepage showcase. +

    +
    + +
    +

    5. Conversation History

    +

    + Every scenario execution captures the full conversation history between the agent, + your tools, and any outputs. This enables: +

    +
      +
    • + Full Transparency: See exactly what the agent asked and how + each tool responded +
      • +
    • + Debugging: Inspect tool inputs, outputs, and errors in detail +
      • +
    • + Raw JSON Export: Copy the entire conversation for analysis or + automation +
      • +
    • + Usage Metrics: Track token usage, execution time, and costs per + scenario run +
    -
    -
    โš ๏ธ On Failure
    -
      -
    • -20% base penalty
      • -
    • -5% penalty per consecutive fail
      • -
    • Minimum: 0%
      • -
    -
    -
-

- Example: A scenario that passes 5 times consecutively earns 25% bonus - (50% + 5ร—5) for a total score of ~75%. High-quality scenarios (75%+) are featured on - the TPMJS homepage showcase. -

-
+ + -
-

5. Conversation History

-

- Every scenario execution captures the full conversation history between the agent, - your tools, and any outputs. This enables: -

-
    -
  • - Full Transparency: See exactly what the agent asked and how each - tool responded -
    • -
  • - Debugging: Inspect tool inputs, outputs, and errors in detail -
    • -
  • - Raw JSON Export: Copy the entire conversation for analysis or - automation -
    • -
  • - Usage Metrics: Track token usage, execution time, and costs per - scenario run -
    • -
-
- - - - {/* Developer Use Cases */} - - - Developer Use Cases - - Practical scenarios for integrating scenario testing into your workflow - - - -
-

๐Ÿš€ CI/CD Integration

-

- Automate scenario testing in your deployment pipeline to catch regressions before they - reach production. -

- + + Developer Use Cases + + Practical scenarios for integrating scenario testing into your workflow + + + +
+

๐Ÿš€ CI/CD Integration

+

+ Automate scenario testing in your deployment pipeline to catch regressions before + they reach production. +

+ -

- Benefits: Block deploys on failures, generate test reports, and track - quality metrics over time. -

-
+ language="yaml" + showCopy={true} + /> +

+ Benefits: Block deploys on failures, generate test reports, and + track quality metrics over time. +

+
-
-

๐Ÿ” Local Development Testing

-

- Test scenarios locally during development to verify tool behavior without consuming - quota. -

- +

๐Ÿ” Local Development Testing

+

+ Test scenarios locally during development to verify tool behavior without + consuming quota. +

+ analysis.json`} - language="bash" - showCopy={true} - /> -

- Benefits: Fast feedback loop, no quota consumption, and detailed - execution traces for debugging. -

-
+ language="bash" + showCopy={true} + /> +

+ Benefits: Fast feedback loop, no quota consumption, and detailed + execution traces for debugging. +

+
-
-

๐Ÿ“Š Quality Monitoring Dashboard

-

- Use scenario results to build dashboards showing tool reliability, success rates, and - performance trends. -

- +

๐Ÿ“Š Quality Monitoring Dashboard

+

+ Use scenario results to build dashboards showing tool reliability, success rates, + and performance trends. +

+ metrics.json # Analyze with scripts @@ -368,23 +377,23 @@ node analyze-metrics.js \\ curl -X POST https://your-monitoring.com/webhook \\ -H "Content-Type: application/json" \\ -d @metrics.json`} - language="bash" - showCopy={true} - /> -

- Benefits: Track improvements over time, identify flaky tools, and - demonstrate reliability to users. -

-
+ language="bash" + showCopy={true} + /> +

+ Benefits: Track improvements over time, identify flaky tools, and + demonstrate reliability to users. +

+
-
-

๐ŸŽ“ Tool Library Development

-

- Ensure your tools work correctly with scenarios before publishing them. High-quality - scenarios increase tool visibility on the TPMJS homepage. -

- +

๐ŸŽ“ Tool Library Development

+

+ Ensure your tools work correctly with scenarios before publishing them. + High-quality scenarios increase tool visibility on the TPMJS homepage. +

+ 75% get featured in homepage showcase Quality metrics attract more users and increase adoption.`} - language="bash" - showCopy={false} - /> -
- - + language="bash" + showCopy={false} + /> + + + - {/* Comparison Table */} - - - Testing Approaches Comparison - Scenarios vs Traditional Testing - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FeatureUnit Tests - Integration Tests - Scenarios
- Execution Model - - Code runs individual functions - - Full system with external services - AI agent + your tools
- Assertions - - Hard-coded conditions (a === b) - - Integration with test database - LLM natural language evaluation
- Coverage - - Function-level code coverage - - End-to-end workflow coverage - Realistic user prompt coverage
- Maintenance - - Brittle with implementation changes - Better with refactoringSelf-healing with AI prompts
- Debugging - Stack traces, breakpoints - Network logs, service logs - - Full conversation history with tool I/O -
- Best For - - Pure algorithms, mathematical functions - - Business logic, workflows, APIs - - AI tools, LLM interaction, realistic prompts -
-
- - + {/* Comparison Table */} + + + Testing Approaches Comparison + Scenarios vs Traditional Testing + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Feature + Unit Tests + + Integration Tests + Scenarios
+ Execution Model + + Code runs individual functions + + Full system with external services + AI agent + your tools
+ Assertions + + Hard-coded conditions (a === b) + + Integration with test database + LLM natural language evaluation
+ Coverage + + Function-level code coverage + + End-to-end workflow coverage + Realistic user prompt coverage
+ Maintenance + + Brittle with implementation changes + + Better with refactoring + Self-healing with AI prompts
+ Debugging + + Stack traces, breakpoints + + Network logs, service logs + + Full conversation history with tool I/O +
+ Best For + + Pure algorithms, mathematical functions + + Business logic, workflows, APIs + + AI tools, LLM interaction, realistic prompts +
+
+ + - {/* Future of Scenarios */} - - - The Future of Scenarios - Roadmap and upcoming capabilities - - -
-

๐Ÿค– Self-Healing Scenarios

-

- Scenarios that analyze their own failures and automatically improve their prompts. - Failed scenarios can generate fixes or suggest alternative approaches. -

-
+ {/* Future of Scenarios */} + + + The Future of Scenarios + Roadmap and upcoming capabilities + + +
+

๐Ÿค– Self-Healing Scenarios

+

+ Scenarios that analyze their own failures and automatically improve their prompts. + Failed scenarios can generate fixes or suggest alternative approaches. +

+
-
-

๐Ÿ”„ Continuous Improvement Loop

-

- Each scenario run contributes to quality metrics, creating a feedback loop that - continuously improves tool quality and scenario design over time. -

-
+
+

๐Ÿ”„ Continuous Improvement Loop

+

+ Each scenario run contributes to quality metrics, creating a feedback loop that + continuously improves tool quality and scenario design over time. +

+
-
-

๐Ÿš€ CI/CD Integration

-

- Automate scenario testing in your deployment pipeline to catch regressions before they - reach production. -

- +

๐Ÿš€ CI/CD Integration

+

+ Automate scenario testing in your deployment pipeline to catch regressions before + they reach production. +

+ -

- Benefits: Block deploys on failures, generate test reports, and track - quality metrics over time. -

-
+ language="bash" + showCopy={true} + /> +

+ Benefits: Block deploys on failures, generate test reports, and + track quality metrics over time. +

+ -
-

๐Ÿ“ˆ Advanced Evaluation Metrics

-

- Beyond simple pass/fail, future scenarios will measure token efficiency, response - quality, semantic correctness, and hallucination rates. -

-
+
+

๐Ÿ“ˆ Advanced Evaluation Metrics

+

+ Beyond simple pass/fail, future scenarios will measure token efficiency, response + quality, semantic correctness, and hallucination rates. +

+
-
-

๐Ÿ”Œ Scenario Marketplace

-

- Share high-quality scenarios across organizations, enabling collaborative testing and - faster scenario coverage for popular tools. -

-
+
+

๐Ÿ”Œ Scenario Marketplace

+

+ Share high-quality scenarios across organizations, enabling collaborative testing + and faster scenario coverage for popular tools. +

+
-
-

Vision

-

- Scenarios will become the de facto standard for tool quality assurance on TPMJS. - Before users install a tool, they'll see its scenario history, quality scores, - and real-world performance metrics. -

-
- - +
+

Vision

+

+ Scenarios will become the de facto standard for tool quality assurance on TPMJS. + Before users install a tool, they'll see its scenario history, quality + scores, and real-world performance metrics. +

+
+ + - {/* Getting Started */} - - - Quick Start - Get started with scenarios in 5 minutes - - -
-

1. Install TPMJS CLI

- -
+ {/* Getting Started */} + + + Quick Start + Get started with scenarios in 5 minutes + + +
+

1. Install TPMJS CLI

+ +
-
-

2. Authenticate

-

- Get a TPMJS API key from your{' '} - - dashboard settings - -

- -
+
+

2. Authenticate

+

+ Get a TPMJS API key from your{' '} + + dashboard settings + +

+ +
-
-

3. Create a Collection

-

- Add tools to a collection if you don't have one yet. -

-
+
+

3. Create a Collection

+

+ Add tools to a collection if you don't have one yet. +

+
-
-

4. Generate Your First Scenario

- +

4. Generate Your First Scenario

+ -
+ language="bash" + showCopy={true} + /> + -
-

5. Run Your Scenarios

- +

5. Run Your Scenarios

+ -
- - + language="bash" + showCopy={true} + /> + + + - {/* Next Steps */} - - - Next Steps - Continue your journey - - -
    -
  • - - Scenarios API Reference โ†’ - -

    - REST API endpoints for programmatic scenario management -

    -
    • -
  • - - Scenarios CLI Guide โ†’ - -

    - Complete CLI commands for scenario generation and execution -

    -
    • -
  • - - Agents Documentation โ†’ - -

    - Learn how scenarios use agents internally for execution -

    -
    • -
  • - - My Collections โ†’ - -

    - Create and manage collections with your tools -

    -
    • -
- - + {/* Next Steps */} + + + Next Steps + Continue your journey + + +
    +
  • + + Scenarios API Reference โ†’ + +

    + REST API endpoints for programmatic scenario management +

    +
    • +
  • + + Scenarios CLI Guide โ†’ + +

    + Complete CLI commands for scenario generation and execution +

    +
    • +
  • + + Agents Documentation โ†’ + +

    + Learn how scenarios use agents internally for execution +

    +
    • +
  • + + My Collections โ†’ + +

    + Create and manage collections with your tools +

    +
    • +
+ + + + + + ); }