diff --git a/.dependency-cruiser.js b/.dependency-cruiser.js index 2ffd268..8710f08 100644 --- a/.dependency-cruiser.js +++ b/.dependency-cruiser.js @@ -105,8 +105,8 @@ export default { from: {}, to: { couldNotResolve: true, - // Allow TypeScript path aliases that are resolved by the TS compiler - pathNot: ['^~/'], + // Allow TypeScript path aliases and workspace packages that are resolved by the TS compiler + pathNot: ['^~/', '^@/', '^@tpmjs/'], }, }, { @@ -143,6 +143,10 @@ export default { doNotFollow: { path: ['node_modules', '\\.next', 'dist', '\\.turbo', 'storybook-static'], }, + exclude: { + // Exclude railway-executor - it's a Deno app with HTTP imports that can't be resolved + path: '^apps/railway-executor', + }, tsPreCompilationDeps: true, tsConfig: { fileName: './tsconfig.json', diff --git a/.env.vercel.production b/.env.vercel.production new file mode 100644 index 0000000..bf0ac19 --- /dev/null +++ b/.env.vercel.production @@ -0,0 +1,41 @@ +# Created by Vercel CLI +CRON_SECRET="CRON_SECRET=6c806d35cf6212f489c76414d38d2b6acbc44590ac78bb08aadea28dd04a29d0\n" +DATABASE_URL="postgresql://neondb_owner:npg_euvYo4OTi1lX@ep-broad-darkness-a4lml85k-pooler.us-east-1.aws.neon.tech/neondb?sslmode=require" +DATABASE_URL_UNPOOLED="postgresql://neondb_owner:npg_euvYo4OTi1lX@ep-broad-darkness-a4lml85k.us-east-1.aws.neon.tech/neondb?sslmode=require" +NEXT_PUBLIC_STACK_PROJECT_ID="d786bd3a-a31d-4c6b-9497-5d6803dd9d86" +NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY="pck_hafmpkaj047z331x5azv8bk5zggfnbgdedbj9pfqh1rn0" +NX_DAEMON="false" +PGDATABASE="neondb" +PGHOST="ep-broad-darkness-a4lml85k-pooler.us-east-1.aws.neon.tech" +PGHOST_UNPOOLED="ep-broad-darkness-a4lml85k.us-east-1.aws.neon.tech" +PGPASSWORD="npg_euvYo4OTi1lX" +PGUSER="neondb_owner" +POSTGRES_DATABASE="neondb" +POSTGRES_HOST="ep-broad-darkness-a4lml85k-pooler.us-east-1.aws.neon.tech" +POSTGRES_PASSWORD="npg_euvYo4OTi1lX" +POSTGRES_PRISMA_URL="postgresql://neondb_owner:npg_euvYo4OTi1lX@ep-broad-darkness-a4lml85k-pooler.us-east-1.aws.neon.tech/neondb?connect_timeout=15&sslmode=require" +POSTGRES_URL="postgresql://neondb_owner:npg_euvYo4OTi1lX@ep-broad-darkness-a4lml85k-pooler.us-east-1.aws.neon.tech/neondb?sslmode=require" +POSTGRES_URL_NON_POOLING="postgresql://neondb_owner:npg_euvYo4OTi1lX@ep-broad-darkness-a4lml85k.us-east-1.aws.neon.tech/neondb?sslmode=require" +POSTGRES_URL_NO_SSL="postgresql://neondb_owner:npg_euvYo4OTi1lX@ep-broad-darkness-a4lml85k-pooler.us-east-1.aws.neon.tech/neondb" +POSTGRES_USER="neondb_owner" +STACK_SECRET_SERVER_KEY="ssk_p05kwe938wx13rpera9xf1fewc816dwkbq658xcsbwj1g" +TURBO_CACHE="remote:rw" +TURBO_DOWNLOAD_LOCAL_ENABLED="true" +TURBO_REMOTE_ONLY="true" +TURBO_RUN_SUMMARY="true" +VERCEL="1" +VERCEL_ENV="production" +VERCEL_GIT_COMMIT_AUTHOR_LOGIN="" +VERCEL_GIT_COMMIT_AUTHOR_NAME="" +VERCEL_GIT_COMMIT_MESSAGE="" +VERCEL_GIT_COMMIT_REF="" +VERCEL_GIT_COMMIT_SHA="" +VERCEL_GIT_PREVIOUS_SHA="" +VERCEL_GIT_PROVIDER="" +VERCEL_GIT_PULL_REQUEST_ID="" +VERCEL_GIT_REPO_ID="" +VERCEL_GIT_REPO_OWNER="" +VERCEL_GIT_REPO_SLUG="" +VERCEL_OIDC_TOKEN="eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6Im1yay00MzAyZWMxYjY3MGY0OGE5OGFkNjFkYWRlNGEyM2JlNyJ9.eyJpc3MiOiJodHRwczovL29pZGMudmVyY2VsLmNvbS90cG1qcyIsInN1YiI6Im93bmVyOnRwbWpzOnByb2plY3Q6dHBtanMtd2ViOmVudmlyb25tZW50OmRldmVsb3BtZW50Iiwic2NvcGUiOiJvd25lcjp0cG1qczpwcm9qZWN0OnRwbWpzLXdlYjplbnZpcm9ubWVudDpkZXZlbG9wbWVudCIsImF1ZCI6Imh0dHBzOi8vdmVyY2VsLmNvbS90cG1qcyIsIm93bmVyIjoidHBtanMiLCJvd25lcl9pZCI6InRlYW1femtHV0NXYjdWakhvbmk2VmJ5ZmQyc3c4IiwicHJvamVjdCI6InRwbWpzLXdlYiIsInByb2plY3RfaWQiOiJwcmpfNWd1MEkwVzFjUFhkQ3ozd1RjQ0ZIejQzNUJ0MCIsImVudmlyb25tZW50IjoiZGV2ZWxvcG1lbnQiLCJwbGFuIjoicHJvIiwidXNlcl9pZCI6IkxKZk05VzdIdlljb2gyclVCaXRWd283ViIsIm5iZiI6MTc2NDM4OTAxMiwiaWF0IjoxNzY0Mzg5MDEyLCJleHAiOjE3NjQ0MzIyMTJ9.OF4IHrcmteA2lU1tkqHO1a9ITGGrCjCo29G8jI991q8_SQjgHZHVqcBj3AYVKZJDh6BjHib4HyNKdjO8nwUblF2dCFbYDv6y4hwB6jHNpsz32BE1JDKcXEJOKPtg_tBOFUDKtzMkPk7VOPWDVYw8Tz4_HZ_MR3SNoy1Pk9AFL-hEl3E-zR3bAYMDB8tKrIm9y9K4sZF6efMU7BR_J6Bf-i3IsbbrH-Axgq5dewlpogf-xHWmWaTXoUp6UFejNKhSMXqg3sAWTnizYeSGc2Ut6zNuAYPumUPBdQ37Kk7vuRNwS1h7RJz3vtEg6aOuw0-Ld0LdF-tWkfDGsVqanR_sxw" +VERCEL_TARGET_ENV="production" +VERCEL_URL="" diff --git a/.github/workflows/health-check.yml b/.github/workflows/health-check.yml new file mode 100644 index 0000000..ddf2a15 --- /dev/null +++ b/.github/workflows/health-check.yml @@ -0,0 +1,18 @@ +name: Daily Health Check + +on: + schedule: + # Run daily at 2am UTC + - cron: '0 2 * * *' + workflow_dispatch: + +jobs: + health-check: + runs-on: ubuntu-latest + steps: + - name: Trigger health check sync + run: | + curl -X POST "${{ secrets.VERCEL_PRODUCTION_URL }}/api/sync/health-check" \ + -H "Authorization: Bearer ${{ secrets.CRON_SECRET }}" \ + -H "Content-Type: application/json" \ + -f -s -S -w "\nHTTP Status: %{http_code}\n" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5bbf39e..b758495 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 21 + node-version: 22 cache: 'pnpm' registry-url: 'https://registry.npmjs.org' diff --git a/.github/workflows/sync-changes.yml b/.github/workflows/sync-changes.yml new file mode 100644 index 0000000..fa2886e --- /dev/null +++ b/.github/workflows/sync-changes.yml @@ -0,0 +1,18 @@ +name: Sync NPM Changes Feed + +on: + schedule: + # Run every 2 minutes + - cron: '*/2 * * * *' + workflow_dispatch: + +jobs: + sync-changes: + runs-on: ubuntu-latest + steps: + - name: Trigger changes feed sync + run: | + curl -X POST "${{ secrets.VERCEL_PRODUCTION_URL }}/api/sync/changes" \ + -H "Authorization: Bearer ${{ secrets.CRON_SECRET }}" \ + -H "Content-Type: application/json" \ + -f -s -S -w "\nHTTP Status: %{http_code}\n" diff --git a/.github/workflows/sync-keyword.yml b/.github/workflows/sync-keyword.yml new file mode 100644 index 0000000..55c6399 --- /dev/null +++ b/.github/workflows/sync-keyword.yml @@ -0,0 +1,131 @@ +name: Sync NPM Keyword Search + +on: + schedule: + # Run every 15 minutes + - cron: '*/15 * * * *' + workflow_dispatch: + +jobs: + sync-keyword: + runs-on: ubuntu-latest + steps: + - name: Trigger keyword search sync + id: sync + run: | + # Call the sync API and capture response + response=$(curl -X POST "${{ secrets.VERCEL_PRODUCTION_URL }}/api/sync/keyword" \ + -H "Authorization: Bearer ${{ secrets.CRON_SECRET }}" \ + -H "Content-Type: application/json" \ + -f -s -S) + + echo "Response: $response" + + # Extract data using jq + processed=$(echo "$response" | jq -r '.data.processed') + skipped=$(echo "$response" | jq -r '.data.skipped') + errors=$(echo "$response" | jq -r '.data.errors') + packagesFound=$(echo "$response" | jq -r '.data.packagesFound') + durationMs=$(echo "$response" | jq -r '.data.durationMs') + + # Extract and display error messages + errorMessages=$(echo "$response" | jq -r '.data.errorMessages[]?' 2>/dev/null || echo "") + + if [ -n "$errorMessages" ]; then + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "⚠️ SYNC ERRORS ($errors total):" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "$response" | jq -r '.data.errorMessages[]?' | while IFS= read -r error; do + echo " • $error" + done + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + fi + + # Set outputs for Discord notification + echo "processed=$processed" >> $GITHUB_OUTPUT + echo "skipped=$skipped" >> $GITHUB_OUTPUT + echo "errors=$errors" >> $GITHUB_OUTPUT + echo "packagesFound=$packagesFound" >> $GITHUB_OUTPUT + echo "durationMs=$durationMs" >> $GITHUB_OUTPUT + + # Store error messages for Discord (first 3, truncated) + if [ "$errors" -gt 0 ]; then + errorSummary=$(echo "$response" | jq -r '.data.errorMessages[0:3]? | join("\n• ")' 2>/dev/null || echo "") + if [ -n "$errorSummary" ]; then + # Save to file to preserve newlines + echo "• $errorSummary" > /tmp/error_summary.txt + fi + fi + + # Store skipped packages for Discord + if [ "$skipped" -gt 0 ]; then + skippedList=$(echo "$response" | jq -r '.data.skippedPackages[]? | "\(.name) (by \(.author)) - \(.reason)"' 2>/dev/null | paste -sd "\n" - || echo "") + if [ -n "$skippedList" ]; then + echo "$skippedList" > /tmp/skipped_packages.txt + fi + fi + + # Determine status emoji + if [ "$errors" -gt 0 ]; then + echo "status_emoji=⚠️" >> $GITHUB_OUTPUT + echo "status_color=16776960" >> $GITHUB_OUTPUT # Yellow + else + echo "status_emoji=✅" >> $GITHUB_OUTPUT + echo "status_color=5763719" >> $GITHUB_OUTPUT # Green + fi + + - name: Send Discord notification + if: always() + run: | + # Format duration + duration_sec=$(echo "scale=2; ${{ steps.sync.outputs.durationMs }} / 1000" | bc) + + # Build Discord payload using jq for proper JSON escaping + # Read optional data + error_text="" + skipped_text="" + + if [ -f /tmp/error_summary.txt ] && [ ${{ steps.sync.outputs.errors }} -gt 0 ]; then + error_text=$(cat /tmp/error_summary.txt | head -c 800) + fi + + if [ -f /tmp/skipped_packages.txt ] && [ ${{ steps.sync.outputs.skipped }} -gt 0 ]; then + skipped_text=$(cat /tmp/skipped_packages.txt) + fi + + # Build fields array dynamically + base_fields='[ + { "name": "📦 Packages Found", "value": "${{ steps.sync.outputs.packagesFound }}", "inline": true }, + { "name": "✨ Processed", "value": "${{ steps.sync.outputs.processed }}", "inline": true }, + { "name": "⏭️ Skipped", "value": "${{ steps.sync.outputs.skipped }}", "inline": true }, + { "name": "❌ Errors", "value": "${{ steps.sync.outputs.errors }}", "inline": true }, + { "name": "⏱️ Duration", "value": "'"${duration_sec}s"'", "inline": true }, + { "name": "🔗 Run", "value": "[View Logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})", "inline": true } + ]' + + # Create payload with dynamic fields + payload=$(jq -n \ + --arg title "${{ steps.sync.outputs.status_emoji }} NPM Keyword Search Sync" \ + --argjson color ${{ steps.sync.outputs.status_color }} \ + --argjson baseFields "$base_fields" \ + --arg error_text "$error_text" \ + --arg skipped_text "$skipped_text" \ + --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%S.000Z)" \ + ' + { + embeds: [{ + title: $title, + color: $color, + fields: ( + $baseFields + + (if $skipped_text != "" then [{ name: "📋 Skipped Packages", value: $skipped_text, inline: false }] else [] end) + + (if $error_text != "" then [{ name: "🔍 Error Details", value: ("```\n" + $error_text + "\n```"), inline: false }] else [] end) + ), + timestamp: $timestamp + }] + }') + + # Send to Discord + curl -X POST "${{ secrets.DISCORD_WEBHOOK }}" \ + -H "Content-Type: application/json" \ + -d "$payload" diff --git a/.github/workflows/sync-manual.yml b/.github/workflows/sync-manual.yml new file mode 100644 index 0000000..135f5af --- /dev/null +++ b/.github/workflows/sync-manual.yml @@ -0,0 +1,97 @@ +name: Sync Manual Tools + +on: + schedule: + # Run daily at midnight UTC + - cron: '0 0 * * *' + workflow_dispatch: + # Run on pushes to main that modify manual-tools.ts + push: + branches: + - main + paths: + - 'manual-tools.ts' + - 'sync-manual-tools.ts' + +jobs: + sync-manual: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup pnpm + uses: pnpm/action-setup@v2 + with: + version: 8 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Generate Prisma Client + run: pnpm --filter=@tpmjs/db db:generate + + - name: Run manual tools sync + id: sync + run: | + # Run the sync script and capture output + output=$(pnpm tsx sync-manual-tools.ts 2>&1) + echo "$output" + + # Extract statistics from output + processed=$(echo "$output" | grep "Processed:" | awk '{print $2}') + skipped=$(echo "$output" | grep "Skipped:" | awk '{print $2}') + errors=$(echo "$output" | grep "Errors:" | awk '{print $2}') + total=$(echo "$output" | grep "Total manual tools:" | awk '{print $4}') + + # Set outputs for Discord notification + echo "processed=${processed:-0}" >> $GITHUB_OUTPUT + echo "skipped=${skipped:-0}" >> $GITHUB_OUTPUT + echo "errors=${errors:-0}" >> $GITHUB_OUTPUT + echo "total=${total:-0}" >> $GITHUB_OUTPUT + + # Determine status + if [ "${errors:-0}" -gt 0 ]; then + echo "status_emoji=⚠️" >> $GITHUB_OUTPUT + echo "status_color=16776960" >> $GITHUB_OUTPUT # Yellow + else + echo "status_emoji=✅" >> $GITHUB_OUTPUT + echo "status_color=5763719" >> $GITHUB_OUTPUT # Green + fi + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + + - name: Send Discord notification + if: always() + run: | + # Build Discord payload + payload=$(jq -n \ + --arg title "${{ steps.sync.outputs.status_emoji }} Manual Tools Sync" \ + --argjson color ${{ steps.sync.outputs.status_color }} \ + --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%S.000Z)" \ + ' + { + embeds: [{ + title: $title, + color: $color, + fields: [ + { name: "📦 Total Tools", value: "${{ steps.sync.outputs.total }}", inline: true }, + { name: "✨ Processed", value: "${{ steps.sync.outputs.processed }}", inline: true }, + { name: "⏭️ Skipped", value: "${{ steps.sync.outputs.skipped }}", inline: true }, + { name: "❌ Errors", value: "${{ steps.sync.outputs.errors }}", inline: true }, + { name: "🔗 Run", value: "[View Logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})", inline: true } + ], + timestamp: $timestamp + }] + }') + + # Send to Discord + curl -X POST "${{ secrets.DISCORD_WEBHOOK }}" \ + -H "Content-Type: application/json" \ + -d "$payload" diff --git a/.github/workflows/sync-metrics.yml b/.github/workflows/sync-metrics.yml new file mode 100644 index 0000000..7bd5c40 --- /dev/null +++ b/.github/workflows/sync-metrics.yml @@ -0,0 +1,18 @@ +name: Sync NPM Metrics + +on: + schedule: + # Run every hour + - cron: '0 * * * *' + workflow_dispatch: + +jobs: + sync-metrics: + runs-on: ubuntu-latest + steps: + - name: Trigger metrics sync + run: | + curl -X POST "${{ secrets.VERCEL_PRODUCTION_URL }}/api/sync/metrics" \ + -H "Authorization: Bearer ${{ secrets.CRON_SECRET }}" \ + -H "Content-Type: application/json" \ + -f -s -S -w "\nHTTP Status: %{http_code}\n" diff --git a/.github/workflows/sync-vercel-registry.yml b/.github/workflows/sync-vercel-registry.yml new file mode 100644 index 0000000..af21029 --- /dev/null +++ b/.github/workflows/sync-vercel-registry.yml @@ -0,0 +1,267 @@ +name: Sync Vercel AI Registry + +on: + schedule: + # Run every hour + - cron: '0 * * * *' + workflow_dispatch: + # Run on pushes to main that modify the sync script + push: + branches: + - main + paths: + - 'sync-vercel-registry.ts' + +permissions: + contents: write + +jobs: + sync-vercel: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.14.0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'pnpm' + + - name: Install dependencies + run: | + echo "📦 Installing dependencies..." + pnpm install --frozen-lockfile + echo "✅ Dependencies installed" + + - name: Run Vercel registry sync + id: sync + run: | + echo "════════════════════════════════════════" + echo "🚀 Starting Vercel AI Registry Sync" + echo "════════════════════════════════════════" + echo "" + echo "📅 Time: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" + echo "🔑 OpenAI API Key: ${OPENAI_API_KEY:0:8}..." + echo "" + + # Run the sync script and capture output + output=$(pnpm tsx sync-vercel-registry.ts 2>&1) + exit_code=$? + + echo "$output" + echo "" + + # Extract statistics from output + processed=$(echo "$output" | grep "Processed:" | tail -1 | awk '{print $2}') + skipped=$(echo "$output" | grep "Skipped:" | tail -1 | awk '{print $2}') + errors=$(echo "$output" | grep "Errors:" | tail -1 | awk '{print $2}') + total=$(echo "$output" | grep "Total:" | tail -1 | awk '{print $2}') + + # Set default values if extraction failed + processed=${processed:-0} + skipped=${skipped:-0} + errors=${errors:-0} + total=${total:-0} + + echo "════════════════════════════════════════" + echo "📊 Sync Statistics" + echo "════════════════════════════════════════" + echo "✨ Processed: $processed" + echo "⏭️ Skipped: $skipped" + echo "❌ Errors: $errors" + echo "📦 Total: $total" + echo "════════════════════════════════════════" + echo "" + + # Set outputs for later steps + echo "processed=$processed" >> $GITHUB_OUTPUT + echo "skipped=$skipped" >> $GITHUB_OUTPUT + echo "errors=$errors" >> $GITHUB_OUTPUT + echo "total=$total" >> $GITHUB_OUTPUT + echo "exit_code=$exit_code" >> $GITHUB_OUTPUT + + # Check if manual-tools.ts was modified + if git diff --quiet manual-tools.ts; then + echo "has_changes=false" >> $GITHUB_OUTPUT + echo "ℹ️ No changes to manual-tools.ts" + else + echo "has_changes=true" >> $GITHUB_OUTPUT + echo "✅ manual-tools.ts was modified" + echo "" + echo "📝 Changes preview:" + git diff --stat manual-tools.ts + echo "" + git diff manual-tools.ts | head -50 + fi + + # Determine status for notifications + if [ "$exit_code" -ne 0 ]; then + echo "status_emoji=❌" >> $GITHUB_OUTPUT + echo "status_color=15158332" >> $GITHUB_OUTPUT # Red + echo "status_text=Failed" >> $GITHUB_OUTPUT + elif [ "$errors" -gt 0 ]; then + echo "status_emoji=⚠️" >> $GITHUB_OUTPUT + echo "status_color=16776960" >> $GITHUB_OUTPUT # Yellow + echo "status_text=Completed with errors" >> $GITHUB_OUTPUT + else + echo "status_emoji=✅" >> $GITHUB_OUTPUT + echo "status_color=5763719" >> $GITHUB_OUTPUT # Green + echo "status_text=Success" >> $GITHUB_OUTPUT + fi + + # Exit with the original exit code + exit $exit_code + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + - name: Commit and push changes + if: steps.sync.outputs.has_changes == 'true' + run: | + echo "════════════════════════════════════════" + echo "📝 Committing changes to manual-tools.ts" + echo "════════════════════════════════════════" + echo "" + + # Configure git + git config --local user.email "github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + + # Show what's being committed + echo "📋 Files to commit:" + git status --short + echo "" + + # Commit changes + git add manual-tools.ts + + # Create commit message + COMMIT_MSG="chore: sync ${{ steps.sync.outputs.processed }} new tools from Vercel AI registry + + Added ${{ steps.sync.outputs.processed }} tools from Vercel AI SDK registry: + - Total tools in registry: ${{ steps.sync.outputs.total }} + - Already synced: ${{ steps.sync.outputs.skipped }} + - Newly added: ${{ steps.sync.outputs.processed }} + - Errors: ${{ steps.sync.outputs.errors }} + + 🤖 Automated by GitHub Actions + Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + + git commit -m "$COMMIT_MSG" + + echo "✅ Changes committed" + echo "" + + # Push changes + echo "📤 Pushing to remote..." + git push + + echo "✅ Changes pushed successfully" + echo "════════════════════════════════════════" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Send Discord notification + if: always() + run: | + echo "════════════════════════════════════════" + echo "📢 Sending Discord notification" + echo "════════════════════════════════════════" + + # Build fields array + base_fields='[ + { "name": "📦 Total Tools", "value": "${{ steps.sync.outputs.total }}", "inline": true }, + { "name": "✨ Processed", "value": "${{ steps.sync.outputs.processed }}", "inline": true }, + { "name": "⏭️ Skipped", "value": "${{ steps.sync.outputs.skipped }}", "inline": true }, + { "name": "❌ Errors", "value": "${{ steps.sync.outputs.errors }}", "inline": true }, + { "name": "📝 Changes", "value": "${{ steps.sync.outputs.has_changes }}", "inline": true }, + { "name": "🔗 Run", "value": "[View Logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})", "inline": true } + ]' + + # Add commit info if changes were made + if [ "${{ steps.sync.outputs.has_changes }}" = "true" ]; then + commit_sha=$(git rev-parse HEAD) + commit_url="https://github.com/${{ github.repository }}/commit/${commit_sha}" + additional_fields='[ + { "name": "💾 Commit", "value": "['"${commit_sha:0:7}"']('"$commit_url"')", "inline": false } + ]' + + # Merge fields + all_fields=$(jq -n --argjson base "$base_fields" --argjson additional "$additional_fields" '$base + $additional') + else + all_fields="$base_fields" + fi + + # Create Discord embed + payload=$(jq -n \ + --arg title "${{ steps.sync.outputs.status_emoji }} Vercel AI Registry Sync - ${{ steps.sync.outputs.status_text }}" \ + --argjson color ${{ steps.sync.outputs.status_color }} \ + --argjson fields "$all_fields" \ + --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%S.000Z)" \ + --arg description "Synced Vercel AI SDK tools registry with TPMJS manual tools" \ + ' + { + embeds: [{ + title: $title, + description: $description, + color: $color, + fields: $fields, + timestamp: $timestamp, + footer: { + text: "Vercel AI Registry Sync" + } + }] + }') + + echo "📤 Sending payload to Discord..." + + # Send to Discord + response=$(curl -X POST "${{ secrets.DISCORD_WEBHOOK }}" \ + -H "Content-Type: application/json" \ + -d "$payload" \ + -w "\nHTTP Status: %{http_code}\n" \ + -s) + + echo "$response" + + if echo "$response" | grep -q "HTTP Status: 2"; then + echo "✅ Discord notification sent successfully" + else + echo "⚠️ Discord notification may have failed" + fi + + echo "════════════════════════════════════════" + env: + DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} + + - name: Summary + if: always() + run: | + echo "" + echo "════════════════════════════════════════" + echo "📊 Workflow Summary" + echo "════════════════════════════════════════" + echo "" + echo "Status: ${{ steps.sync.outputs.status_text }}" + echo "Tools Processed: ${{ steps.sync.outputs.processed }}" + echo "Tools Skipped: ${{ steps.sync.outputs.skipped }}" + echo "Errors: ${{ steps.sync.outputs.errors }}" + echo "Total in Registry: ${{ steps.sync.outputs.total }}" + echo "Changes Made: ${{ steps.sync.outputs.has_changes }}" + echo "" + + if [ "${{ steps.sync.outputs.has_changes }}" = "true" ]; then + echo "✅ New tools added to manual-tools.ts and committed" + else + echo "ℹ️ No new tools found - manual-tools.ts is up to date" + fi + + echo "" + echo "════════════════════════════════════════" diff --git a/.vercelignore b/.vercelignore new file mode 100644 index 0000000..028fb4f --- /dev/null +++ b/.vercelignore @@ -0,0 +1,7 @@ +node_modules +.turbo +.next +dist +*.log +.env* +!.env.example diff --git a/2025-BEST-PRACTICES.md b/2025-BEST-PRACTICES.md deleted file mode 100644 index 59676be..0000000 --- a/2025-BEST-PRACTICES.md +++ /dev/null @@ -1,275 +0,0 @@ -# 2025 Best Practices for TPMJS Monorepo - -This document outlines recommendations to make TPMJS a cutting-edge 2025 monorepo optimized for both human and agentic development (Claude Code, Cursor, etc.). - -## High-Impact Additions - -### 1. Agent-First Documentation - -``` -packages/docs/ -├── architecture-decisions/ # ADRs in markdown -├── patterns/ # Common patterns with examples -├── schemas/ # JSON schemas for all data structures -└── examples/ # Working code examples per feature -``` - -**Why:** Claude Code and other agents work better with: -- Explicit decision documentation (ADRs) -- Pattern libraries showing "the right way" -- Machine-readable schemas -- Real working examples to reference - -### 2. Automated Testing Pyramid - -```bash -# Add to package.json scripts -"test:unit": "vitest" # ✅ Already have this -"test:integration": "vitest -c vitest.integration.config.ts" # Add -"test:e2e": "playwright test" # Add -"test:visual": "playwright test --grep @visual" # Add -"test:contracts": "pactum" # Add for API testing -``` - -**Packages to add:** -- `@playwright/test` - E2E testing -- `@playwright/experimental-ct-react` - Component testing -- `pactum` or `msw` integration tests (you have mocks setup) -- `chromatic` or `percy` - Visual regression - -### 3. Type Coverage & Quality Gates - -```json -// Add to root package.json -{ - "scripts": { - "type-check": "tsc --noEmit", - "type-coverage": "type-coverage --at-least 95", - "find-deadcode": "knip", - "check-architecture": "depcruiser --validate" - } -} -``` - -**Add packages:** -- `type-coverage` - Ensure no implicit `any` -- `knip` - Find unused files/exports/dependencies -- `dependency-cruiser` - Enforce architecture rules -- `@total-typescript/ts-reset` - Better built-in types - -### 4. Development Containers - -```json -// .devcontainer/devcontainer.json -{ - "name": "TPMJS Dev", - "dockerComposeFile": "docker-compose.yml", - "service": "dev", - "features": { - "ghcr.io/devcontainers/features/node:1": {}, - "ghcr.io/devcontainers-contrib/features/pnpm:2": {} - }, - "customizations": { - "vscode": { - "extensions": [ - "biomejs.biome", - "bradlc.vscode-tailwindcss", - "lokalise.i18n-ally" - ] - } - } -} -``` - -**Why:** Agents like Claude Code work better when environment is reproducible. This also helps human developers. - -### 5. Code Generation & Scaffolding - -```typescript -// packages/cli/ - Internal dev tool -import { scaffold } from '@tpmjs/cli'; - -// Commands: -pnpm gen:component ButtonGroup -pnpm gen:package @tpmjs/new-package -pnpm gen:app marketing-site -``` - -**Create:** -- `plop` or `hygen` templates -- Component scaffolding (with tests, stories, exports) -- Package scaffolding (with tsconfig, package.json, exports) -- Consistent file structure generation - -**Why:** Agents can use these commands to create new code following your exact patterns. - -### 6. Enhanced Strict Mode TypeScript - -```json -// packages/tsconfig/base.json - Add these -{ - "compilerOptions": { - "exactOptionalPropertyTypes": true, - "noUncheckedIndexedAccess": true, - "noPropertyAccessFromIndexSignature": true, - "allowUnusedLabels": false, - "allowUnreachableCode": false, - "noImplicitOverride": true - } -} -``` - -### 7. Bundle Analysis & Performance - -```json -{ - "scripts": { - "analyze": "turbo run build --filter=@tpmjs/web -- --analyze", - "lighthouse": "lhci autorun", - "bundle-size": "size-limit" - } -} -``` - -**Add:** -- `@next/bundle-analyzer` -- `@lhci/cli` - Lighthouse CI -- `size-limit` - Bundle size tracking in CI - -### 8. Smart Dependency Management - -```json -// .github/renovate.json -{ - "extends": ["config:base"], - "packageRules": [ - { - "matchPackagePatterns": ["*"], - "matchUpdateTypes": ["minor", "patch"], - "groupName": "all non-major dependencies", - "groupSlug": "all-minor-patch" - } - ] -} -``` - -**Use:** Renovate or Dependabot with auto-merge for passing tests - -### 9. API Documentation Generation - -```bash -pnpm add -D -w typedoc typedoc-plugin-markdown -``` - -Auto-generate API docs from TSDoc comments that both humans and agents can read. - -### 10. Schema-First Development - -```typescript -// packages/schemas/ - Central schema definitions -export * from './tool-schema'; -export * from './registry-api-schema'; -export * from './event-schema'; - -// Use Zod for runtime + type generation -// Agents can read schemas to understand contracts -``` - -## Monorepo-Specific Improvements - -### 11. Better Local Development - -```typescript -// turbo.json -{ - "pipeline": { - "dev": { - "cache": false, - "persistent": true, - "dependsOn": ["^build"] - }, - "build": { - "dependsOn": ["^build"], - "outputs": ["dist/**", ".next/**"] - } - } -} -``` - -### 12. Workspace Protocols & Constraints - -```yaml -# .pnpm-workspace.yaml -packages: - - 'apps/*' - - 'packages/*' - -# Add constraints -pnpm-workspace-constraints: - dependencies: - '@tpmjs/ui': 'workspace:*' - '@tpmjs/utils': 'workspace:*' -``` - -## Recommended Final Structure - -``` -. -├── .devcontainer/ # Dev containers config -├── .github/ -│ ├── workflows/ # CI/CD -│ └── renovate.json # Dependency automation -├── apps/ -│ └── web/ -├── packages/ -│ ├── cli/ # ⭐ NEW: Dev tooling -│ ├── schemas/ # ⭐ NEW: Central schemas -│ └── ...existing -├── docs/ -│ ├── adr/ # ⭐ NEW: Architecture decisions -│ ├── patterns/ # ⭐ NEW: Code patterns -│ └── examples/ # ⭐ NEW: Working examples -├── scripts/ -│ ├── scaffold.ts # ⭐ NEW: Code generation -│ └── validate-deps.ts # ⭐ NEW: Architecture validation -├── playwright.config.ts # ⭐ NEW: E2E testing -├── .lighthouserc.json # ⭐ NEW: Performance -└── knip.json # ⭐ NEW: Dead code detection -``` - -## Priority Order for Implementation - -### Phase 1 (Foundation) -1. **Knip + type-coverage** - Catch issues early -2. **Code generation scripts** - Ensure consistency -3. **ADR documentation structure** - Decision tracking - -### Phase 2 (Quality) -4. **E2E testing with Playwright** - Full user flow coverage -5. **Bundle analysis + performance budgets** - Keep app fast -6. **Stricter TypeScript settings** - Catch more bugs at compile time - -### Phase 3 (DX) -7. **Dev containers** - Reproducible environments -8. **API documentation generation** - Auto-generated from code -9. **Renovate automation** - Keep dependencies fresh - -## Benefits for Agent-Driven Development - -1. **Explicit Patterns** - Agents can reference documented patterns instead of guessing -2. **Code Generation** - Consistent scaffolding commands agents can use -3. **Machine-Readable Schemas** - JSON schemas help agents understand data structures -4. **Quality Gates** - Automated checks catch agent mistakes early -5. **Working Examples** - Agents can copy-paste-adapt proven patterns -6. **Architecture Enforcement** - Dependency rules prevent agents from creating invalid imports - -## Next Steps - -Start with the highest ROI items: -1. Install Knip to find dead code -2. Set up code generation for components/packages -3. Create docs/patterns/ with common examples -4. Add stricter TypeScript compiler options -5. Set up Playwright for E2E testing - -These changes will make the codebase more maintainable and significantly improve the experience of working with AI coding agents. diff --git a/API_ROUTES_TIMEOUT_INVESTIGATION.md b/API_ROUTES_TIMEOUT_INVESTIGATION.md deleted file mode 100644 index 575a6bd..0000000 --- a/API_ROUTES_TIMEOUT_INVESTIGATION.md +++ /dev/null @@ -1,932 +0,0 @@ -# API Routes Timeout Issue - Complete Investigation Report - -## Problem Statement - -API routes deployed to Vercel are timing out with no response. The Next.js application pages work perfectly, but all API endpoints at `/api/*` return timeouts or "Redirecting..." messages. - -**Affected URLs:** -- `https://tpmjs.com/api/health` - Returns "Redirecting..." -- `https://tpmjs.com/api/tools` - Returns "Redirecting..." -- `https://tpmjs-1chh44d1u-tpmjs.vercel.app/api/health` - Timeouts (exit code 28) -- `https://tpmjs-1chh44d1u-tpmjs.vercel.app/api/tools` - Timeouts (exit code 28) - -**Working:** -- All page routes work correctly (e.g., `/`, `/tool/[slug]`) -- UI navigation and client-side routing function normally -- Local development API routes work perfectly - -## Environment Details - -### Project Structure -- **Monorepo:** Turborepo setup with pnpm workspaces -- **Framework:** Next.js 16.0.4 (App Router) -- **Node Version:** 24.x (on Vercel) -- **Deployment Platform:** Vercel -- **Custom Domains:** tpmjs.com, www.tpmjs.com - -### Repository Structure -``` -tpmjs/ -├── apps/ -│ └── web/ # Next.js 16 App Router application -│ ├── src/ -│ │ └── app/ -│ │ ├── api/ -│ │ │ ├── health/route.ts -│ │ │ ├── stats/route.ts -│ │ │ ├── tools/ -│ │ │ │ ├── route.ts -│ │ │ │ ├── [id]/route.ts -│ │ │ │ ├── [slug]/route.ts -│ │ │ │ └── validate/route.ts -│ │ │ └── sync/ -│ │ │ ├── changes/route.ts -│ │ │ ├── keyword/route.ts -│ │ │ └── metrics/route.ts -│ │ ├── page.tsx -│ │ └── tool/[slug]/page.tsx -│ ├── next.config.ts -│ └── vercel.json -├── packages/ -│ ├── db/ # Prisma client -│ ├── types/ # Shared TypeScript types -│ ├── utils/ # Utility functions -│ ├── env/ # Environment validation -│ └── ui/ # React component library -├── vercel.json # Root Vercel configuration -└── turbo.json -``` - -## API Route Examples - -### `/apps/web/src/app/api/health/route.ts` -```typescript -import { NextResponse } from 'next/server'; - -export const runtime = 'nodejs'; -export const dynamic = 'force-dynamic'; -export const maxDuration = 60; - -/** - * GET /api/health - * Simple health check endpoint that doesn't touch the database - */ -export async function GET() { - return NextResponse.json({ - status: 'ok', - timestamp: new Date().toISOString(), - env: { - hasDatabase: !!process.env.DATABASE_URL, - nodeEnv: process.env.NODE_ENV, - }, - }); -} -``` - -### `/apps/web/src/app/api/tools/route.ts` -```typescript -import { NextResponse } from 'next/server'; -import { prisma } from '@tpmjs/db/client'; - -export const runtime = 'nodejs'; -export const dynamic = 'force-dynamic'; - -export async function GET(request: Request) { - // ... query string parsing - - const [tools, totalCount] = await Promise.all([ - prisma.tool.findMany({ - where, - orderBy: [ - { qualityScore: 'desc' }, - { npmDownloadsLastMonth: 'desc' }, - { createdAt: 'desc' }, - ], - take: limit, - skip: offset, - }), - prisma.tool.count({ where }), - ]); - - return NextResponse.json({ - data: tools, - pagination: { - page, - limit, - total: totalCount, - totalPages: Math.ceil(totalCount / limit), - }, - }); -} -``` - -## Configuration Files - -### `/apps/web/next.config.ts` (Current) -```typescript -import type { NextConfig } from 'next'; - -const nextConfig: NextConfig = { - transpilePackages: ['@tpmjs/ui', '@tpmjs/utils', '@tpmjs/db', '@tpmjs/types', '@tpmjs/env'], - reactStrictMode: true, -}; - -export default nextConfig; -``` - -### `/apps/web/vercel.json` (Current) -```json -{ - "$schema": "https://openapi.vercel.sh/vercel.json", - "buildCommand": "cd ../.. && pnpm --filter=@tpmjs/web build", - "installCommand": "pnpm install" -} -``` - -### `/vercel.json` (Root) -```json -{ - "$schema": "https://openapi.vercel.sh/vercel.json", - "git": { - "deploymentEnabled": { - "main": true - } - }, - "github": { - "silent": false, - "autoJobCancelation": true - }, - "crons": [ - { - "path": "/api/sync/changes", - "schedule": "*/2 * * * *" - }, - { - "path": "/api/sync/keyword", - "schedule": "*/15 * * * *" - }, - { - "path": "/api/sync/metrics", - "schedule": "0 * * * *" - } - ] -} -``` - -## Local Build Verification - -### Local Build Output Structure -```bash -$ ls -R /Users/ajaxdavis/repos/tpmjs/tpmjs/apps/web/.next/server/app/api/ - -health/ -stats/ -sync/ -tools/ - -/apps/web/.next/server/app/api/health: -route -route.js -route.js.map -route.js.nft.json -route_client-reference-manifest.js - -/apps/web/.next/server/app/api/stats: -route -route.js -route.js.map -route.js.nft.json -route_client-reference-manifest.js - -/apps/web/.next/server/app/api/tools: -[id]/ -[slug]/ -validate/ -route -route.js -route.js.map -route.js.nft.json -route_client-reference-manifest.js -``` - -### Routes Manifest Confirmation -```bash -$ cat /apps/web/.next/routes-manifest.json | jq '.staticRoutes[] | select(.page | contains("api"))' - -{ - "page": "/api/health", - "regex": "^/api/health(?:/)?$", - "routeKeys": {}, - "namedRegex": "^/api/health(?:/)?$" -} -{ - "page": "/api/stats", - "regex": "^/api/stats(?:/)?$", - "routeKeys": {}, - "namedRegex": "^/api/stats(?:/)?$" -} -{ - "page": "/api/sync/changes", - "regex": "^/api/sync/changes(?:/)?$", - "routeKeys": {}, - "namedRegex": "^/api/sync/changes(?:/)?$" -} -{ - "page": "/api/sync/keyword", - "regex": "^/api/sync/keyword(?:/)?$", - "routeKeys": {}, - "namedRegex": "^/api/sync/keyword(?:/)?$" -} -{ - "page": "/api/sync/metrics", - "regex": "^/api/sync/metrics(?:/)?$", - "routeKeys": {}, - "namedRegex": "^/api/sync/metrics(?:/)?$" -} -``` - -### Node File Trace (NFT) Verification -```bash -$ cat /apps/web/.next/server/app/api/health/route.js.nft.json - -{ - "version": 1, - "files": [ - "../../../../../../../node_modules/.pnpm/next@16.0.4_@babel+core@7.28.5_react-dom@19.2.0_react@19.2.0__react@19.2.0/node_modules/next/dist/client/components/app-router-headers.js", - "../../../../../../../node_modules/.pnpm/next@16.0.4_@babel+core@7.28.5_react-dom@19.2.0_react@19.2.0__react@19.2.0/node_modules/next/dist/compiled/@opentelemetry/api/index.js", - // ... many more dependencies - ] -} -``` - -**Conclusion:** API routes build correctly locally with all dependencies properly traced. - -## Vercel Deployment Analysis - -### Deployment Inspection Output -```bash -$ vercel inspect https://tpmjs-1chh44d1u-tpmjs.vercel.app - -General - id dpl_2FHyiTWtBZzvT8EcohwYEZmdb8rb - name tpmjs-web - target production - status ● Ready - url https://tpmjs-1chh44d1u-tpmjs.vercel.app - created Fri Nov 28 2025 20:44:03 GMT+1000 - -Aliases - ╶ https://www.tpmjs.com - ╶ https://tpmjs-web.vercel.app - ╶ https://tpmjs-web-tpmjs.vercel.app - ╶ https://tpmjs-web-git-main-tpmjs.vercel.app - ╶ https://tpmjs.com - -Builds - ┌ . [0ms] - ├── λ tool/[slug] (562.92KB) [iad1] - ├── λ tool/[slug].rsc (562.92KB) [iad1] - ├── λ _global-error (642.48KB) [iad1] - ├── λ _global-error.rsc (642.48KB) [iad1] - ├── λ _global-error.segments/__PAGE__.segment.rsc (642.48KB) [iad1] - └── 56 output items hidden -``` - -**CRITICAL FINDING:** No API routes are listed in the build output. Only pages (`tool/[slug]`, `_global-error`, etc.) appear as serverless functions (`λ`). - -Expected API routes that should appear: -- `λ api/health` -- `λ api/tools` -- `λ api/tools/[id]` -- `λ api/tools/[slug]` -- `λ api/sync/changes` -- etc. - -### Testing Results -```bash -# Direct Vercel URL - Timeouts -$ curl -s -m 10 https://tpmjs-1chh44d1u-tpmjs.vercel.app/api/health -# Exit code 28 (timeout) - -# Custom Domain - Returns "Redirecting..." -$ curl -s -m 10 https://tpmjs.com/api/health -Redirecting... - -# Custom Domain - Returns "Redirecting..." -$ curl -s -m 10 https://tpmjs.com/api/tools -Redirecting... - -# Check redirect headers -$ curl -I https://tpmjs.com -HTTP/2 307 -cache-control: public, max-age=0, must-revalidate -content-type: text/plain -date: Fri, 28 Nov 2025 10:38:12 GMT -location: https://www.tpmjs.com/ -server: Vercel -``` - -## Vercel Configuration Details - -### User-Confirmed Settings -- **Root Directory:** `apps/web` (set in Vercel dashboard) -- **DATABASE_URL:** Configured in Vercel environment variables (Production) -- **Framework Preset:** (Unknown - needs verification) -- **Build Output Directory:** (Unknown - using default `.next`) - -### Project List -```bash -$ vercel project ls | grep -i tpmjs - -tpmjs -- 16h 24.x -v0-tool-registry-page https://tpmjs.com 3d 22.x -``` - -**NOTE:** Two projects exist: -1. `tpmjs` - Current project (Node 24.x) -2. `v0-tool-registry-page` - Also has tpmjs.com domain (Node 22.x) - -This could indicate a domain routing conflict or outdated project. - -## Investigation Timeline & Attempts - -### Attempt 1: Remove Root-Level Redirects -**Hypothesis:** The redirect in `/vercel.json` was intercepting API requests. - -**Original `/vercel.json`:** -```json -{ - "redirects": [ - { - "source": "/:path*", - "has": [ - { - "type": "host", - "value": "www.tpmjs.com" - } - ], - "destination": "https://tpmjs.com/:path*", - "permanent": true - } - ] -} -``` - -**Action:** Removed the `redirects` array from root `/vercel.json`. - -**Result:** ❌ API routes still timeout. Redirect rule was not the root cause. - -**Commit:** `9be3af7 fix(routing): move www redirect from vercel.json to Next.js config` - -### Attempt 2: Move Redirects to Next.js Config -**Hypothesis:** Next.js should handle redirects after routing. - -**Action:** Added `async redirects()` to `apps/web/next.config.ts`: -```typescript -async redirects() { - return [ - { - source: '/:path*', - has: [ - { - type: 'host', - value: 'www.tpmjs.com', - }, - ], - destination: 'https://tpmjs.com/:path*', - permanent: true, - }, - ]; -} -``` - -**Result:** ❌ API routes returned "Redirecting..." instead of executing. Next.js `async redirects()` applies to ALL routes including API routes. - -**Commit:** `9be3af7 fix(routing): move www redirect from vercel.json to Next.js config` - -### Attempt 3: Exclude API Routes from Redirect -**Hypothesis:** Use regex to exclude `/api/*` from redirects. - -**Action:** Modified redirect pattern: -```typescript -async redirects() { - return [ - { - source: '/((?!api).*)', // Negative lookahead to exclude /api/* - has: [ - { - type: 'host', - value: 'www.tpmjs.com', - }, - ], - destination: 'https://tpmjs.com/$1', - permanent: true, - }, - ]; -} -``` - -**Result:** ❌ API routes back to timing out (not redirecting anymore, but still not working). - -**Commit:** `c42bfb6 fix(redirects): exclude API routes from www redirect` - -### Attempt 4: Remove All Redirects -**Hypothesis:** Eliminate redirect loop causing ERR_TOO_MANY_REDIRECTS. - -**Action:** Removed `async redirects()` entirely from `next.config.ts`. - -**Result:** ✅ Redirect loop fixed. ❌ API routes still timeout. - -**Commit:** `b93cd42 fix: remove redirects to resolve redirect loop` - -### Attempt 5: Add Vercel Functions Configuration -**Hypothesis:** Vercel needs explicit configuration to detect API routes. - -**Action:** Added to `apps/web/vercel.json`: -```json -{ - "functions": { - "app/api/**/*.ts": { - "maxDuration": 60 - } - } -} -``` - -**Result:** ❌ No change. API routes still timeout. - -**Commit:** `748ca4d fix(api): configure Vercel functions for API routes with maxDuration` - -### Attempt 6: Add maxDuration to Route Files -**Hypothesis:** Export configuration directly in route handlers. - -**Action:** Added to `apps/web/src/app/api/health/route.ts`: -```typescript -export const maxDuration = 60; -``` - -**Result:** ❌ No change. API routes still timeout. - -**Commit:** `8281f8f fix(build): disable Turbopack for Vercel deployment` - -### Attempt 7: Disable Turbopack -**Hypothesis:** Turbopack (Next.js 16 default) has compatibility issues with Vercel. - -**Action:** Added `--webpack` flag to build command: -```json -{ - "buildCommand": "cd ../.. && turbo build --filter=@tpmjs/web -- --webpack" -} -``` - -**Result:** ❌ Build failed completely. Invalid flag syntax. - -**Commit:** `8281f8f fix(build): disable Turbopack for Vercel deployment` - -### Attempt 8: Simplify Build Command -**Hypothesis:** Use direct pnpm build instead of Turbo wrapper. - -**Action:** Changed to: -```json -{ - "buildCommand": "cd ../.. && pnpm --filter=@tpmjs/web build" -} -``` - -**Result:** ⏳ Pending deployment test. - -**Commit:** `065196d fix(build): simplify Vercel build command` - -## Root Cause Analysis - -### What We Know FOR SURE - -1. ✅ **API routes build correctly locally** - - All 9 API routes compile to `.next/server/app/api/` - - NFT (Node File Trace) files are generated with proper dependencies - - Routes manifest includes all API routes - -2. ✅ **Next.js configuration is correct** - - `export const runtime = 'nodejs'` set correctly - - `export const dynamic = 'force-dynamic'` set correctly - - `transpilePackages` includes all workspace packages - -3. ✅ **Pages deploy and work perfectly** - - `/tool/[slug]` renders correctly - - Homepage loads - - Client-side navigation works - -4. ❌ **API routes are NOT deployed as serverless functions** - - `vercel inspect` shows NO API routes in build output - - Only pages appear as `λ` (lambda) functions - - This is the PRIMARY issue - -5. ❌ **Direct Vercel URLs timeout** - - Not just a custom domain issue - - Affects `*.vercel.app` URLs - - Exit code 28 (timeout) - no response at all - -6. ❌ **Custom domain shows "Redirecting..."** - - Even with all redirects removed from config - - Suggests a redirect at Vercel platform level OR DNS level - - Could be from the `v0-tool-registry-page` project conflict - -### Possible Root Causes - -#### Theory 1: Vercel Project Misconfiguration -**Likelihood:** HIGH - -**Evidence:** -- Two projects with same domain (`tpmjs` and `v0-tool-registry-page`) -- Framework Preset might not be set to "Next.js" -- Root Directory is `apps/web` but Vercel might not be detecting Next.js properly - -**What to Check:** -1. Vercel Dashboard → Project Settings → General - - Framework Preset: Should be "Next.js" - - Root Directory: Should be "apps/web" - - Build Command: Should match vercel.json - - Output Directory: Should be blank (default `.next`) - -2. Vercel Dashboard → Domains - - Check if both projects have tpmjs.com - - Remove domain from `v0-tool-registry-page` if present - -3. Vercel Dashboard → Deployments → Build Logs - - Search for "API" or "route" - - Look for errors about missing functions - - Check if Next.js is detected correctly - -#### Theory 2: Monorepo Detection Issue -**Likelihood:** MEDIUM - -**Evidence:** -- Build command uses `cd ../.. && pnpm --filter=@tpmjs/web build` -- Vercel might not be correctly detecting workspace structure -- `transpilePackages` includes workspace packages - -**What to Check:** -1. Build logs for workspace resolution errors -2. Check if `node_modules` is being created in correct location -3. Verify pnpm workspace configuration - -**Potential Fix:** -Try setting `installCommand` to: -```json -{ - "installCommand": "pnpm install --shamefully-hoist" -} -``` - -#### Theory 3: Next.js 16 + Vercel Incompatibility -**Likelihood:** MEDIUM - -**Evidence:** -- Next.js 16 released recently (November 2024) -- Turbopack is default (might have Vercel issues) -- App Router API routes behave differently than Pages Router - -**What to Check:** -1. Vercel build logs for Next.js version detection -2. Any warnings about incompatible features -3. Check Vercel's Next.js 16 support status - -**Potential Fix:** -Downgrade to Next.js 15.x temporarily to test: -```json -{ - "dependencies": { - "next": "^15.0.0" - } -} -``` - -#### Theory 4: Environment Variable Issue -**Likelihood:** LOW - -**Evidence:** -- DATABASE_URL is configured -- Pages work (they might not need env vars) -- API routes use Prisma (requires DATABASE_URL) - -**What to Check:** -1. Vercel Dashboard → Settings → Environment Variables - - Verify DATABASE_URL is set for Production - - Verify it's not blocked or empty - - Check if other vars are needed - -2. Build logs for Prisma generation errors - -**Potential Fix:** -None - user confirmed DATABASE_URL is set. - -#### Theory 5: Build Output Issue -**Likelihood:** MEDIUM-HIGH - -**Evidence:** -- `vercel inspect` doesn't show API routes -- Only pages are listed as functions -- Build completes successfully (38-41 seconds) - -**What to Check:** -1. Build logs: Does Next.js report building API routes? - - Look for "Route (app)" or "λ" indicators for API routes - - Compare to local build output - -2. Check if Vercel is using correct build output structure - - App Router uses `.next/server/app/` - - Pages Router uses `.next/server/pages/` - -**Potential Fix:** -Try forcing Vercel to recognize the build: -```json -{ - "builds": [ - { - "src": "package.json", - "use": "@vercel/next" - } - ] -} -``` - -(Note: `builds` is legacy, modern Next.js should auto-detect) - -## Recommended Next Steps - -### Immediate Actions (High Priority) - -1. **Check Vercel Project Settings** - - Go to Vercel Dashboard → tpmjs-web project - - Verify Framework Preset is "Next.js" - - Verify Root Directory is `apps/web` - - Screenshot settings for reference - -2. **Review Build Logs** - - Go to latest deployment - - Download complete build logs - - Search for: - - "Route (app)" - should show API routes - - "λ" - should show API functions - - "api" - any mentions - - Errors or warnings - -3. **Check Domain Configuration** - - Verify only ONE project has tpmjs.com domain - - Remove domain from `v0-tool-registry-page` project if present - - Check DNS settings aren't redirecting - -4. **Test Simple API Route** - - Create minimal API route: - ```typescript - // apps/web/src/app/api/test/route.ts - export async function GET() { - return new Response('Hello from API', { status: 200 }); - } - ``` - - Deploy and test - - If this doesn't work, confirms platform issue - -### Investigation Actions (Medium Priority) - -5. **Compare Working vs Non-Working** - - Find a deployment where pages DO work - - Compare build output between page routes and API routes - - Look for differences in how they're compiled - -6. **Test Vercel CLI Deploy** - - Deploy directly via CLI: `vercel --prod` - - Check if behavior differs from Git-based deploy - - Might reveal configuration issues - -7. **Check Vercel Function Logs** - - Even though functions aren't in build output, try: - - `vercel logs --since 1h` - - Look for any API route invocations or errors - -8. **Review Turbo Configuration** - ```bash - # Check turbo.json for Next.js build config - cat /turbo.json - - # Verify build runs correctly locally - pnpm --filter=@tpmjs/web build - ``` - -### Alternative Approaches (If Above Fails) - -9. **Create New Vercel Project** - - Import from Git fresh - - Use identical settings - - Test if fresh project works - -10. **Contact Vercel Support** - - This may be a platform bug with Next.js 16 - - Provide this document as context - - Ask specifically why API routes aren't in build output - -11. **Temporary Workaround** - - Deploy API routes separately (different service) - - Use Vercel proxy to route `/api/*` to separate deployment - - Not ideal but unblocks development - -## Environment Variables Needed - -```bash -# Required for API routes -DATABASE_URL="postgresql://..." - -# Optional (check if needed) -NODE_ENV="production" -NEXT_PUBLIC_* # Any public env vars -``` - -## Build Commands Reference - -### Local Development -```bash -# Install dependencies -pnpm install - -# Generate Prisma client -pnpm --filter=@tpmjs/db db:generate - -# Run development server -pnpm --filter=@tpmjs/web dev - -# Build for production -pnpm --filter=@tpmjs/web build - -# Test build locally -pnpm --filter=@tpmjs/web start -``` - -### Vercel Configuration -**Current:** -```json -{ - "buildCommand": "cd ../.. && pnpm --filter=@tpmjs/web build", - "installCommand": "pnpm install" -} -``` - -**Alternative to try:** -```json -{ - "buildCommand": "cd ../.. && turbo build --filter=@tpmjs/web", - "installCommand": "pnpm install", - "framework": "nextjs" -} -``` - -## Key Files to Review - -1. `/apps/web/next.config.ts` - Next.js configuration -2. `/apps/web/vercel.json` - Vercel app-level config -3. `/vercel.json` - Vercel root config -4. `/turbo.json` - Turborepo configuration -5. `/apps/web/.next/routes-manifest.json` - Route definitions -6. `/apps/web/.next/build-manifest.json` - Build output -7. Vercel build logs (from dashboard) - -## Questions for Vercel Support - -If escalating to Vercel support, ask: - -1. Why are API routes not appearing in the build output (`vercel inspect`) when pages are deploying correctly? - -2. Is there a known issue with Next.js 16 App Router API routes in Turborepo monorepos? - -3. What's the correct way to configure `vercel.json` for a Next.js 16 app in a monorepo with custom build commands? - -4. Could having two projects (`tpmjs` and `v0-tool-registry-page`) with the same domain cause routing issues? - -5. Are there any specific requirements for deploying Next.js 16 API routes that differ from Next.js 15? - -## Related Documentation - -- [Next.js 16 Upgrade Guide](https://nextjs.org/docs/app/guides/upgrading/version-16) -- [Vercel Next.js Deployment](https://vercel.com/docs/frameworks/nextjs) -- [Vercel Functions Configuration](https://vercel.com/docs/functions/configuring-functions) -- [Turborepo with Vercel](https://vercel.com/docs/monorepos/turborepo) -- [Next.js App Router API Routes](https://nextjs.org/docs/app/api-reference/file-conventions/route) - -## Recent Commits Related to This Issue - -``` -065196d fix(build): simplify Vercel build command -8281f8f fix(build): disable Turbopack for Vercel deployment -748ca4d fix(api): configure Vercel functions for API routes with maxDuration -b93cd42 fix: remove redirects to resolve redirect loop -c42bfb6 fix(redirects): exclude API routes from www redirect -9be3af7 fix(routing): move www redirect from vercel.json to Next.js config -a92dfff fix(build): add workspace packages to Next.js transpilePackages -cc6c824 fix(vercel): configure Turborepo monorepo build for apps/web -``` - -## ✅ CONCLUSION - ROOT CAUSE IDENTIFIED - -### The Real Problem - -**Vercel is NOT detecting this project as a Next.js application.** - -When Vercel doesn't detect Next.js, it: -- Uses `@vercel/static-builder` instead of `@vercel/next` -- Treats the deployment as a static site -- Deploys pages (static HTML) successfully -- **Completely drops all App Router API routes** -- Never generates serverless functions for `/api/*` routes - -This explains EVERY symptom: -- ✅ Pages work (they're static files) -- ❌ API routes timeout (they were never deployed) -- ❌ No `λ api/*` in build output (functions don't exist) -- ❌ Direct Vercel URLs timeout (not a DNS issue) -- ❌ "Redirecting..." on custom domain (wrong project owns the domain) - -### Why Vercel Doesn't Detect Next.js - -**1. Wrong Root Directory** -- Vercel project likely has Root Directory set to `.` or empty -- Should be exactly: `apps/web` -- A single character difference breaks Next.js detection - -**2. Wrong Framework Preset** -- When Vercel can't auto-detect Next.js (due to monorepo + wrong root) -- It defaults to Framework Preset = "Other" -- "Other" uses static builder, not Next.js builder - -**3. Domain Conflict** -- Two projects exist: `tpmjs` and `v0-tool-registry-page` -- Both have `tpmjs.com` domain assigned -- Requests randomly route to wrong project -- "Redirecting..." comes from the old project, not your code - -**4. Custom Build Commands** -- Custom build commands can bypass Vercel's auto-detection -- Should let Vercel auto-detect and use default commands - -### Evidence - -**Local build works:** -```bash -$ ls apps/web/.next/server/app/api/ -health/ stats/ sync/ tools/ -# All API routes compile correctly -``` - -**Vercel deployment missing API routes:** -```bash -$ vercel inspect https://tpmjs-1chh44d1u-tpmjs.vercel.app -Builds - ├── λ tool/[slug] # Pages present - ├── λ _global-error # Pages present - └── 56 output items hidden -# NO api/health, NO api/tools, NO api/* at all -``` - -This is **not a Next.js bug, not a Turborepo bug, not a code issue**. -This is **pure Vercel project misconfiguration**. - -### The Fix (4 Steps, 5 Minutes) - -See `VERCEL_FIX_CHECKLIST.md` for detailed instructions. - -**Quick version:** -1. Set Root Directory to `apps/web` (exactly) -2. Set Framework Preset to "Next.js" -3. Remove `tpmjs.com` domain from `v0-tool-registry-page` project -4. Clear custom build commands (let Vercel auto-detect) - -**No code changes needed. Just fix Vercel dashboard settings.** - -### After the Fix - -Next deployment will show: -```bash -$ vercel inspect -Builds - ├── λ api/health - ├── λ api/tools - ├── λ api/sync/changes - ├── λ api/tools/[id] - ├── λ api/tools/[slug] - ├── λ tool/[slug] - ... -``` - -And API routes will work instantly: -```bash -$ curl https://tpmjs.com/api/health -{"status":"ok","timestamp":"..."} -``` - ---- - -## Summary for ChatGPT - -The core issue is that **Vercel is not detecting the project as Next.js** due to incorrect Root Directory, Framework Preset set to "Other", domain conflicts between two projects, and custom build commands bypassing auto-detection. - -When Vercel doesn't detect Next.js, it uses a static site builder which deploys pages but **completely drops all App Router API routes** from the build. This is why pages work but all `/api/*` requests timeout - the API route serverless functions were never created. - -**The fix is purely configuration** - no code changes needed. Set Root Directory to `apps/web`, Framework Preset to "Next.js", remove the domain from the old project, and clear custom build commands. See `VERCEL_FIX_CHECKLIST.md` for step-by-step instructions. diff --git a/CLAUDE.md b/CLAUDE.md index 3c9063c..da7946e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -678,4 +678,423 @@ API timeouts in serverless environments often stem from build configuration issu 4. Use `vercel inspect` to verify lambda deployment 5. Test database performance locally before deploying -The full working implementation is live at [tpmjs.com](https://tpmjs.com). \ No newline at end of file +The full working implementation is live at [tpmjs.com](https://tpmjs.com). + +--- + +## NPM Package Syncing System + +TPMJS.com automatically mirrors npm packages with the `tpmjs` keyword to keep the tool registry up-to-date. This section documents how the syncing system works. + +### Overview + +The sync system uses three automated strategies running on Vercel Cron to discover and update TPMJS tools: + +1. **Changes Feed** - Monitors npm's real-time changes feed for all package updates +2. **Keyword Search** - Actively searches npm for packages with the `tpmjs` keyword +3. **Metrics Sync** - Updates download stats and calculates quality scores + +### Sync Endpoints + +All sync endpoints are located in `apps/web/src/app/api/sync/`: + +#### 1. Changes Feed Sync (`/api/sync/changes`) + +**Purpose:** Monitors npm's changes feed to catch new packages and updates in real-time. + +**Schedule:** Every 2 minutes (`*/2 * * * *`) + +**How it works:** +1. Fetches the last checkpoint sequence number from the database +2. Calls npm's `/_changes` endpoint with `since=` (limit 100 per run) +3. For each changed package, fetches full metadata with `fetchLatestPackageWithMetadata()` +4. Validates that the package has a valid `tpmjs` field using `validateTpmjsField()` +5. Upserts the tool to the database with `discoveryMethod: 'changes-feed'` +6. Updates the checkpoint with the new sequence number for next run + +**Key Features:** +- Uses checkpoints to track progress and avoid reprocessing +- Processes up to 100 changes per run to avoid timeouts +- Logs all sync operations to `syncLog` table +- Requires `Authorization: Bearer ` header + +**Example Response:** +```json +{ + "success": true, + "data": { + "processed": 5, + "skipped": 93, + "errors": 0, + "lastSeq": "12345678", + "pending": 1250, + "durationMs": 2834 + } +} +``` + +#### 2. Keyword Search Sync (`/api/sync/keyword`) + +**Purpose:** Actively searches npm for packages with the `tpmjs` keyword. + +**Schedule:** Every 15 minutes (`*/15 * * * *`) + +**How it works:** +1. Searches npm registry for packages with keyword `tpmjs` (up to 250 results) +2. Fetches full metadata for each package +3. Validates the `tpmjs` field +4. Upserts tools with `discoveryMethod: 'keyword'` +5. Updates checkpoint with last run timestamp + +**Key Features:** +- Catches packages that might be missed by changes feed +- Useful for backfilling existing packages +- Processes up to 250 packages per run + +**Example Response:** +```json +{ + "success": true, + "data": { + "processed": 12, + "skipped": 3, + "errors": 0, + "packagesFound": 15, + "durationMs": 4521 + } +} +``` + +#### 3. Metrics Sync (`/api/sync/metrics`) + +**Purpose:** Updates download statistics and calculates quality scores for all tools. + +**Schedule:** Every hour (`0 * * * *`) + +**How it works:** +1. Fetches all tools from the database +2. For each tool, calls `fetchDownloadStats()` to get last 30 days of downloads +3. Calculates quality score based on: + - Tier (rich = 0.6, minimal = 0.4) + - Downloads (logarithmic scale, max 0.3) + - GitHub stars (logarithmic scale, max 0.1) +4. Updates `npmDownloadsLastMonth` and `qualityScore` fields + +**Quality Score Formula:** +```typescript +function calculateQualityScore(params: { + tier: string; + downloads: number; + githubStars: number; +}): number { + const tierScore = tier === 'rich' ? 0.6 : 0.4; + const downloadsScore = Math.min(0.3, Math.log10(downloads + 1) / 10); + const starsScore = Math.min(0.1, Math.log10(githubStars + 1) / 10); + return Math.min(1.0, tierScore + downloadsScore + starsScore); +} +``` + +**Example Response:** +```json +{ + "success": true, + "data": { + "processed": 25, + "skipped": 0, + "errors": 0, + "totalTools": 25, + "durationMs": 8234 + } +} +``` + +### Automated Sync Configuration + +The sync system can run via two methods: + +#### Option 1: Vercel Cron (Primary) + +Cron jobs are configured in `vercel.json` at the repository root: + +```json +{ + "crons": [ + { + "path": "/api/sync/changes", + "schedule": "*/2 * * * *" + }, + { + "path": "/api/sync/keyword", + "schedule": "*/15 * * * *" + }, + { + "path": "/api/sync/metrics", + "schedule": "0 * * * *" + } + ] +} +``` + +**Pros:** +- Native Vercel integration +- Automatic authentication with `CRON_SECRET` +- Same infrastructure as the app +- No setup required (works automatically on deploy) + +#### Option 2: GitHub Actions (Backup) + +A GitHub Actions workflow (`.github/workflows/sync.yml`) provides redundancy: + +```yaml +name: NPM Package Sync + +on: + schedule: + - cron: '*/2 * * * *' # Changes feed + - cron: '*/15 * * * *' # Keyword search + - cron: '0 * * * *' # Metrics + workflow_dispatch: # Manual trigger +``` + +**Pros:** +- Redundancy if Vercel Cron fails +- Manual trigger via GitHub UI +- Free on GitHub (included in free tier) +- Runs from GitHub's infrastructure + +**Setup:** + +1. Add secrets to GitHub repository settings: + - `VERCEL_PRODUCTION_URL` - Your production URL (e.g., `https://tpmjs.com`) + - `CRON_SECRET` - Same secret used in Vercel environment variables + +2. Enable GitHub Actions in repository settings + +3. The workflow will run automatically on schedule OR manually via: + - GitHub Actions tab → NPM Package Sync → Run workflow → Select sync type + +**Schedule Breakdown:** +- Changes feed: Every 2 minutes (30 times per hour) +- Keyword search: Every 15 minutes (4 times per hour) +- Metrics: Every hour (once per hour) + +**Recommendation:** Use Vercel Cron as primary and GitHub Actions as backup. Both can run simultaneously - the sync endpoints are idempotent. + +### Database Schema + +The sync system uses these Prisma models: + +**`Tool` - The main tool registry:** +```prisma +model Tool { + id String @id @default(cuid()) + npmPackageName String @unique + npmVersion String + npmDownloadsLastMonth Int @default(0) + qualityScore Float? + discoveryMethod String // 'changes-feed' | 'keyword' + tier String // 'minimal' | 'rich' + // ... other fields + + @@index([qualityScore]) + @@index([npmDownloadsLastMonth]) +} +``` + +**`SyncCheckpoint` - Tracks sync progress:** +```prisma +model SyncCheckpoint { + id String @id @default(cuid()) + source String @unique // 'changes-feed' | 'keyword-search' | 'metrics' + checkpoint Json // { lastSeq: string, lastRun: string, ... } +} +``` + +**`SyncLog` - Records all sync operations:** +```prisma +model SyncLog { + id String @id @default(cuid()) + source String + status String // 'success' | 'partial' | 'error' + processed Int + skipped Int + errors Int + message String? + metadata Json? + createdAt DateTime @default(now()) +} +``` + +### Manual Sync Triggers + +To manually trigger a sync (useful for testing or debugging): + +```bash +# Trigger changes feed sync +curl -X POST https://tpmjs.com/api/sync/changes \ + -H "Authorization: Bearer $CRON_SECRET" + +# Trigger keyword search +curl -X POST https://tpmjs.com/api/sync/keyword \ + -H "Authorization: Bearer $CRON_SECRET" + +# Trigger metrics update +curl -X POST https://tpmjs.com/api/sync/metrics \ + -H "Authorization: Bearer $CRON_SECRET" +``` + +**Note:** You need the `CRON_SECRET` environment variable set in Vercel. The endpoints return 401 Unauthorized without it. + +### Monitoring Sync Health + +Check sync logs in the database: + +```typescript +// Get recent sync operations +const recentSyncs = await prisma.syncLog.findMany({ + orderBy: { createdAt: 'desc' }, + take: 20, +}); + +// Check last successful sync for each source +const checkpoints = await prisma.syncCheckpoint.findMany(); +``` + +**Sync Log Example:** +```json +{ + "id": "clx...", + "source": "changes-feed", + "status": "success", + "processed": 5, + "skipped": 93, + "errors": 0, + "message": "Successfully processed 5 packages", + "metadata": { + "durationMs": 2834, + "lastSeq": "12345678", + "pending": 1250 + }, + "createdAt": "2025-11-30T12:00:00Z" +} +``` + +### Error Handling + +All sync endpoints follow this error handling pattern: + +1. **Partial Success:** If some packages fail but others succeed, status is `partial` +2. **Complete Failure:** If the entire sync fails, status is `error` +3. **Error Messages:** First 3 errors are included in the response +4. **Logging:** All operations are logged to `syncLog` regardless of success + +**Example Partial Failure:** +```json +{ + "success": true, + "data": { + "processed": 5, + "skipped": 2, + "errors": 3, + "durationMs": 5234 + } +} +``` + +The sync log will contain: +```json +{ + "status": "partial", + "message": "Processed with errors: Failed to process pkg1: Network timeout; Failed to process pkg2: Invalid tpmjs field; ..." +} +``` + +### Configuration + +Required environment variables in Vercel: + +```bash +# Database connection +DATABASE_URL="postgresql://..." + +# Cron job authentication +CRON_SECRET="your-secret-key" +``` + +**Important:** Vercel Cron automatically adds the `Authorization: Bearer $CRON_SECRET` header when calling the endpoints. No manual configuration needed. + +### Performance Considerations + +**Timeouts:** +- All sync routes have `maxDuration: 300` (5 minutes) +- Changes feed processes max 100 packages per run to avoid timeouts +- Keyword search processes max 250 packages per run +- Metrics sync processes all tools but runs only once per hour + +**Rate Limiting:** +- npm API has rate limits - be cautious when testing manually +- Vercel Cron jobs run from Vercel's infrastructure (different IP than dev) +- Consider implementing exponential backoff for npm API errors + +**Cold Starts:** +- First request to each sync endpoint may be slow due to Prisma initialization +- Subsequent requests are faster with warm Prisma Client +- This is acceptable for background cron jobs + +### Debugging Sync Issues + +**Check if cron jobs are running:** + +```bash +# View recent deployments +vercel ls + +# Check logs for a specific deployment +vercel logs + +# Filter for sync-related logs +vercel logs | grep sync +``` + +**Common issues:** + +1. **"Unauthorized" errors:** Check that `CRON_SECRET` is set in Vercel environment variables +2. **Timeouts:** Reduce batch size in changes feed (currently 100) +3. **Missing packages:** Check `syncLog` for errors during processing +4. **Stale data:** Verify metrics sync is running every hour + +**Test sync locally:** + +```bash +# Start dev server +pnpm dev --filter=@tpmjs/web + +# Trigger sync (requires CRON_SECRET in .env.local) +curl -X POST http://localhost:3000/api/sync/changes \ + -H "Authorization: Bearer $CRON_SECRET" +``` + +### Package Discovery Flow + +Here's how a new TPMJS tool gets discovered: + +1. **Developer publishes package to npm** with `tpmjs` keyword and `tpmjs` field in package.json +2. **Within 2 minutes:** Changes feed sync picks it up from npm's `/_changes` endpoint +3. **Validation:** `validateTpmjsField()` checks that the `tpmjs` field meets requirements +4. **Database Insert:** Tool is upserted with initial data +5. **Within 1 hour:** Metrics sync updates download stats and calculates quality score +6. **Visible on tpmjs.com:** Tool appears in search results and category pages + +**Backup Discovery:** If changes feed misses a package, the keyword search (every 15 minutes) will catch it. + +### Future Improvements + +Potential enhancements to the sync system: + +- [ ] Add webhook endpoint for instant npm package notifications +- [ ] Implement exponential backoff for npm API rate limits +- [ ] Add Slack/Discord notifications for sync failures +- [ ] Create admin dashboard to monitor sync health +- [ ] Support GitHub stars syncing (requires GitHub API integration) +- [ ] Add sync metrics to Vercel Analytics +- [ ] Implement differential sync to reduce database writes \ No newline at end of file diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 914560e..dd4d51c 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -167,7 +167,7 @@ Not needed for Option 1 (Deployment Protection). Add to README.md to show CI status: ```markdown -[![CI](https://github.com/YOUR_ORG/YOUR_REPO/actions/workflows/ci.yml/badge.svg)](https://github.com/YOUR_ORG/YOUR_REPO/actions/workflows/ci.yml) +[![CI](https://github.com/tpmjs/tpmjs/actions/workflows/ci.yml/badge.svg)](https://github.com/tpmjs/tpmjs/actions/workflows/ci.yml) ``` ## Summary diff --git a/HOW_TO_PUBLISH_A_TOOL.md b/HOW_TO_PUBLISH_A_TOOL.md new file mode 100644 index 0000000..9ef0fc4 --- /dev/null +++ b/HOW_TO_PUBLISH_A_TOOL.md @@ -0,0 +1,424 @@ +# How to Publish a TPMJS Tool + +This guide shows you how to create and publish an AI tool that will be automatically discovered and listed on tpmjs.com. + +## Quick Start + +1. Create a new NPM package +2. Add `"tpmjs"` to the `keywords` array in package.json +3. Add a `tpmjs` field with your tool's metadata +4. Publish to NPM +5. Your tool will automatically appear on tpmjs.com within 15 minutes + +## Step-by-Step Guide + +### 1. Create Your NPM Package + +Create a standard NPM package with your tool implementation: + +```bash +mkdir my-awesome-tool +cd my-awesome-tool +npm init -y +``` + +### 2. Add the Required Keyword + +In your `package.json`, add `"tpmjs"` to the keywords array: + +```json +{ + "name": "@yourname/my-awesome-tool", + "version": "1.0.0", + "keywords": ["tpmjs", "ai", "other-keywords"], + ... +} +``` + +**Important:** The `"tpmjs"` keyword is REQUIRED for automatic discovery! + +### 3. Add TPMJS Metadata + +Add a `tpmjs` field to your `package.json` with your tool's metadata. There are three tiers: + +#### Tier 1: Minimal (Required Fields Only) + +The bare minimum to get listed: + +```json +{ + "tpmjs": { + "category": "text-analysis", + "description": "A concise description of what your tool does" + } +} +``` + +**Required fields:** +- `category` - One of: `text-analysis`, `code-generation`, `data-processing`, `image-generation`, `audio-processing`, `search`, `integration`, `other` +- `description` - Clear description of what the tool does (1-3 sentences) + +#### Tier 2: Basic (Recommended) + +Add parameter and return type information: + +```json +{ + "tpmjs": { + "category": "text-analysis", + "description": "Analyzes sentiment in text and returns a score", + "parameters": [ + { + "name": "text", + "type": "string", + "description": "The text to analyze", + "required": true + }, + { + "name": "language", + "type": "string", + "description": "Language code (e.g., 'en', 'es')", + "required": false, + "default": "en" + } + ], + "returns": { + "type": "SentimentResult", + "description": "Object containing score (-1 to 1) and label (positive/negative/neutral)" + } + } +} +``` + +#### Tier 3: Rich (Full Documentation) + +Complete metadata for maximum visibility: + +```json +{ + "tpmjs": { + "category": "text-analysis", + "description": "Advanced sentiment analysis with emotion detection", + "parameters": [ + { + "name": "text", + "type": "string", + "description": "The text to analyze", + "required": true + }, + { + "name": "language", + "type": "string", + "description": "Language code", + "required": false, + "default": "en" + }, + { + "name": "includeEmotions", + "type": "boolean", + "description": "Whether to include emotion breakdown", + "required": false, + "default": false + } + ], + "returns": { + "type": "SentimentResult", + "description": "Object with score, label, and optional emotions array" + }, + "env": [ + { + "name": "SENTIMENT_API_KEY", + "description": "API key for sentiment analysis service", + "required": true + } + ], + "frameworks": ["vercel-ai", "langchain"], + "aiAgent": { + "useCase": "Use this tool when users need to analyze sentiment in text, detect emotions, or understand the tone of customer feedback, reviews, or social media posts.", + "limitations": "Only supports English and Spanish. Maximum 10,000 characters per request.", + "examples": [ + "Analyze customer review sentiment", + "Detect emotions in user feedback", + "Monitor social media sentiment" + ] + } + } +} +``` + +### 4. Implement Your Tool + +Write your tool's implementation. Here's the example from `@tpmjs/createblogpost`: + +```typescript +// src/index.ts +export interface BlogPostOptions { + title: string; + author: string; + content: string; + tags?: string[]; + format?: 'markdown' | 'mdx'; + excerpt?: string; +} + +export interface BlogPost { + frontmatter: { + title: string; + author: string; + date: string; + tags: string[]; + excerpt?: string; + slug: string; + wordCount: number; + readingTime: number; + }; + content: string; + formattedOutput: string; +} + +export async function createBlogPost(options: BlogPostOptions): Promise { + // Your implementation here + const { title, author, content, tags = [], format = 'markdown', excerpt } = options; + + // Validate inputs + if (!title || !author || !content) { + throw new Error('Title, author, and content are required'); + } + + // Process and return result + return { + frontmatter: { /* ... */ }, + content, + formattedOutput: '...' + }; +} + +export default createBlogPost; +``` + +### 5. Build and Publish + +Build your package and publish to NPM: + +```bash +# Build your package +npm run build + +# Publish to NPM +npm publish --access public +``` + +### 6. Verification + +Your tool will be automatically discovered through: + +1. **Keyword Search** - Runs every 15 minutes, searches NPM for `"tpmjs"` +2. **Changes Feed** - Monitors NPM publishes in real-time (every 2 minutes) + +After publishing, your tool should appear on https://tpmjs.com within 15 minutes! + +You can verify by searching: https://tpmjs.com/api/tools?q=yourpackagename + +## Real Example: @tpmjs/createblogpost + +Here's the complete `package.json` from the published example: + +```json +{ + "name": "@tpmjs/createblogpost", + "version": "0.2.0", + "description": "A tool for creating structured blog posts with AI-generated content", + "type": "module", + "keywords": ["tpmjs", "blog", "content", "ai", "writing"], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "files": ["dist"], + "scripts": { + "build": "tsup", + "dev": "tsup --watch", + "type-check": "tsc --noEmit" + }, + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "https://github.com/ajaxdavis/tpmjs.git", + "directory": "packages/tools/createBlogPost" + }, + "homepage": "https://tpmjs.com", + "license": "MIT", + "tpmjs": { + "category": "text-analysis", + "description": "Creates structured blog posts with customizable frontmatter, content sections, and SEO metadata. Supports multiple output formats including Markdown and MDX.", + "parameters": [ + { + "name": "title", + "type": "string", + "description": "The title of the blog post", + "required": true + }, + { + "name": "author", + "type": "string", + "description": "The author of the blog post", + "required": true + }, + { + "name": "content", + "type": "string", + "description": "The main content of the blog post", + "required": true + }, + { + "name": "tags", + "type": "string[]", + "description": "Array of tags for categorization", + "required": false, + "default": [] + }, + { + "name": "format", + "type": "'markdown' | 'mdx'", + "description": "Output format for the blog post", + "required": false, + "default": "markdown" + }, + { + "name": "excerpt", + "type": "string", + "description": "Short excerpt or summary of the post", + "required": false + } + ], + "returns": { + "type": "BlogPost", + "description": "A structured blog post object with frontmatter, content, and metadata including slug, wordCount, readingTime, and formattedOutput" + }, + "frameworks": ["vercel-ai", "langchain"], + "aiAgent": { + "useCase": "Use this tool when users need to generate blog posts, articles, or structured content with proper frontmatter and metadata. Ideal for content management systems, static site generators, and documentation sites.", + "limitations": "Does not include AI content generation - you must provide the content. Only formats and structures existing content.", + "examples": [ + "Create a blog post about TypeScript best practices", + "Generate a tutorial post with code examples", + "Format an article with SEO metadata" + ] + } + } +} +``` + +## Field Reference + +### Required Fields (Tier 1 - Minimal) + +| Field | Type | Description | +|-------|------|-------------| +| `category` | string | Tool category (see categories below) | +| `description` | string | Clear description (1-3 sentences) | + +### Optional Fields (Tier 2 - Basic) + +| Field | Type | Description | +|-------|------|-------------| +| `parameters` | array | Array of parameter objects | +| `returns` | object | Return type information | + +### Optional Fields (Tier 3 - Rich) + +| Field | Type | Description | +|-------|------|-------------| +| `env` | array | Required environment variables | +| `frameworks` | array | Compatible frameworks | +| `aiAgent` | object | AI agent integration info | + +### Categories + +Choose one of these for the `category` field: + +- `text-analysis` - NLP, sentiment, summarization +- `code-generation` - Code generation and transformation +- `data-processing` - Data manipulation and transformation +- `image-generation` - Image creation and editing +- `audio-processing` - Audio/speech processing +- `search` - Search and retrieval +- `integration` - Third-party integrations +- `other` - Anything else + +### Environment Variables + +If your tool requires environment variables: + +```json +"env": [ + { + "name": "OPENAI_API_KEY", + "description": "API key for OpenAI services", + "required": true + }, + { + "name": "API_ENDPOINT", + "description": "Custom API endpoint URL", + "required": false, + "default": "https://api.example.com" + } +] +``` + +## Quality Score + +Your tool gets a quality score based on: + +- **Tier**: Rich (1.0) > Basic (0.5) > Minimal (0.25) +- **Downloads**: Logarithmic scale based on monthly NPM downloads +- **GitHub Stars**: Logarithmic scale based on repository stars + +Higher scores = better visibility on tpmjs.com! + +## Tips for Success + +1. **Use descriptive names** - Make your package name clear and searchable +2. **Complete metadata** - Tier 3 (Rich) tools get 4x the base score +3. **Good documentation** - Add documentation URL to package.json homepage or repository fields +4. **Active maintenance** - Regular updates boost download counts +5. **AI-friendly descriptions** - Write the `aiAgent.useCase` field as guidance for AI agents + +## Testing Locally + +Before publishing, you can validate your `tpmjs` field using the validation schema: + +```bash +# In the tpmjs monorepo +pnpm --filter=@tpmjs/types test +``` + +Or manually check the structure matches the examples above. + +## Troubleshooting + +**Tool not appearing after 15 minutes?** +- Check that you added `"tpmjs"` to keywords +- Verify your `tpmjs` field has required fields (category, description) +- Check the NPM package is public: `npm view yourpackage` + +**Tool showing as "minimal" tier?** +- Add `parameters` and `returns` fields for Basic tier +- Add all Rich tier fields for maximum visibility + +**Want to force a sync?** +You can manually trigger a sync (requires auth): +```bash +curl -X POST "https://tpmjs.com/api/sync/keyword" \ + -H "Authorization: Bearer YOUR_CRON_SECRET" +``` + +## Support + +Questions or issues? +- File an issue: https://github.com/ajaxdavis/tpmjs/issues +- Check the API: https://tpmjs.com/api/tools diff --git a/IMPLEMENTATION_CHECKLIST.md b/IMPLEMENTATION_CHECKLIST.md deleted file mode 100644 index f5afd62..0000000 --- a/IMPLEMENTATION_CHECKLIST.md +++ /dev/null @@ -1,782 +0,0 @@ -# TPMJS NPM Registry - Implementation Checklist - -> **Reference:** See [NPM_MIRROR.md](./NPM_MIRROR.md) for complete architecture details - -**Stack Decision:** Vercel + Neon + Vercel Cron (polling-based sync) - ---- - -## 🎯 Implementation Strategy - -### Architecture Simplification - -**Original Plan (NPM_MIRROR.md):** -- Separate Node.js sync service with persistent changes feed connection -- Self-hosted PostgreSQL -- More complex deployment - -**Revised Plan (This Checklist):** -- All-in-one Next.js app on Vercel -- Neon Postgres (serverless) -- Vercel Cron for sync jobs (polling-based, no persistent connections) -- Simpler, faster to ship - -### Why This Approach? - -✅ **Simpler Infrastructure** -- One deployment (Vercel) -- Managed database (Neon) -- Built-in cron (Vercel Cron) - -✅ **Lower Cost** -- No separate sync service hosting -- Neon free tier generous -- Vercel free/hobby tier sufficient for MVP - -✅ **Same Functionality** -- Poll NPM changes feed every 1-2 minutes (effectively real-time) -- All discovery features from NPM_MIRROR.md maintained -- Quality scoring, validation, etc. all work the same - ---- - -## 📋 Phase 1: Foundation (Week 1) - -**Goal:** Set up database, types, and NPM client - -### 1.1 Database Setup - -- [ ] **Create Neon project** - - Go to https://neon.tech/ - - Create new project - - Save connection string - -- [ ] **Create `packages/db` package** - ```bash - mkdir -p packages/db - cd packages/db - pnpm init - pnpm add prisma @prisma/client - pnpm add -D typescript @types/node - ``` - -- [ ] **Initialize Prisma** - ```bash - npx prisma init - ``` - -- [ ] **Create Prisma schema** - - Copy schema from NPM_MIRROR.md Database Design section - - File: `packages/db/prisma/schema.prisma` - - Include all three models: `Tool`, `SyncCheckpoint`, `SyncLog` - -- [ ] **Add database URL to `.env`** - ```env - DATABASE_URL="postgresql://..." - ``` - -- [ ] **Run first migration** - ```bash - npx prisma migrate dev --name init - npx prisma generate - ``` - -- [ ] **Create Prisma client singleton** - - File: `packages/db/src/client.ts` - ```typescript - import { PrismaClient } from '@prisma/client'; - - const globalForPrisma = globalThis as unknown as { - prisma: PrismaClient | undefined; - }; - - export const prisma = globalForPrisma.prisma ?? new PrismaClient(); - - if (process.env.NODE_ENV !== 'production') { - globalForPrisma.prisma = prisma; - } - ``` - -- [ ] **Export from package** - - File: `packages/db/src/index.ts` - ```typescript - export { prisma } from './client'; - export * from '@prisma/client'; - ``` - -- [ ] **Update package.json** - ```json - { - "name": "@tpmjs/db", - "main": "./src/index.ts", - "types": "./src/index.ts" - } - ``` - -- [ ] **Seed initial sync state** - ```sql - INSERT INTO sync_checkpoints (source, checkpoint) - VALUES - ('changes-feed', '{"sequence": 0}'::jsonb), - ('keyword-search', '{"lastRun": null}'::jsonb), - ('metrics', '{"lastRun": null}'::jsonb) - ON CONFLICT (source) DO NOTHING; - ``` - -**Verification:** -```bash -cd packages/db -npx prisma studio # Should open DB browser with empty tables -``` - ---- - -### 1.2 Types Package - -**Reference:** See NPM_MIRROR.md "The 'tpmjs' Field Schema" section - -- [ ] **Update `packages/types/src/tool.ts`** - - Add `TpmjsMinimalSchema` with Zod - - Add `TpmjsRichSchema` extending minimal - - Export both schemas and inferred types - -- [ ] **Create validation helper** - - File: `packages/types/src/validator.ts` - ```typescript - export function validateTpmjsField(tpmjs: unknown): { - valid: boolean; - tier: 'minimal' | 'rich' | null; - data?: unknown; - errors?: ZodError[]; - } - ``` - -- [ ] **Update exports** - - File: `packages/types/src/index.ts` - - Export all schemas and validators - -**Verification:** -```typescript -import { validateTpmjsField } from '@tpmjs/types'; - -const result = validateTpmjsField({ - category: 'web-scraping', - description: 'Test description that is long enough', - example: 'const x = await tool.test()' -}); - -console.log(result); // Should be { valid: true, tier: 'minimal', ... } -``` - ---- - -### 1.3 NPM Client Package - -**Reference:** See NPM_MIRROR.md "NPM Integration Strategy" section - -- [ ] **Create `packages/npm-client`** - ```bash - mkdir -p packages/npm-client/src - cd packages/npm-client - pnpm init - pnpm add zod - pnpm add -D typescript @types/node - ``` - -- [ ] **Implement changes feed client** - - File: `packages/npm-client/src/changes.ts` - ```typescript - export async function fetchChanges(since: string, limit = 100): Promise<{ - results: Array<{ id: string; seq: string }>; - lastSeq: string; - }> - ``` - - Use endpoint: `https://replicate.npmjs.com/registry/_changes` - - Poll-based (no EventSource needed) - -- [ ] **Implement keyword search** - - File: `packages/npm-client/src/search.ts` - ```typescript - export async function searchByKeyword( - keyword: string, - size = 250, - from = 0 - ): Promise> - ``` - - Use endpoint: `/-/v1/search?text=keywords:${keyword}` - -- [ ] **Implement package metadata fetcher** - - File: `packages/npm-client/src/package.ts` - ```typescript - export async function fetchPackageMetadata(packageName: string): Promise<{ - name: string; - 'dist-tags': { latest: string }; - versions: Record; - time: Record; - } | null> - ``` - - Use endpoint: `https://registry.npmjs.org/${packageName}` - -- [ ] **Implement download stats** - - File: `packages/npm-client/src/stats.ts` - ```typescript - export async function fetchDownloadStats( - packageName: string - ): Promise - ``` - - Use endpoint: `https://api.npmjs.org/downloads/point/last-month/${packageName}` - -- [ ] **Implement GitHub stats** (optional Phase 4) - - File: `packages/npm-client/src/github.ts` - ```typescript - export async function fetchGithubStars( - repoUrl: string - ): Promise - ``` - -- [ ] **Add rate limiting helper** - - File: `packages/npm-client/src/rate-limiter.ts` - - Simple delay between requests - - Exponential backoff on 429 - -- [ ] **Export all functions** - - File: `packages/npm-client/src/index.ts` - -**Verification:** -```typescript -import { fetchPackageMetadata } from '@tpmjs/npm-client'; - -const pkg = await fetchPackageMetadata('express'); -console.log(pkg?.name); // Should print 'express' -``` - ---- - -## 📋 Phase 2: Core API Routes (Week 2) - -**Goal:** Build public API for searching/listing tools - -### 2.1 Tool Search/List API - -**Reference:** See NPM_MIRROR.md "API Routes" section - -- [ ] **Create `apps/web/src/app/api/tools/route.ts`** - - Implement `GET` handler - - Query params: `q`, `category`, `official`, `limit`, `offset` - - Use Prisma to query `tools` table - - Return paginated results with metadata - -- [ ] **Add full-text search** - - Use Postgres `ts_vector` for search - - Or simple `ILIKE` for MVP - - Search across: `npmPackageName`, `description`, `tags` - -- [ ] **Add filtering** - - By `category` - - By `isOfficial` - - By `tier` (optional) - -- [ ] **Add sorting** - - Default: `qualityScore DESC`, `npmDownloadsLastMonth DESC` - - Optional: `createdAt DESC`, `npmPackageName ASC` - -**Verification:** -```bash -curl "http://localhost:3001/api/tools?q=web&limit=5" -# Should return JSON with tools array and pagination -``` - ---- - -### 2.2 Tool Detail API - -- [ ] **Create `apps/web/src/app/api/tools/[id]/route.ts`** - - Implement `GET` handler - - Accept ID or package name - - Return full tool details - -**Verification:** -```bash -curl "http://localhost:3001/api/tools/1" -# Should return single tool object -``` - ---- - -### 2.3 Validation API - -- [ ] **Create `apps/web/src/app/api/tools/validate/route.ts`** - - Implement `POST` handler - - Accept JSON body with `tpmjs` field - - Use `@tpmjs/types` validator - - Return validation result with errors - -**Verification:** -```bash -curl -X POST http://localhost:3001/api/tools/validate \ - -H "Content-Type: application/json" \ - -d '{"category":"web-scraping","description":"Test tool for validation","example":"const x = await tool.test()"}' -# Should return { valid: true, tier: "minimal" } -``` - ---- - -### 2.4 Stats API - -- [ ] **Create `apps/web/src/app/api/stats/route.ts`** - - Implement `GET` handler - - Aggregate counts by category - - Total tools, official tools, etc. - -**Verification:** -```bash -curl "http://localhost:3001/api/stats" -# Should return { totalTools: 0, officialTools: 0, categories: {} } -``` - ---- - -## 📋 Phase 3: Sync Workers (Week 2-3) - -**Goal:** Implement automatic NPM package discovery - -**Reference:** See NPM_MIRROR.md "NPM Integration Strategy" section - -### 3.1 Changes Feed Sync - -- [ ] **Create `apps/web/src/app/api/sync/changes/route.ts`** - -- [ ] **Implement POST handler** - ```typescript - export async function POST(request: Request) { - // 1. Verify CRON_SECRET header - // 2. Get last sequence from sync_checkpoints - // 3. Fetch changes from NPM (limit 100-500) - // 4. For each change: - // - Fetch package metadata - // - Check for tpmjs field - // - Validate with @tpmjs/types - // - Upsert to tools table - // - Log to sync_logs - // 5. Update checkpoint with new sequence - // 6. Return summary (processed, skipped, errors) - } - ``` - -- [ ] **Add secret protection** - ```typescript - const secret = request.headers.get('x-cron-secret'); - if (secret !== process.env.CRON_SECRET) { - return new Response('Unauthorized', { status: 401 }); - } - ``` - -- [ ] **Add timeout protection** - - Limit processing to 50 packages per run - - Or 50 seconds max execution time - - Save checkpoint frequently - -- [ ] **Add error handling** - - Try/catch around each package - - Log errors to `sync_logs` - - Continue processing other packages - -**Verification:** -```bash -curl -X POST http://localhost:3001/api/sync/changes \ - -H "x-cron-secret: your-secret" -# Should process changes and return summary -``` - ---- - -### 3.2 Keyword Search Sync - -- [ ] **Create `apps/web/src/app/api/sync/keyword/route.ts`** - -- [ ] **Implement POST handler** - ```typescript - export async function POST(request: Request) { - // 1. Verify CRON_SECRET header - // 2. Search NPM for keyword 'tpmjs-tool' - // 3. For each result: - // - Fetch package metadata - // - Validate tpmjs field - // - Upsert with isOfficial=true - // - Log to sync_logs - // 4. Update checkpoint - // 5. Return summary - } - ``` - -- [ ] **Handle pagination** - - NPM allows `size` up to 250 - - May need multiple requests for all results - -**Verification:** -```bash -curl -X POST http://localhost:3001/api/sync/keyword \ - -H "x-cron-secret: your-secret" -# Should search and process keyword packages -``` - ---- - -### 3.3 Metrics Sync (Phase 4) - -- [ ] **Create `apps/web/src/app/api/sync/metrics/route.ts`** - -- [ ] **Implement POST handler** - ```typescript - export async function POST(request: Request) { - // 1. Verify CRON_SECRET - // 2. Select tools to update (recent, popular, or sample) - // 3. For each tool: - // - Fetch NPM download stats - // - Fetch GitHub stars (if repo exists) - // - Calculate quality score - // - Update tools table - // 4. Update checkpoint - // 5. Return summary - } - ``` - -**Verification:** -```bash -curl -X POST http://localhost:3001/api/sync/metrics \ - -H "x-cron-secret: your-secret" -# Should update metrics for tools -``` - ---- - -### 3.4 Vercel Cron Configuration - -- [ ] **Add to `vercel.json`** - ```json - { - "crons": [ - { - "path": "/api/sync/changes", - "schedule": "*/2 * * * *" - }, - { - "path": "/api/sync/keyword", - "schedule": "*/15 * * * *" - }, - { - "path": "/api/sync/metrics", - "schedule": "0 * * * *" - } - ] - } - ``` - -- [ ] **Set up environment variables in Vercel** - - `DATABASE_URL` - Neon connection string - - `CRON_SECRET` - Generate random secret - - `NPM_REGISTRY_URL` - https://registry.npmjs.org - - `NPM_CHANGES_URL` - https://replicate.npmjs.com/registry - ---- - -## 📋 Phase 4: Frontend Integration (Week 3) - -**Goal:** Replace mock data with real API calls - -### 4.1 Update Tool Listing Page - -- [ ] **Update `apps/web/src/app/tools/page.tsx`** - - Remove mock data import - - Fetch from `/api/tools` - - Add loading state - - Add error handling - -- [ ] **Add search functionality** - - Search input component - - Debounced API calls - - Update URL with search params - -- [ ] **Add category filter** - - Category dropdown/pills - - Filter API calls by category - -- [ ] **Add pagination** - - Next/previous buttons - - Or infinite scroll - -**Verification:** -- Visit http://localhost:3001/tools -- Should show real tools from database -- Search should work -- Filters should work - ---- - -### 4.2 Update Tool Detail Page - -- [ ] **Update `apps/web/src/app/tools/[id]/page.tsx`** - - Fetch from `/api/tools/[id]` - - Display all tool metadata - - Show rich tier fields if available - -- [ ] **Add install instructions** - - npm install command - - Usage example from `tpmjs.example` - -- [ ] **Add links** - - NPM package page - - GitHub repository - - Documentation - - Playground (if available) - -**Verification:** -- Visit http://localhost:3001/tools/some-package -- Should show full tool details - ---- - -### 4.3 Update Homepage - -**Reference:** See NPM_MIRROR.md for stats display - -- [ ] **Update stats in hero section** - - Fetch from `/api/stats` - - Show real tool count - - Show category breakdown - -- [ ] **Update live metrics** - - Real download counts - - Real tool counts - - Update frequently (client-side polling or static) - -**Verification:** -- Visit http://localhost:3001 -- Stats should be real, not mock - ---- - -## 📋 Phase 5: Testing & Polish (Week 4) - -### 5.1 Create Test Packages - -- [ ] **Publish 3-5 real NPM packages with `tpmjs` field** - - At least one with minimal tier - - At least one with rich tier - - Use `tpmjs-tool` keyword for official listing - -- [ ] **Verify automatic discovery** - - Wait for next sync run - - Check they appear in database - - Check they appear on website - ---- - -### 5.2 Documentation - -- [ ] **Create docs section** - - `apps/web/src/app/docs/page.tsx` - - Getting started guide - - Schema reference - - Examples - -- [ ] **Add validation playground** - - `apps/web/src/app/docs/validate/page.tsx` - - Form to test `tpmjs` field - - Real-time validation feedback - - Uses `/api/tools/validate` - ---- - -### 5.3 CLI Tool (Optional) - -- [ ] **Create `packages/cli`** - - Command: `tpmjs validate` - - Reads local `package.json` - - Validates `tpmjs` field - - Calls `/api/tools/validate` - ---- - -### 5.4 Monitoring - -- [ ] **Add health endpoint** - - `apps/web/src/app/api/health/route.ts` - - Check database connectivity - - Check sync status (last run times) - -- [ ] **Set up uptime monitoring** - - Use UptimeRobot or Better Stack - - Monitor `/api/health` - - Alert if down or sync stale - -- [ ] **Add error tracking** - - Set up Sentry for Next.js - - Track API errors - - Track sync errors - -**Verification:** -```bash -curl http://localhost:3001/api/health -# Should return { status: "ok", db: "ok", sync: { ... } } -``` - ---- - -## 📋 Phase 6: Launch (Week 5) - -### 6.1 Pre-Launch Checklist - -- [ ] **Database** - - ✓ Prisma schema deployed - - ✓ Indexes created - - ✓ Backups enabled in Neon - -- [ ] **Environment Variables** - - ✓ All secrets in Vercel - - ✓ `CRON_SECRET` set - - ✓ `DATABASE_URL` set - -- [ ] **API Routes** - - ✓ All endpoints working - - ✓ Rate limiting added (optional) - - ✓ Error handling complete - -- [ ] **Sync Workers** - - ✓ Changes feed running every 2 min - - ✓ Keyword search running every 15 min - - ✓ Checkpoints updating correctly - -- [ ] **Frontend** - - ✓ All pages loading real data - - ✓ Search working - - ✓ Mobile responsive - -- [ ] **Monitoring** - - ✓ Health check endpoint live - - ✓ Uptime monitoring active - - ✓ Error tracking active - ---- - -### 6.2 Launch Steps - -- [ ] **Deploy to production** - ```bash - git push origin main - # Vercel auto-deploys - ``` - -- [ ] **Verify deployment** - - Check all pages load - - Check API endpoints work - - Check cron jobs run - -- [ ] **Publish announcement** - - Tweet/post about TPMJS - - Explain how to add `tpmjs` field - - Share validation endpoint - -- [ ] **Monitor for 24 hours** - - Watch error logs - - Check sync is working - - Fix any issues - ---- - -## 📋 Phase 7: Post-Launch (Ongoing) - -### Enhancements - -- [ ] **Semantic search** - - Add embeddings to tools table - - Use OpenAI/Cohere for semantic search - -- [ ] **Usage analytics** - - Track tool views - - Track search queries - - Popular tools widget - -- [ ] **Tool recommendations** - - "Similar tools" section - - "You might also like" - -- [ ] **GitHub Actions** - - Validate `tpmjs` field in CI - - Auto-comment validation results - -- [ ] **NPM webhooks** - - Listen for package updates - - Immediate sync (instead of polling) - ---- - -## 🎯 Success Criteria - -Check these metrics after launch: - -### Week 1 -- [ ] 10+ official tools listed -- [ ] All sync jobs running successfully -- [ ] Zero API errors - -### Month 1 -- [ ] 50+ official tools -- [ ] 5+ package authors using TPMJS -- [ ] <200ms API response time (p95) - -### Month 3 -- [ ] 200+ tools -- [ ] 20+ package authors -- [ ] Community contributions - -### Month 6 -- [ ] 1000+ tools -- [ ] 50+ active package authors -- [ ] Established as go-to AI tool registry - ---- - -## 🔄 Ongoing Maintenance - -Weekly: -- [ ] Check sync logs for errors -- [ ] Review new tools for quality -- [ ] Update documentation - -Monthly: -- [ ] Database optimization (indexes, vacuum) -- [ ] Review and adjust quality scoring -- [ ] Update NPM_MIRROR.md with learnings - ---- - -## 📚 Key Documents - -**Read frequently during implementation:** - -1. **NPM_MIRROR.md** - Complete architecture reference - - Database schema - - API specifications - - Validation rules - - Quality scoring - - All examples - -2. **This checklist** - Implementation order and verification steps - -3. **Plan file** - `.claude/plans/goofy-inventing-stearns.md` - Detailed planning notes - ---- - -## 🚀 Ready to Build - -This checklist is your complete implementation guide. Work through it phase by phase, checking off items as you go. - -**Start with Phase 1, Step 1.1** and work sequentially. Each step has verification instructions to ensure it's working before moving on. - -Good luck! 🎉 diff --git a/LAUNCH_REVIEW.md b/LAUNCH_REVIEW.md new file mode 100644 index 0000000..38d77b8 --- /dev/null +++ b/LAUNCH_REVIEW.md @@ -0,0 +1,331 @@ +# TPMJS Launch Review & Checklist + +**STATUS: COMPLETED** - All critical issues have been fixed. + +A comprehensive review of all public-facing content for Hacker News launch readiness. + +--- + +## Executive Summary + +**Overall Readiness: 7/10 - Needs Work Before Launch** + +The website has excellent technical content and professional design, but fails the "5-second test" - a first-time visitor cannot quickly understand what TPMJS is or why they need it. The documentation is strong for existing users but assumes too much prior knowledge about AI agents and tooling. + +### Critical Issues (Must Fix) +1. **Landing page doesn't explain what TPMJS is** - Hero section uses jargon without definition +2. **"Tool" vs "Package" never defined** - Core concepts assumed, not explained +3. **Knowledge gaps** - Assumes familiarity with AI agents, Zod, semantic search +4. **Category inconsistency** - HOW_TO_PUBLISH and NPM_MIRROR have different category lists +5. **NPM_MIRROR.md conflicts with other docs** - Appears outdated, creates confusion + +### What's Working Well +- Publishing guide (HOW_TO_PUBLISH_A_TOOL.md) is excellent +- How It Works page has great technical depth +- Developer testimonials are concrete with real metrics +- No obvious AI-generated language on website +- Code examples are practical and well-placed + +--- + +## The 5-Second Test: FAILED + +**Question:** Can a developer understand what TPMJS is within 5 seconds of landing on the homepage? + +**Answer:** No. + +### What They See First +``` +TOOL REGISTRY FOR AI AGENTS +Discover, share, and integrate tools that give your agents superpowers +``` + +### What's Missing +- What is a "tool" in this context? +- What is an "AI agent"? +- Why would I use this vs npm directly? +- Is this a package manager? A marketplace? An SDK? + +### The "Aha Moment" is Unclear +A visitor still doesn't know: +- WHO should use TPMJS (tool builders? agent developers? both?) +- WHEN they would use it (at development time? runtime?) +- HOW it differs from regular npm packages +- WHY they can't just install packages normally + +--- + +## Page-by-Page Clarity Ratings + +| Page | Clarity | Human Feel | Issues | +|------|---------|------------|--------| +| **Landing Page** | 5/10 | Yes | No 5-second explanation, jargon-heavy | +| **Hero Section** | 3/10 | Yes | "Tool registry" undefined, circular language | +| **Problem Section** | 7/10 | Yes | Best section - concrete pain points | +| **Vision Section** | 5/10 | Yes | "Semantic search" unexplained | +| **Developer Stories** | 7/10 | Yes | Good metrics, but code unexplained | +| **Publish Section** | 6/10 | Yes | Assumes visitor is a tool builder | +| **How It Works** | 9/10 | Excellent | Minor density issues | +| **FAQ** | 8/10 | Yes | Missing some common questions | +| **Publish Guide** | 8.5/10 | Yes | Tier system could be clearer upfront | +| **Spec Page** | 8.5/10 | Yes | Assumes Zod/AI SDK knowledge | +| **Docs Page** | 9/10 | Excellent | Overwhelming length | +| **SDK Page** | 8.5/10 | Yes | Assumes Vercel AI SDK familiarity | +| **Privacy** | 8/10 | Yes | Hardcoded email address | +| **Terms** | 8/10 | Yes | Hardcoded date | + +--- + +## Documentation Clarity Ratings + +| Document | Clarity | Necessary | Critical Issues | +|----------|---------|-----------|-----------------| +| README.md | 8/10 | YES | Missing "what is TPMJS" explanation | +| HOW_TO_PUBLISH_A_TOOL.md | 9/10 | YES | Minor - excellent overall | +| DEPLOYMENT.md | 8/10 | YES | Confusing exit code explanation | +| QUALITY-GATES.md | 7/10 | OPTIONAL | Could merge into README | +| MANUAL_TOOLS.md | 8.5/10 | YES | Good for maintainers | +| NPM_MIRROR.md | 6.5/10 | **REMOVE** | **Conflicts with other docs, appears outdated** | + +--- + +## Knowledge Gaps (Things Visitors Won't Understand) + +### Not Explained Anywhere +1. **What is an "AI Agent"?** - The entire site assumes you know this +2. **What is a "Tool" vs a "Package"?** - Used interchangeably, never defined +3. **Why semantic search matters** - Just says "semantic" without explaining benefit +4. **What frameworks are supported** - Mentioned in FAQ but not prominently +5. **The Package → Tool relationship** - Can one package have multiple tools? + +### Assumed Technical Knowledge +- Zod schemas (used throughout, never introduced) +- AI SDK tool format (referenced as "standard" but what standard?) +- esm.sh and Deno sandboxing (mentioned in How It Works) +- BM25 ranking algorithm (mentioned in docs) + +### Missing Use Cases +- "Use TPMJS when..." section doesn't exist +- No comparison to alternatives (why not just npm?) +- No "before/after" showing the problem solved + +--- + +## Human-Written Assessment + +### Reads Like Human: YES ✓ +- Developer stories use specific metrics ("500 lines to 3") +- Technical explanations show genuine understanding +- Problem section addresses real pain points +- No buzzword soup or meaningless marketing phrases + +### Minor AI-Sounding Phrases Found +| Location | Phrase | Issue | +|----------|--------|-------| +| NPM_MIRROR.md:7 | "automated NPM-integrated registry" | Marketing speak | +| NPM_MIRROR.md:27 | "✨ Listed automatically" | Emoji in technical doc | +| NPM_MIRROR.md:500 | "Built with ❤️" | Remove emoji | +| HOW_TO_PUBLISH:389 | "AI-friendly descriptions" | Vague - what makes it "AI-friendly"? | +| Vision Section | "gives agents superpowers" | Metaphor without substance | + +--- + +## Critical Inconsistencies Found + +### Category Lists Don't Match +**HOW_TO_PUBLISH_A_TOOL.md says:** +``` +text-analysis, code-generation, data-processing, +image-generation, audio-processing, search, integration, other +``` + +**NPM_MIRROR.md says:** +``` +web-scraping, data-processing, file-operations, communication, +database, api-integration, image-processing, text-analysis, +automation, ai-ml, security, monitoring +``` + +**These are completely different!** Which is correct? + +### Quality Score Formula Conflicts +- HOW_TO_PUBLISH: "Tier: Rich (1.0) > Basic (0.5) > Minimal (0.25)" +- MANUAL_TOOLS: "Rich tier tools get 4x quality score multiplier" +- NPM_MIRROR: Different formula entirely + +### Field Names Inconsistent +- `exportName` used in MANUAL_TOOLS but not in HOW_TO_PUBLISH +- Deprecated fields (`parameters`, `returns`) mentioned but unclear when deprecated + +--- + +## Hardcoded Values to Fix + +| File | Issue | Line | +|------|-------|------| +| FAQ, Privacy, Terms | `thomasalwyndavis@gmail.com` hardcoded | Multiple | +| Privacy, Terms | Date "December 14, 2025" hardcoded | Multiple | +| Changelog page | Package list hardcoded in code | ~95-110 | +| Developer Stories | Fictional company names (Support.ai, DocFlow) | homePageData.ts | + +--- + +## Launch Checklist + +### Must Fix Before Launch (Blocking) - ALL DONE ✓ + +- [x] **Rewrite hero section** to explain TPMJS in one sentence + - Current: "TOOL REGISTRY FOR AI AGENTS" + - Suggested: "TPMJS lets AI agents discover and use npm packages as tools at runtime. Publish once to npm, get discovered automatically." + +- [x] **Add "What is TPMJS?" section** to landing page + - Define: What is an AI agent? + - Define: What is a "tool" in this context? + - Explain: Why not just use npm directly? + - Show: 3-step "how it works" visual + +- [x] **Reconcile category lists** between docs (deleted NPM_MIRROR.md) + - Pick one canonical list + - Update all docs to match + - Add categories to types package + +- [x] **Delete or archive NPM_MIRROR.md** (deleted) + - Conflicts with HOW_TO_PUBLISH + - Appears to be old design doc, not current state + - Move to `/docs/internal/` if historical value + +- [x] **Fix hardcoded values** (emails → hello@tpmjs.com, dates → December 2024) + - Email addresses → environment variable + - Dates → dynamic or remove + - Package lists → generated from filesystem + +### Should Fix (High Priority) - MOSTLY DONE + +- [x] **Add "Use TPMJS when..." section** to landing page (covered in "What is TPMJS?" section) + - List concrete scenarios: "Building a chatbot that needs web access" + - "Agent that processes different file formats" + - "Tool that should be discoverable by other agents" + +- [x] **Explain Package vs Tool distinction** (covered in "What is TPMJS?" section) + - Add glossary or definitions section + - Clarify: 1 package can have N tools + +- [x] **Add framework compatibility section** (mentioned in hero and publish sections) + - Which AI frameworks work with TPMJS? + - Are there adapters needed? + - Show code for each framework + +- [ ] **Simplify developer stories code** + - Current code snippet unexplained: + ```js + const agent = new Agent({ tools: await tpmjs.search(...) }) + ``` + - Add: Where does `Agent` come from? What's happening here? + +- [x] **Add README context** (completely rewritten with clear explanation) + - What is TPMJS for? + - Link to tpmjs.com + - Explain discovery mechanism + +### Nice to Have (Post-Launch) + +- [ ] Add video walkthrough (30-60 seconds) +- [ ] Interactive playground link from homepage +- [ ] "Compare to alternatives" section +- [ ] Case studies with real company names +- [ ] Quick links sidebar for docs page +- [ ] Status badges for each quality gate + +--- + +## Recommended Hero Section Rewrite + +### Current +``` +TOOL REGISTRY FOR AI AGENTS +Discover, share, and integrate tools that give your agents superpowers +The registry for AI tools +``` + +### Suggested +``` +MAKE YOUR AI AGENT SMARTER +TPMJS connects your AI agent to 2,500+ npm packages at runtime. +No config files. No manual imports. Just describe what you need. + +"Find me a tool that can scrape websites" → Your agent gets web-scraper +"I need to process markdown" → Your agent gets markdown-formatter + +Publish your npm package → It's discoverable by every AI agent in 15 minutes. +``` + +This version: +- Explains what it DOES (connects agents to npm packages) +- Shows HOW it works (natural language → tool) +- States the VALUE (no config, automatic discovery) +- Gives concrete examples + +--- + +## Recommended "What is TPMJS?" Section + +Add after hero, before featured tools: + +```markdown +## What is TPMJS? + +**The Problem:** AI agents need tools (web scraping, file processing, API calls) +but developers must manually configure each one. As the ecosystem grows, +this becomes unmanageable. + +**The Solution:** TPMJS is a registry that automatically discovers npm packages +designed for AI agents. Agents can search for tools by description and load them +at runtime. + +**For Tool Builders:** Add `tpmjs` keyword to your package.json. +Your tool appears on tpmjs.com within 15 minutes. + +**For Agent Developers:** Use semantic search to find tools: +```javascript +import { searchRegistry } from '@tpmjs/sdk'; +const tools = await searchRegistry('send emails and slack messages'); +// Returns: email-sender, slack-notifier, ... +``` + +**One registry. Thousands of tools. Zero configuration.** +``` + +--- + +## Final Assessment + +### Ready for Launch? +**Not yet.** The core product is solid but messaging fails first-time visitors. + +### Estimated Fixes +- Hero rewrite: 30 minutes +- "What is TPMJS?" section: 1 hour +- Category reconciliation: 1 hour +- Hardcoded values: 30 minutes +- README updates: 30 minutes +- NPM_MIRROR cleanup: 15 minutes + +**Total: ~4 hours of work** + +### After Fixes +The site will be launch-ready. The technical content is excellent - it just needs a better front door. + +--- + +## Appendix: Positive Highlights + +Things that are already great and should NOT change: + +1. **How It Works page** - Excellent technical depth, clear structure +2. **Publishing guide** - Best-in-class documentation, real examples +3. **Problem section** - Concrete pain points, relatable issues +4. **Spec page** - Clear field reference, good validation info +5. **SDK documentation** - Quick start is excellent +6. **Code examples throughout** - Practical, copy-pasteable +7. **Visual design** - Clean, professional, developer-focused +8. **Quality scoring explanation** - Transparent, well-documented diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7d72b03 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024-2025 TPMJS + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MANUAL_TOOLS.md b/MANUAL_TOOLS.md new file mode 100644 index 0000000..fe27e30 --- /dev/null +++ b/MANUAL_TOOLS.md @@ -0,0 +1,293 @@ +# Manual Tools Registry + +## Overview + +This system allows TPMJS to include high-quality tools that don't follow the standard `tpmjs` field specification in their package.json. These tools are manually curated and synced to the database. + +## Why Manual Tools? + +Some excellent tools (like Vercel's code execution, Exa search, Firecrawl, etc.) don't include the `tpmjs` field in their package.json. Rather than wait for these package maintainers to adopt the spec, we manually curate metadata for these tools. + +## Architecture + +### Files + +1. **`manual-tools.ts`** - The registry of manually curated tools +2. **`sync-manual-tools.ts`** - Script to sync manual tools to database +3. **`MANUAL_TOOLS.md`** - This documentation + +### How It Works + +1. **Manual Tool Registry** (`manual-tools.ts`) + - Exports a `manualTools` array with metadata for each tool + - Each entry includes npm package name, export name, category, description, parameters, etc. + - Follows the same schema as the standard `tpmjs` field + +2. **Sync Script** (`sync-manual-tools.ts`) + - Fetches latest package metadata from npm + - Combines npm metadata with manual metadata + - Upserts Package + Tool records to database + - Marks tools with `discoveryMethod: 'manual'` + +3. **Database Storage** + - Manual tools stored in same `packages` and `tools` tables as auto-discovered tools + - No special handling needed in API or frontend + - `discoveryMethod: 'manual'` field distinguishes them + +## Adding a New Manual Tool + +### Step 1: Add to Registry + +Edit `manual-tools.ts` and add a new entry: + +```typescript +{ + npmPackageName: 'example-package', + category: 'search', + frameworks: ['vercel-ai'], + exportName: 'exampleTool', + description: 'A clear, concise description of what this tool does', + + // Optional but recommended for 'rich' tier + parameters: [ + { + name: 'query', + type: 'string', + description: 'The search query', + required: true, + }, + ], + + returns: { + type: 'array', + description: 'Array of search results', + }, + + aiAgent: { + useCase: 'Use when you need to search for X', + limitations: 'Rate limits apply', + examples: [ + 'Search for current news', + 'Find specific information', + ], + }, + + // Environment variables + env: [ + { + name: 'EXAMPLE_API_KEY', + description: 'API key for the service', + required: true, + }, + ], + + // Additional metadata + tags: ['search', 'web'], + docsUrl: 'https://example.com/docs', + apiKeyUrl: 'https://example.com/api-keys', + websiteUrl: 'https://example.com', +} +``` + +### Step 2: Run Sync Script + +```bash +# From repository root +pnpm tsx sync-manual-tools.ts +``` + +This will: +1. Fetch the package from npm +2. Create/update Package record +3. Create/update Tool record(s) +4. Set `discoveryMethod: 'manual'` + +### Step 3: Verify + +Check that the tool appears on tpmjs.com: + +```bash +# Start dev server +pnpm dev --filter=@tpmjs/web + +# Visit http://localhost:3000/tool/tool-search +# Search for your package name +``` + +## Multi-Tool Packages + +If a package exports multiple tools, add multiple entries with the same `npmPackageName` but different `exportName`: + +```typescript +{ + npmPackageName: 'firecrawl-aisdk', + exportName: 'scrapeTool', + description: 'Scrape websites...', + // ... +}, +{ + npmPackageName: 'firecrawl-aisdk', + exportName: 'searchTool', + description: 'Search the web...', + // ... +}, +{ + npmPackageName: 'firecrawl-aisdk', + exportName: 'crawlTool', + description: 'Crawl entire websites...', + // ... +}, +``` + +## Tier Calculation + +Tools are automatically assigned a tier: + +- **Rich tier**: Has `parameters` OR `returns` OR `aiAgent` fields +- **Minimal tier**: Only has basic metadata + +Rich tier tools get 4x quality score multiplier, so add detailed metadata when possible. + +## Maintenance + +### Updating Manual Tools + +1. Edit the entry in `manual-tools.ts` +2. Run `pnpm tsx sync-manual-tools.ts` +3. The upsert will update existing records + +### Removing Manual Tools + +1. Remove the entry from `manual-tools.ts` +2. Manually delete from database OR wait for metrics sync to mark as stale + +### Version Updates + +The sync script automatically fetches the latest version from npm unless you specify `npmVersion` in the manual tool entry. + +## Production Deployment + +### Option 1: Manual Sync on Deploy + +Add to your deployment workflow: + +```yaml +# .github/workflows/deploy.yml +- name: Sync manual tools + run: pnpm tsx sync-manual-tools.ts + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} +``` + +### Option 2: Scheduled Sync + +Create a cron job or GitHub Action to sync periodically: + +```yaml +# .github/workflows/sync-manual.yml +name: Sync Manual Tools + +on: + schedule: + - cron: '0 0 * * 0' # Weekly on Sunday + workflow_dispatch: # Manual trigger + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'pnpm' + - run: pnpm install + - run: pnpm tsx sync-manual-tools.ts + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} +``` + +### Option 3: API Endpoint + +Create a sync endpoint (similar to keyword/changes sync): + +```typescript +// apps/web/src/app/api/sync/manual/route.ts +import { manualTools } from '@/manual-tools'; +// ... sync logic + +export async function POST(request: Request) { + // Verify CRON_SECRET + // Run manual sync + // Return results +} +``` + +## Currently Included Manual Tools + +As of this documentation: + +- **ai-sdk-tool-code-execution** - Vercel Sandbox code execution +- **@exalabs/ai-sdk** - Exa web search +- **@parallel-web/ai-sdk-tools** - Parallel search and extraction (2 tools) +- **ctx-zip** - MCP + Vercel Sandbox integration +- **@perplexity-ai/ai-sdk** - Perplexity search +- **@tavily/ai-sdk** - Tavily web research +- **firecrawl-aisdk** - Firecrawl scraping, search, crawling (3 tools) +- **bedrock-agentcore** - AWS Bedrock code interpreter and browser (2 tools) +- **@superagent-ai/ai-sdk** - Superagent security tools (3 tools) +- **@valyu/ai-sdk** - Valyu domain-specific search tools (8 tools) + +**Total: 24 manually curated tools across 10 packages** + +## FAQ + +### Why not just ask package maintainers to add the tpmjs field? + +We should! But: +1. Some packages are from large companies (Vercel, AWS, etc.) with slow adoption cycles +2. We want these tools available on TPMJS now +3. Manual curation lets us provide better metadata than package authors might + +### Will manual tools be replaced by auto-discovered ones? + +Yes! If a package adds a proper `tpmjs` field, the auto-discovery sync will update it with `discoveryMethod: 'keyword'` or `'changes-feed'`. Manual entries can then be removed from `manual-tools.ts`. + +### Can I mix manual and auto-discovered tools from the same package? + +Yes. If a package has some tools in the `tpmjs` field but is missing others, you can manually add the missing ones. The sync scripts will coexist peacefully. + +### How do I know if a tool is manually curated? + +Check the `discoveryMethod` field in the database: +- `'manual'` = Manually curated +- `'keyword'` = Auto-discovered via keyword search +- `'changes-feed'` = Auto-discovered via npm changes feed + +## Best Practices + +1. **Complete Metadata** - Provide as much metadata as possible for rich tier +2. **Accurate Descriptions** - Tool descriptions should be clear and specific +3. **AI-Friendly** - Write `aiAgent.useCase` as guidance for LLMs +4. **Keep Updated** - Periodically check if packages have added native `tpmjs` support +5. **Link to Docs** - Always include `docsUrl` when available +6. **API Key URLs** - Include `apiKeyUrl` for tools requiring authentication + +## Contributing + +To contribute new manual tools: + +1. Fork the repository +2. Add your tool to `manual-tools.ts` +3. Test with `pnpm tsx sync-manual-tools.ts` +4. Open a pull request with: + - Why this tool should be included + - Link to the npm package + - Screenshot of it working in TPMJS + +## Related Documentation + +- [HOW_TO_PUBLISH_A_TOOL.md](./HOW_TO_PUBLISH_A_TOOL.md) - Standard tpmjs field spec +- [CLAUDE.md](./CLAUDE.md) - General project documentation +- [packages/types/src/tpmjs.ts](./packages/types/src/tpmjs.ts) - TypeScript schema definitions diff --git a/NPM_MIRROR.md b/NPM_MIRROR.md deleted file mode 100644 index 61d05cb..0000000 --- a/NPM_MIRROR.md +++ /dev/null @@ -1,500 +0,0 @@ -# TPMJS NPM-Integrated Registry Architecture - -> **Automated tool discovery from NPM with zero-click submission** - -## Vision - -Transform TPMJS from a manual directory into an **automated NPM-integrated registry** where package authors simply publish to NPM with a `tpmjs` field in their `package.json` and their tools are discovered and listed within seconds—no manual submission, no forms, no waiting. - -## Quick Start for Package Authors - -```json -{ - "name": "my-awesome-tool", - "version": "1.0.0", - "keywords": ["tpmjs-tool"], - "tpmjs": { - "category": "web-scraping", - "description": "Extract product data from e-commerce websites with ease", - "example": "const data = await scraper.extract('https://shop.com')" - } -} -``` - -```bash -npm publish -# ✨ Listed automatically within 15 minutes (keyword) or seconds (changes feed) -``` - ---- - -## Architecture Overview - -``` -NPM Ecosystem - ↓ -Changes Feed + Keyword Search - ↓ -Package Validator (Zod) - ↓ -PostgreSQL Database - ↓ -Next.js API Routes - ↓ -TPMJS Web App -``` - -### Core Components - -1. **NPM Sync Service** (Node.js) - Monitors NPM registry for new packages -2. **PostgreSQL Database** - Stores validated tool metadata -3. **Next.js API** - Serves tool data with search/filtering -4. **Web Frontend** - Browse, search, and discover tools - ---- - -## Discovery Mechanism: Hybrid Approach - -### Method 1: Keyword Search (Official) -- Search NPM for packages with `tpmjs-tool` keyword -- Runs every 15 minutes via cron -- Packages marked as "Official" - -### Method 2: Changes Feed (Automatic) -- Monitors `replicate.npmjs.com/registry/_changes` in real-time -- Detects packages with `tpmjs` field instantly -- Packages marked as "Community" (unless they also have keyword) - -### Why Hybrid? -- **Keywords** = Clear opt-in, queryable, respects NPM conventions -- **Changes Feed** = Real-time, catches packages without keywords -- **Together** = Best discoverability with fallback - ---- - -## The "tpmjs" Field: Tiered Schema - -### Minimal Tier (Required) - -```json -{ - "tpmjs": { - "category": "web-scraping", - "description": "Extract structured data from websites using CSS selectors", - "example": "const data = await tool.scrape({ url: 'https://example.com', selector: '.price' })" - } -} -``` - -**Categories:** -- web-scraping -- data-processing -- file-operations -- communication -- database -- api-integration -- image-processing -- text-analysis -- automation -- ai-ml -- security -- monitoring - -### Rich Tier (Optional) - -Extend with any of these optional fields: - -```json -{ - "tpmjs": { - // ... Required fields ... - - "parameters": [ - { - "name": "url", - "type": "string", - "description": "Target URL to scrape", - "required": true - } - ], - "returns": { - "type": "object", - "description": "Extracted data matching the selector" - }, - "authentication": { - "required": false, - "type": "api-key", - "envVar": "SCRAPER_API_KEY", - "docsUrl": "https://docs.example.com/auth" - }, - "pricing": { - "model": "freemium", - "freeLimit": "100 requests/month", - "paidUrl": "https://example.com/pricing" - }, - "frameworks": ["vercel-ai", "langchain", "llamaindex"], - "links": { - "documentation": "https://docs.example.com", - "playground": "https://example.com/try", - "repository": "https://github.com/user/repo" - }, - "tags": ["web", "scraping", "html", "css"], - "status": "stable", - "aiAgent": { - "useCase": "Use when agent needs to extract data from websites", - "limitations": "Cannot handle JavaScript-heavy SPAs" - } - } -} -``` - ---- - -## Database Schema - -### Tools Table - -```sql -CREATE TABLE tools ( - -- NPM Metadata - npm_package_name VARCHAR(214) UNIQUE NOT NULL, - npm_version VARCHAR(50) NOT NULL, - npm_published_at TIMESTAMP NOT NULL, - npm_description TEXT, - npm_repository JSONB, - npm_homepage TEXT, - npm_license VARCHAR(50), - - -- TPMJS Metadata - category VARCHAR(50) NOT NULL, - description TEXT NOT NULL, - example TEXT NOT NULL, - parameters JSONB, - authentication JSONB, - pricing JSONB, - frameworks TEXT[], - links JSONB, - tags TEXT[], - status VARCHAR(20), - - -- Discovery - discovery_method VARCHAR(20) NOT NULL, -- 'keyword' | 'changes-feed' - is_official BOOLEAN DEFAULT false, - tier VARCHAR(20) NOT NULL, -- 'minimal' | 'rich' - - -- Metrics - npm_downloads_last_month INTEGER DEFAULT 0, - github_stars INTEGER DEFAULT 0, - quality_score DECIMAL(3,2), -- 0.00 to 1.00 - - -- Timestamps - created_at TIMESTAMP DEFAULT NOW(), - updated_at TIMESTAMP DEFAULT NOW() -); -``` - ---- - -## Sync Service Architecture - -### Workers - -**1. Changes Feed Worker** -- Connects to `replicate.npmjs.com/registry/_changes` -- Receives real-time change events -- Fetches package metadata for each change -- Checks for `tpmjs` field -- Validates and inserts to database - -**2. Keyword Search Worker** -- Runs every 15 minutes (cron) -- Searches `/-/v1/search?text=keywords:tpmjs-tool` -- Processes all results -- Marks as "Official" - -**3. Metrics Worker** (Optional Phase 4) -- Updates download counts from NPM API -- Fetches GitHub stars -- Calculates quality scores - -### Package Processing Pipeline - -``` -1. Fetch package metadata from NPM -2. Extract `tpmjs` field from latest version -3. Validate against Zod schema -4. If valid → Insert/Update database -5. If invalid → Log error -6. If no field → Skip -``` - ---- - -## API Routes - -### GET /api/tools -Search and list tools - -**Query Parameters:** -- `q` - Search query -- `category` - Filter by category -- `official` - Only official tools (true/false) -- `limit` - Results per page (default 20) -- `offset` - Pagination offset - -**Response:** -```json -{ - "tools": [...], - "pagination": { - "total": 150, - "limit": 20, - "offset": 0, - "hasMore": true - } -} -``` - -### GET /api/tools/[id] -Get tool details by ID - -### POST /api/tools/validate -Validate a `tpmjs` field before publishing - -**Request:** -```json -{ - "category": "web-scraping", - "description": "...", - "example": "..." -} -``` - -**Response:** -```json -{ - "valid": true, - "tier": "minimal", - "errors": [] -} -``` - -### GET /api/stats -Registry statistics - -```json -{ - "totalTools": 2847, - "officialTools": 150, - "categories": { - "web-scraping": 320, - "communication": 280, - ... - } -} -``` - ---- - -## Quality Scoring Algorithm - -Tools are scored 0.00 to 1.00 based on: - -- **Base validity** (0.3) - Has valid schema -- **Tier** (0.1-0.2) - Rich tier > Minimal tier -- **NPM downloads** (0.2) - Based on monthly downloads -- **GitHub stars** (0.15) - Repository popularity -- **Documentation** (0.1) - Has docs URL -- **Example quality** (0.05) - Example length > 100 chars - -Score is used for default sorting and quality indicators. - ---- - -## Implementation Phases - -### Phase 1: Foundation (Week 1-2) -- Set up PostgreSQL + Prisma -- Create Zod schemas in `@tpmjs/types` -- Build sync service structure -- Implement NPM API client - -### Phase 2: Discovery (Week 2-3) -- Implement changes feed worker -- Implement keyword search worker -- Deploy sync service (Railway/Fly.io) -- Test with real packages - -### Phase 3: API & Frontend (Week 3-4) -- Build Next.js API routes -- Update tool listing page -- Update tool detail pages -- Add validation endpoint - -### Phase 4: Polish (Week 4-5) -- Add metrics worker -- Create documentation -- Build CLI validator -- Launch to community - -### Phase 5: Enhancements (Post-Launch) -- Semantic search (embeddings) -- Usage analytics -- Tool recommendations -- GitHub Actions integration - ---- - -## Infrastructure Requirements - -### Sync Service -- **Platform:** Railway or Fly.io -- **Runtime:** Node.js 22+ -- **Resources:** 512MB RAM, 1 CPU -- **Cost:** ~$5-10/month - -### Database -- **Platform:** Neon Postgres (serverless) -- **Size:** Free tier (start), scale as needed -- **Backups:** Automatic with Neon -- **Cost:** Free tier available, ~$10-20/month for production - -### Web App -- **Platform:** Vercel (existing) -- **No changes required** - ---- - -## Monitoring & Health - -### Metrics to Track - -1. **Sync Health** - - Changes feed uptime - - Packages processed per hour - - Validation success rate - -2. **Database** - - Total tools - - Official vs community ratio - - Tier distribution - -3. **API** - - Request latency (p95 < 200ms) - - Search performance - - Error rates - -### Alerts - -- Sync service down > 5 minutes -- Database connection failures -- Validation error rate > 10% - ---- - -## Developer Experience - -### Validation Before Publishing - -```bash -# Using TPMJS CLI (to be built) -npx tpmjs validate - -# Or via API -curl -X POST https://tpmjs.com/api/tools/validate \ - -H "Content-Type: application/json" \ - -d '{"category":"web-scraping","description":"...","example":"..."}' -``` - -### Documentation Pages Needed - -1. **Getting Started** - Adding TPMJS support -2. **Schema Reference** - Complete field docs -3. **Best Practices** - Tips for quality tools -4. **Examples** - Sample configurations -5. **FAQ** - Common questions - ---- - -## Migration from Mock Data - -### Current State -- 12 mock tools in `toolData.ts` -- Client-side search -- Hard-coded categories - -### Migration Strategy - -1. **Publish Real Packages** - - Create NPM packages for mock tools - - Add `tpmjs` fields - - Publish with `tpmjs-tool` keyword - -2. **Update Frontend** - - Replace mock data with API calls - - Keep existing UI components - - Update types to match Prisma models - -3. **Gradual Rollout** - - Dual mode (mock + real) - - Real data primary, mock fallback - - Remove mock entirely - ---- - -## Success Metrics - -### Technical -- ✓ Discovery latency < 60 seconds -- ✓ API response time < 200ms p95 -- ✓ Support 10,000+ tools -- ✓ 99.9% uptime - -### User Experience -- ✓ 0-click submission (automatic) -- ✓ Instant validation feedback -- ✓ <100ms search speed -- ✓ 100% mobile features - -### Business -- Week 1: 10 official tools -- Month 1: 50 official tools -- Month 3: 200+ tools -- Month 6: 1000+ tools -- 50+ active package authors - ---- - -## Comparison to Vercel's Approach - -| Feature | Vercel AI SDK | TPMJS | -|---------|---------------|-------| -| **Submission** | Manual file edit + PR | Automatic via NPM | -| **Discovery** | None | Real-time changes feed | -| **Validation** | Manual review | Automated Zod schema | -| **Updates** | New PR required | Automatic on publish | -| **Search** | Static array | Full-text + categories | -| **Scale** | 6 tools | 1000+ tools ready | - ---- - -## Next Steps - -1. Review this architecture plan -2. Approve database schema and API design -3. Set up infrastructure (Railway + Postgres) -4. Start Phase 1: Foundation -5. Launch MVP in 4-5 weeks - ---- - -## References - -- [NPM Registry API Docs](https://github.com/npm/registry/blob/main/docs/REGISTRY-API.md) -- [NPM Changes Feed](https://github.com/npm/registry/blob/main/docs/REPLICATE-API.md) -- [Vercel AI Tools Registry](https://github.com/vercel/ai/blob/main/content/tools-registry/registry.ts) -- [TPMJS Architecture Plan](/.claude/plans/goofy-inventing-stearns.md) (Full details) - ---- - -**Built with ❤️ for the AI agent ecosystem** diff --git a/README.md b/README.md index 484cd9f..057aaff 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,59 @@ -# TPMJS Monorepo +# TPMJS -[![CI](https://github.com/YOUR_ORG/tpmjs/actions/workflows/ci.yml/badge.svg)](https://github.com/YOUR_ORG/tpmjs/actions/workflows/ci.yml) +[![CI](https://github.com/tpmjs/tpmjs/actions/workflows/ci.yml/badge.svg)](https://github.com/tpmjs/tpmjs/actions/workflows/ci.yml) -Tool Package Manager for AI Agents - A Turborepo monorepo with strict TypeScript, Next.js 16, and best practices. +**TPMJS is a registry for discovering AI tools published to npm.** -## Structure +Browse, search, and find tools at [tpmjs.com](https://tpmjs.com). Publish your tool by adding the `tpmjs` keyword to your package.json—it appears in the registry within 15 minutes. + +## Why TPMJS? + +- **Discover tools** - Search and browse AI tools by category, quality score, and popularity +- **Publish easily** - Add one keyword to package.json, publish to npm, done +- **Quality metrics** - Tools are scored based on documentation, downloads, and metadata completeness +- **Agent integration** - Optional SDK for agents to search and execute tools at runtime + +## Quick Start + +### Publishing a Tool + +```bash +npx @tpmjs/create-basic-tools +``` + +Or add manually to your package.json: +```json +{ + "keywords": ["tpmjs"], + "tpmjs": { + "category": "text-analysis" + } +} +``` + +Publish to npm and your tool appears on [tpmjs.com](https://tpmjs.com) within 15 minutes. + +See [HOW_TO_PUBLISH_A_TOOL.md](./HOW_TO_PUBLISH_A_TOOL.md) for the full guide. + +### For AI Agents (Optional) + +Agents can search and execute tools from the registry: + +```bash +npm install @tpmjs/registry-search @tpmjs/registry-execute +``` + +```typescript +import { registrySearchTool } from '@tpmjs/registry-search'; +import { registryExecuteTool } from '@tpmjs/registry-execute'; + +// Add to your agent's tools +const tools = [registrySearchTool, registryExecuteTool]; +``` + +--- + +## Monorepo Structure ``` apps/ diff --git a/TPMJS_TALK.md b/TPMJS_TALK.md new file mode 100644 index 0000000..5a2b8ed --- /dev/null +++ b/TPMJS_TALK.md @@ -0,0 +1,391 @@ +# TPMJS: The Missing Layer Between "LLMs Can Call Tools" and "Which Tool, Exactly?" + +--- + +## The Setup + +You're building an AI agent. It needs to do things in the world—scrape a webpage, send an email, query a database, generate an image. These capabilities come from **tools**. + +The problem isn't that tools don't exist. They do. Thousands of them. The problem is: + +- **You can't find them.** npm has 2 million packages. Which ones are AI-callable tools? Which ones actually work? +- **You can't trust them.** No schema. No examples. README says "AI-ready" but the function signature is `(opts: any) => Promise`. +- **You can't compare them.** Three packages do "web scraping." Which one handles JavaScript rendering? Which one returns structured data? Which one is maintained? + +Discovery is the bottleneck. Not capability—discovery. + +--- + +## What TPMJS Actually Is + +TPMJS is infrastructure. Specifically: + +1. **A registry** that indexes npm packages designed for AI tool use +2. **A metadata extraction pipeline** that pulls schemas directly from code +3. **A quality scoring system** that ranks tools by completeness and adoption +4. **A health monitoring system** that verifies tools actually work +5. **A playground** where you can test tools before integrating them + +It's not magic. It's plumbing. Good plumbing. + +--- + +## How It Works (The Technical Reality) + +### Discovery: Finding Tools in the Wild + +TPMJS runs three automated sync jobs: + +**1. npm Changes Feed (every 2 minutes)** +``` +npm registry → /_changes endpoint → filter for tpmjs keyword → process +``` +This catches new packages and updates in near-real-time. We track sequence numbers so we never reprocess. + +**2. Keyword Search (every 15 minutes)** +``` +npm search "tpmjs" → up to 250 results → validate → ingest +``` +Backup mechanism. Catches anything the changes feed missed. + +**3. Metrics Sync (hourly)** +``` +for each package → fetch download stats → recalculate quality scores → update health status +``` +Keeps the registry fresh. + +### The Publisher Contract + +To get indexed, a package needs two things: + +```json +{ + "name": "@acme/my-tool", + "keywords": ["tpmjs"], + "tpmjs": { + "category": "web-scraping", + "description": "Scrapes URLs and returns structured markdown" + } +} +``` + +That's the minimum. Category + description. Everything else is either optional or auto-extracted. + +**Categories are fixed** (12 total): web-scraping, data-processing, file-operations, communication, database, api-integration, image-processing, text-analysis, automation, ai-ml, security, monitoring. + +Why fixed? Because agents need to filter. "Give me all database tools" has to mean something. + +### Schema Extraction: The Hard Part + +Here's what makes TPMJS different from a glorified npm search. + +When we ingest a package, we don't just read the README. We **execute it in a sandbox** and extract the actual schema: + +``` +1. Spin up isolated executor (Railway) +2. npm install the package +3. Import and inspect exports +4. Extract JSON Schema from TypeScript types +5. Store schema in database +``` + +The result: + +```json +{ + "name": "scrapeUrl", + "inputSchema": { + "type": "object", + "properties": { + "url": { "type": "string", "format": "uri" }, + "waitForSelector": { "type": "string" }, + "timeout": { "type": "number", "default": 30000 } + }, + "required": ["url"] + } +} +``` + +This isn't documentation. This is **extracted from the actual function signature**. It's ground truth. + +If the author provides a schema in the `tpmjs` field, we use that. If not, we extract it. Either way, every tool in the registry has a schema. + +### Quality Scoring: Ranking What Matters + +Every tool gets a score from 0.00 to 1.00: + +```typescript +// Base score from metadata completeness +const tierScore = tier === 'rich' ? 0.6 : 0.4; + +// Adoption signals +const downloadsScore = Math.min(0.2, Math.log10(downloads + 1) / 15); +const starsScore = Math.min(0.1, Math.log10(githubStars + 1) / 10); + +// Metadata richness bonus +let richnessScore = 0; +if (hasParameters) richnessScore += 0.04; +if (hasReturns) richnessScore += 0.03; +if (hasEnvVars) richnessScore += 0.03; +``` + +**Tier** is binary: +- **Minimal**: Just category + description (40% base) +- **Rich**: Has parameters, returns, env vars, or framework tags (60% base) + +The formula is deliberately simple. We're not trying to be clever. We're trying to surface tools that are well-documented and actually used. + +### Health Checks: Does It Actually Work? + +Two checks, run during sync and periodically: + +**1. Import Health** +``` +Can we require() this package without it exploding? +``` +You'd be surprised how many npm packages fail this. + +**2. Execution Health** +``` +Can we call the main function with minimal parameters without throwing? +``` +Not a full test suite. Just "does it run at all?" + +Results: `HEALTHY`, `BROKEN`, or `UNKNOWN`. + +Broken tools still appear in the registry (with a warning). We don't hide them—we label them. + +--- + +## The Data Model + +Here's what we actually store: + +### Package (npm package level) +``` +npmPackageName (unique) +npmVersion, npmDescription, npmRepository, npmLicense +npmKeywords[], npmReadme, npmAuthor +category (enum) +tier ('minimal' | 'rich') +discoveryMethod ('changes-feed' | 'keyword') +npmDownloadsLastMonth, githubStars +frameworks[] (vercel-ai, langchain, etc.) +env[] (required environment variables) +``` + +### Tool (individual callable within a package) +``` +packageId (FK) +name (export name: "scrapeUrl", "default", etc.) +description +inputSchema (JSON Schema) +schemaSource ('extracted' | 'author') +qualityScore (0.00-1.00) +importHealth, executionHealth (HEALTHY | BROKEN | UNKNOWN) +toolDiscoverySource ('auto' | 'manual') +``` + +One package can have multiple tools. `@acme/web-tools` might export `scrapeUrl`, `screenshotPage`, and `extractLinks`. Each is a separate tool with its own schema and health status. + +### Simulation (playground execution) +``` +toolId +userPrompt (what the user asked) +parameters (JSON, what was passed to the tool) +status (pending | running | success | error | timeout) +executionTimeMs, output, error +model, agentSteps +``` + +We track every playground execution. Not for surveillance—for debugging and improving the system. + +--- + +## The API + +### Search & Discovery + +``` +GET /api/tools + ?q=scrape + &category=web-scraping + &importHealth=HEALTHY + &executionHealth=HEALTHY + &limit=20 + &offset=0 + +→ Returns tools sorted by quality score +``` + +``` +GET /api/tools/search + ?q=I need to extract text from PDFs + +→ BM25-ranked semantic search +``` + +### Execution + +``` +POST /api/tools/execute/{toolId} +{ + "prompt": "Scrape the homepage of Hacker News", + "parameters": { "url": "https://news.ycombinator.com" } +} + +→ Server-Sent Events stream with: + - Agent reasoning steps + - Tool call results + - Final output +``` + +Rate limited: 10 requests per IP per hour. We're not a free compute platform. + +### Schema Operations + +``` +POST /api/tools/extract-schema +{ "packageName": "@acme/my-tool", "toolName": "scrapeUrl" } + +→ Forces re-extraction of schema from source +``` + +--- + +## The Playground + +A Next.js app where you can: + +1. **Browse tools** by category, health status, quality score +2. **Inspect schemas** before you commit to anything +3. **Test execution** with an AI agent +4. **See real responses** with actual latency and token usage + +It's not a demo. It's a debugging tool. "Does this tool do what I think it does?" Answer that question in 30 seconds instead of 30 minutes. + +--- + +## What This Enables + +### For Engineers Building Agents + +Before TPMJS: +``` +1. Search npm for "web scraper" +2. Get 500 results +3. Click through 20 of them +4. Read READMEs that say "easy to use!" +5. npm install three of them +6. Write test code for each +7. Find out two are broken +8. Pick the one that works +9. Hope it keeps working +``` + +After TPMJS: +``` +1. Search tpmjs.com for "web scraper" +2. Filter by HEALTHY status +3. Sort by quality score +4. Click top result +5. See exact input schema +6. Test in playground +7. Integrate +``` + +### For Tool Authors + +Before TPMJS: +``` +Publish to npm → hope someone finds it → no visibility into usage +``` + +After TPMJS: +``` +Publish to npm with tpmjs keyword → indexed within 2 minutes → +schema auto-extracted → quality scored → discoverable by search → +execution stats tracked +``` + +Your tool becomes findable. Not just by humans grepping npm, but by agents querying the registry API. + +### For Agents (Yes, Really) + +Agents can query TPMJS at runtime: + +```typescript +const tools = await fetch('https://tpmjs.com/api/tools?' + new URLSearchParams({ + q: 'send email', + executionHealth: 'HEALTHY', + limit: '5' +})).then(r => r.json()); + +// Agent now has 5 working email tools with full schemas +// It can pick the best one for this specific task +``` + +This is the endgame. Not humans browsing a registry—agents dynamically selecting tools based on capability, health, and fit. + +--- + +## What TPMJS Is Not + +**Not a package manager.** We don't host packages. npm does that. We index and enrich. + +**Not an execution platform.** The playground runs tools for testing. Production execution is your responsibility. + +**Not a security guarantee.** We check if tools work. We don't audit them for malice. Same rules as npm: don't run untrusted code. + +**Not magic.** We're not using AI to understand what tools do. We're extracting schemas and running health checks. Boring, reliable, debuggable. + +--- + +## The Technical Stack + +- **Database**: PostgreSQL via Prisma +- **Web**: Next.js 16 (App Router) +- **Deployment**: Vercel (web) + Railway (sandbox executor) +- **Sync**: Vercel Cron + GitHub Actions backup +- **AI**: Vercel AI SDK for playground execution +- **Monorepo**: Turborepo + pnpm + +Key internal packages: +- `@tpmjs/npm-client` — npm registry integration +- `@tpmjs/package-executor` — sandbox execution client +- `@tpmjs/types` — schema validation and migration +- `@tpmjs/db` — Prisma client and models + +--- + +## Current State + +- **~100 tools indexed** (and growing with every npm publish) +- **12 categories** covering most agent use cases +- **Automated sync** running 24/7 +- **Health checks** on every tool +- **Schema extraction** working for TypeScript and JavaScript +- **Playground** functional for testing + +--- + +## The Pitch (Finally) + +Tools are the API surface of AI agents. The ecosystem is a mess. TPMJS is the index. + +We don't compete with npm—we sit on top of it. We don't replace tool authors—we make them discoverable. We don't build agents—we give agents a way to find their tools. + +Discovery is the bottleneck. We're fixing discovery. + +--- + +## Try It + +- **Browse**: https://tpmjs.com/tool-search +- **Playground**: https://tpmjs.com/playground +- **Publish**: Add `tpmjs` keyword + `tpmjs` field to your package.json +- **API**: `GET https://tpmjs.com/api/tools` + +--- + +*Tools are inevitable. Discovery chaos isn't.* diff --git a/VERCEL_FIX_CHECKLIST.md b/VERCEL_FIX_CHECKLIST.md deleted file mode 100644 index c3c184b..0000000 --- a/VERCEL_FIX_CHECKLIST.md +++ /dev/null @@ -1,324 +0,0 @@ -# 🔥 Vercel API Routes Fix - Action Checklist - -## Problem Summary - -**API routes are not deploying because Vercel is not detecting your project as Next.js.** - -When Vercel doesn't detect Next.js, it treats your deployment as a static site and **drops all App Router API routes** from the build output. Pages work because they're static files, but API routes require serverless function generation which only happens when Next.js is properly detected. - ---- - -## ✅ Fix Checklist (Complete in Order) - -### 1️⃣ Fix Root Directory - -**Where:** Vercel Dashboard → Project `tpmjs-web` → Settings → General → Root Directory - -**Current (likely):** Empty, `.`, or wrong path -**Required:** `apps/web` (exactly this, no leading/trailing slashes) - -**Validation:** -``` -✓ Must be exactly: apps/web -✗ NOT: /apps/web/ -✗ NOT: ./apps/web -✗ NOT: tpmjs/apps/web -``` - ---- - -### 2️⃣ Set Framework Preset - -**Where:** Vercel Dashboard → Project `tpmjs-web` → Settings → General → Framework Preset - -**Current (likely):** "Other" -**Required:** "Next.js" - -**Why this matters:** -- Framework = "Other" → Uses `@vercel/static-builder` → No API routes -- Framework = "Next.js" → Uses `@vercel/next` → API routes deployed - ---- - -### 3️⃣ Remove Domain from Old Project - -**Where:** Vercel Dashboard → Project `v0-tool-registry-page` → Settings → Domains - -**Action:** Remove these domains: -- `tpmjs.com` -- `www.tpmjs.com` - -**Then:** Verify both domains are ONLY assigned to the `tpmjs` project - -**Why this matters:** -The "Redirecting..." message is coming from the old project. Having two projects with the same domain causes shadow routing and API requests hitting the wrong deployment. - ---- - -### 4️⃣ Clear Custom Build Commands - -**Where:** Vercel Dashboard → Project `tpmjs-web` → Settings → Build & Development Settings - -**Set ALL to default/empty:** -``` -Build Command: (empty - let Vercel auto-detect) -Install Command: (empty - let Vercel auto-detect) -Output Directory: .next (default) -``` - -**Why this matters:** -Custom build commands bypass Vercel's Next.js detection. Vercel should automatically: -- Detect monorepo structure -- Run `pnpm install` -- Run `pnpm build` in the correct workspace -- Use `@vercel/next` builder - -**If you must use custom commands, use:** -``` -Build Command: pnpm turbo run build --filter=@tpmjs/web -Install Command: pnpm install -``` - -But try empty first. - ---- - -## 🧪 Verification Steps - -### Before Deploying - -Run this locally to confirm Next.js detection: -```bash -cd apps/web -vercel build -``` - -**Expected output should include:** -``` -● route (app) /api/health -● route (app) /api/tools -● route (app) /api/sync/changes -λ /api/health -λ /api/tools -λ /api/sync/changes -``` - -**If you DON'T see this, Vercel won't deploy API routes.** - -### After Deploying - -1. **Check Build Output:** -```bash -vercel inspect -``` - -Should show: -``` -Builds - ├── λ api/health (XXX KB) [region] - ├── λ api/tools (XXX KB) [region] - ├── λ api/sync/changes (XXX KB) [region] - ├── λ tool/[slug] (XXX KB) [region] - ... -``` - -2. **Test API Routes:** -```bash -# Should return JSON (not timeout, not "Redirecting...") -curl https://tpmjs.com/api/health - -# Should return tool data -curl https://tpmjs.com/api/tools -``` - ---- - -## 📋 Expected Results - -### ✅ Success Indicators - -- [ ] `vercel inspect` shows API routes as `λ` functions -- [ ] `curl https://tpmjs.com/api/health` returns JSON -- [ ] `curl https://tpmjs.com/api/tools` returns tool data -- [ ] No "Redirecting..." messages -- [ ] No timeouts on direct Vercel URLs -- [ ] Build logs show "route (app) /api/*" - -### ❌ Failure Indicators (Need to revisit steps) - -- [ ] Only pages listed in `vercel inspect`, no API routes -- [ ] API endpoints return "Redirecting..." -- [ ] API endpoints timeout (exit code 28) -- [ ] Build logs don't mention API routes -- [ ] Framework Preset still shows "Other" - ---- - -## 🚨 Common Mistakes - -### Mistake 1: Wrong Root Directory Format -``` -✗ /apps/web/ (leading/trailing slashes) -✗ ./apps/web (relative path notation) -✗ apps/web/ (trailing slash) -✓ apps/web (correct) -``` - -### Mistake 2: Leaving Custom Build Commands -If you have: -```json -{ - "buildCommand": "cd ../.. && turbo build --filter=@tpmjs/web" -} -``` - -This MIGHT work, but can break Next.js detection. Start with empty and only add if needed. - -### Mistake 3: Not Removing Domain from Old Project -If `v0-tool-registry-page` still has `tpmjs.com`, your requests will route to the wrong project randomly based on: -- DNS propagation -- Edge cache -- Vercel's routing priority - -### Mistake 4: Not Verifying Framework Preset -"Other" is Vercel's default when it can't detect a framework. This is the #1 cause of missing API routes in monorepos. - ---- - -## 🔧 Troubleshooting - -### If API routes STILL don't deploy after all 4 steps: - -1. **Check Build Logs:** - - Does it say "Detected Next.js"? - - Does it list "route (app) /api/*"? - - Does it show `@vercel/next` builder? - -2. **Check package.json location:** - ``` - ✓ Should exist: apps/web/package.json - ✗ Should NOT be at root ONLY - ``` - -3. **Check next.config.ts location:** - ``` - ✓ Should exist: apps/web/next.config.ts - ``` - -4. **Verify pnpm workspace:** - ```bash - # Should show @tpmjs/web - pnpm list --depth 0 --filter @tpmjs/web - ``` - -5. **Test local build with Vercel CLI:** - ```bash - cd apps/web - vercel build --debug - ``` - Look for "Framework: nextjs" in output. - ---- - -## 📞 When to Contact Vercel Support - -If after completing all 4 steps: -- Build logs show "Detected Next.js" -- Build logs show "route (app) /api/*" -- BUT `vercel inspect` still doesn't list API functions - -Then you have a Vercel platform bug. Contact support with: -- This checklist -- Build logs -- `vercel inspect` output -- Link to `API_ROUTES_TIMEOUT_INVESTIGATION.md` - ---- - -## 🎯 Quick Win Test - -**Don't want to change production settings yet?** - -1. Create a NEW Vercel project -2. Import the SAME repo -3. Set Root Directory to `apps/web` -4. Set Framework Preset to "Next.js" -5. Deploy - -If API routes work in the new project → confirms the fix -If API routes still fail → deeper issue (contact support) - ---- - -## ✨ Post-Fix Cleanup - -Once API routes are working: - -### Optional: Re-add www redirect - -Now that API routes work, you can safely add the www redirect back: - -**Option A - Vercel Project Settings:** -Vercel Dashboard → Domains → tpmjs.com → Redirect www to apex - -**Option B - Next.js config:** -```typescript -// apps/web/next.config.ts -async redirects() { - return [ - { - source: '/:path((?!api).*)*', // Exclude /api/* - has: [{ type: 'host', value: 'www.tpmjs.com' }], - destination: 'https://tpmjs.com/:path*', - permanent: true, - }, - ]; -} -``` - -**Option C - vercel.json (not recommended):** -Only use if you understand the implications. - -### Optional: Remove maxDuration exports - -The `export const maxDuration = 60;` in your route files isn't needed unless you actually need longer timeouts. Default is 10s (Hobby) or 15s (Pro). - ---- - -## 📊 Summary - -| Issue | Root Cause | Fix | -|-------|------------|-----| -| API routes timeout | Vercel doesn't detect Next.js | Set Framework Preset to "Next.js" | -| No λ functions in build | Wrong Root Directory | Set to `apps/web` exactly | -| "Redirecting..." on API calls | Domain on two projects | Remove from old project | -| Build doesn't find API routes | Custom build commands break detection | Clear custom commands | - -**Time to fix:** 5 minutes (just changing dashboard settings) -**Deployments needed:** 1 (changes take effect on next deploy) -**Code changes needed:** 0 (this is pure configuration) - ---- - -## 🎉 When It Works - -You'll know it's fixed when: - -```bash -$ curl https://tpmjs.com/api/health -{"status":"ok","timestamp":"2025-11-28T...","env":{"hasDatabase":true,"nodeEnv":"production"}} - -$ curl https://tpmjs.com/api/tools -{"data":[...],"pagination":{...}} -``` - -And `vercel inspect ` shows: -``` -Builds - ├── λ api/health - ├── λ api/tools - ├── λ api/stats - ... (ALL your API routes) -``` - -**That's it. No code changes. Just fix the Vercel project configuration.** diff --git a/alien.sh b/alien.sh new file mode 100755 index 0000000..de88096 --- /dev/null +++ b/alien.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +printf "\n\033[1m WALKING DOWN\033[0m\n\n" +printf " Frame 1 Frame 2 Frame 3 Frame 4\n\n" +printf " \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m\n" +printf " \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m\n" +printf " \033[38;2;255;200;50m▄███▄\033[0m \033[38;2;255;200;50m▄███▄\033[0m \033[38;2;255;200;50m▄███▄\033[0m \033[38;2;255;200;50m▄███▄\033[0m\n" +printf " \033[38;2;255;200;50m███████\033[0m \033[38;2;255;200;50m███████\033[0m \033[38;2;255;200;50m███████\033[0m \033[38;2;255;200;50m███████\033[0m\n" +printf " \033[38;2;255;200;50m█\033[38;2;0;0;0m██\033[38;2;255;200;50m█\033[38;2;0;0;0m██\033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[38;2;0;0;0m██\033[38;2;255;200;50m█\033[38;2;0;0;0m██\033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[38;2;0;0;0m██\033[38;2;255;200;50m█\033[38;2;0;0;0m██\033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[38;2;0;0;0m██\033[38;2;255;200;50m█\033[38;2;0;0;0m██\033[38;2;255;200;50m█\033[0m\n" +printf " \033[38;2;255;200;50m█████\033[0m \033[38;2;255;200;50m█████\033[0m \033[38;2;255;200;50m█████\033[0m \033[38;2;255;200;50m█████\033[0m\n" +printf " \033[38;2;255;200;50m███\033[0m \033[38;2;255;200;50m███\033[0m \033[38;2;255;200;50m███\033[0m \033[38;2;255;200;50m███\033[0m\n" +printf " \033[38;2;139;90;43m▄█████▄\033[0m \033[38;2;139;90;43m▄█████▄\033[0m \033[38;2;139;90;43m▄█████▄\033[0m \033[38;2;139;90;43m▄█████▄\033[0m\n" +printf " \033[38;2;139;90;43m███\033[38;2;255;215;0m█\033[38;2;139;90;43m███\033[0m \033[38;2;139;90;43m███\033[38;2;255;215;0m█\033[38;2;139;90;43m███\033[0m \033[38;2;139;90;43m███\033[38;2;255;215;0m█\033[38;2;139;90;43m███\033[0m \033[38;2;139;90;43m███\033[38;2;255;215;0m█\033[38;2;139;90;43m███\033[0m\n" +printf " \033[38;2;139;90;43m███████\033[0m \033[38;2;139;90;43m███████\033[0m \033[38;2;139;90;43m███████\033[0m \033[38;2;139;90;43m███████\033[0m\n" +printf " \033[38;2;139;90;43m█████\033[0m \033[38;2;139;90;43m█████\033[0m \033[38;2;139;90;43m█████\033[0m \033[38;2;139;90;43m█████\033[0m\n" +printf " \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m\n" +printf " \033[38;2;101;67;33m██\033[0m \033[38;2;101;67;33m██\033[0m \033[38;2;101;67;33m██\033[0m \033[38;2;101;67;33m█\033[0m \033[38;2;101;67;33m██\033[0m \033[38;2;101;67;33m██\033[0m \033[38;2;101;67;33m█\033[0m \033[38;2;101;67;33m██\033[0m\n" + +printf "\n\033[1m WALKING UP\033[0m\n\n" +printf " Frame 1 Frame 2 Frame 3 Frame 4\n\n" +printf " \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m\n" +printf " \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m\n" +printf " \033[38;2;255;200;50m▄███▄\033[0m \033[38;2;255;200;50m▄███▄\033[0m \033[38;2;255;200;50m▄███▄\033[0m \033[38;2;255;200;50m▄███▄\033[0m\n" +printf " \033[38;2;255;200;50m███████\033[0m \033[38;2;255;200;50m███████\033[0m \033[38;2;255;200;50m███████\033[0m \033[38;2;255;200;50m███████\033[0m\n" +printf " \033[38;2;255;200;50m███████\033[0m \033[38;2;255;200;50m███████\033[0m \033[38;2;255;200;50m███████\033[0m \033[38;2;255;200;50m███████\033[0m\n" +printf " \033[38;2;255;200;50m█████\033[0m \033[38;2;255;200;50m█████\033[0m \033[38;2;255;200;50m█████\033[0m \033[38;2;255;200;50m█████\033[0m\n" +printf " \033[38;2;255;200;50m███\033[0m \033[38;2;255;200;50m███\033[0m \033[38;2;255;200;50m███\033[0m \033[38;2;255;200;50m███\033[0m\n" +printf " \033[38;2;139;90;43m▄█████▄\033[0m \033[38;2;139;90;43m▄█████▄\033[0m \033[38;2;139;90;43m▄█████▄\033[0m \033[38;2;139;90;43m▄█████▄\033[0m\n" +printf " \033[38;2;139;90;43m███████\033[0m \033[38;2;139;90;43m███████\033[0m \033[38;2;139;90;43m███████\033[0m \033[38;2;139;90;43m███████\033[0m\n" +printf " \033[38;2;139;90;43m███████\033[0m \033[38;2;139;90;43m███████\033[0m \033[38;2;139;90;43m███████\033[0m \033[38;2;139;90;43m███████\033[0m\n" +printf " \033[38;2;139;90;43m█████\033[0m \033[38;2;139;90;43m█████\033[0m \033[38;2;139;90;43m█████\033[0m \033[38;2;139;90;43m█████\033[0m\n" +printf " \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m\n" +printf " \033[38;2;101;67;33m██\033[0m \033[38;2;101;67;33m██\033[0m \033[38;2;101;67;33m██\033[0m \033[38;2;101;67;33m█\033[0m \033[38;2;101;67;33m██\033[0m \033[38;2;101;67;33m██\033[0m \033[38;2;101;67;33m█\033[0m \033[38;2;101;67;33m██\033[0m\n" + +printf "\n\033[1m WALKING LEFT\033[0m\n\n" +printf " Frame 1 Frame 2 Frame 3 Frame 4\n\n" +printf " \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m\n" +printf " \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m\n" +printf " \033[38;2;255;200;50m▄███\033[0m \033[38;2;255;200;50m▄███\033[0m \033[38;2;255;200;50m▄███\033[0m \033[38;2;255;200;50m▄███\033[0m\n" +printf " \033[38;2;255;200;50m█████\033[0m \033[38;2;255;200;50m█████\033[0m \033[38;2;255;200;50m█████\033[0m \033[38;2;255;200;50m█████\033[0m\n" +printf " \033[38;2;0;0;0m██\033[38;2;255;200;50m███\033[0m \033[38;2;0;0;0m██\033[38;2;255;200;50m███\033[0m \033[38;2;0;0;0m██\033[38;2;255;200;50m███\033[0m \033[38;2;0;0;0m██\033[38;2;255;200;50m███\033[0m\n" +printf " \033[38;2;255;200;50m████\033[0m \033[38;2;255;200;50m████\033[0m \033[38;2;255;200;50m████\033[0m \033[38;2;255;200;50m████\033[0m\n" +printf " \033[38;2;255;200;50m██\033[0m \033[38;2;255;200;50m██\033[0m \033[38;2;255;200;50m██\033[0m \033[38;2;255;200;50m██\033[0m\n" +printf " \033[38;2;139;90;43m████\033[0m \033[38;2;139;90;43m████\033[0m \033[38;2;139;90;43m████\033[0m \033[38;2;139;90;43m████\033[0m\n" +printf " \033[38;2;139;90;43m██\033[38;2;255;215;0m█\033[38;2;139;90;43m██\033[0m \033[38;2;139;90;43m██\033[38;2;255;215;0m█\033[38;2;139;90;43m██\033[0m \033[38;2;139;90;43m██\033[38;2;255;215;0m█\033[38;2;139;90;43m██\033[0m \033[38;2;139;90;43m██\033[38;2;255;215;0m█\033[38;2;139;90;43m██\033[0m\n" +printf " \033[38;2;139;90;43m████\033[0m \033[38;2;139;90;43m████\033[0m \033[38;2;139;90;43m████\033[0m \033[38;2;139;90;43m████\033[0m\n" +printf " \033[38;2;139;90;43m██\033[0m \033[38;2;139;90;43m██\033[0m \033[38;2;139;90;43m██\033[0m \033[38;2;139;90;43m██\033[0m\n" +printf " \033[38;2;255;200;50m██\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m██\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m\n" +printf " \033[38;2;101;67;33m██\033[0m \033[38;2;101;67;33m█\033[0m \033[38;2;101;67;33m█\033[0m \033[38;2;101;67;33m██\033[0m \033[38;2;101;67;33m██\033[0m \033[38;2;101;67;33m█\033[0m \033[38;2;101;67;33m█\033[0m \033[38;2;101;67;33m██\033[0m\n" + +printf "\n\033[1m WALKING RIGHT\033[0m\n\n" +printf " Frame 1 Frame 2 Frame 3 Frame 4\n\n" +printf " \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m\n" +printf " \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m \033[38;2;139;90;43m█\033[0m\n" +printf " \033[38;2;255;200;50m███▄\033[0m \033[38;2;255;200;50m███▄\033[0m \033[38;2;255;200;50m███▄\033[0m \033[38;2;255;200;50m███▄\033[0m\n" +printf " \033[38;2;255;200;50m█████\033[0m \033[38;2;255;200;50m█████\033[0m \033[38;2;255;200;50m█████\033[0m \033[38;2;255;200;50m█████\033[0m\n" +printf " \033[38;2;255;200;50m███\033[38;2;0;0;0m██\033[0m \033[38;2;255;200;50m███\033[38;2;0;0;0m██\033[0m \033[38;2;255;200;50m███\033[38;2;0;0;0m██\033[0m \033[38;2;255;200;50m███\033[38;2;0;0;0m██\033[0m\n" +printf " \033[38;2;255;200;50m████\033[0m \033[38;2;255;200;50m████\033[0m \033[38;2;255;200;50m████\033[0m \033[38;2;255;200;50m████\033[0m\n" +printf " \033[38;2;255;200;50m██\033[0m \033[38;2;255;200;50m██\033[0m \033[38;2;255;200;50m██\033[0m \033[38;2;255;200;50m██\033[0m\n" +printf " \033[38;2;139;90;43m████\033[0m \033[38;2;139;90;43m████\033[0m \033[38;2;139;90;43m████\033[0m \033[38;2;139;90;43m████\033[0m\n" +printf " \033[38;2;139;90;43m██\033[38;2;255;215;0m█\033[38;2;139;90;43m██\033[0m \033[38;2;139;90;43m██\033[38;2;255;215;0m█\033[38;2;139;90;43m██\033[0m \033[38;2;139;90;43m██\033[38;2;255;215;0m█\033[38;2;139;90;43m██\033[0m \033[38;2;139;90;43m██\033[38;2;255;215;0m█\033[38;2;139;90;43m██\033[0m\n" +printf " \033[38;2;139;90;43m████\033[0m \033[38;2;139;90;43m████\033[0m \033[38;2;139;90;43m████\033[0m \033[38;2;139;90;43m████\033[0m\n" +printf " \033[38;2;139;90;43m██\033[0m \033[38;2;139;90;43m██\033[0m \033[38;2;139;90;43m██\033[0m \033[38;2;139;90;43m██\033[0m\n" +printf " \033[38;2;255;200;50m██\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m██\033[0m \033[38;2;255;200;50m█\033[0m \033[38;2;255;200;50m█\033[0m\n" +printf " \033[38;2;101;67;33m█\033[0m \033[38;2;101;67;33m██\033[0m \033[38;2;101;67;33m██\033[0m \033[38;2;101;67;33m█\033[0m \033[38;2;101;67;33m█\033[0m \033[38;2;101;67;33m██\033[0m \033[38;2;101;67;33m██\033[0m \033[38;2;101;67;33m█\033[0m\n" + +printf "\n" diff --git a/apps/playground/.env.local.example b/apps/playground/.env.local.example new file mode 100644 index 0000000..ad3ac40 --- /dev/null +++ b/apps/playground/.env.local.example @@ -0,0 +1,2 @@ +# Required: OpenAI API key +OPENAI_API_KEY=sk-... diff --git a/apps/playground/eslint.config.mjs b/apps/playground/eslint.config.mjs new file mode 100644 index 0000000..f2ac8a1 --- /dev/null +++ b/apps/playground/eslint.config.mjs @@ -0,0 +1,16 @@ +import reactConfig from '@tpmjs/eslint-config/react.js'; + +export default [ + { + ignores: [ + '.next/**', + '.turbo/**', + 'node_modules/**', + '*.config.js', + '*.config.ts', + 'next-env.d.ts', + 'eslint.config.mjs', + ], + }, + ...reactConfig, +]; diff --git a/apps/playground/next-env.d.ts b/apps/playground/next-env.d.ts new file mode 100644 index 0000000..9edff1c --- /dev/null +++ b/apps/playground/next-env.d.ts @@ -0,0 +1,6 @@ +/// +/// +import "./.next/types/routes.d.ts"; + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/apps/playground/next.config.ts b/apps/playground/next.config.ts new file mode 100644 index 0000000..b94699c --- /dev/null +++ b/apps/playground/next.config.ts @@ -0,0 +1,11 @@ +import type { NextConfig } from 'next'; + +const nextConfig: NextConfig = { + transpilePackages: ['@tpmjs/ui', '@tpmjs/utils', '@tpmjs/types', '@tpmjs/env'], + reactStrictMode: true, + experimental: { + urlImports: ['https://esm.sh/', 'https://cdn.jsdelivr.net/npm/'], + }, +}; + +export default nextConfig; diff --git a/apps/playground/package.json b/apps/playground/package.json new file mode 100644 index 0000000..df5104d --- /dev/null +++ b/apps/playground/package.json @@ -0,0 +1,50 @@ +{ + "name": "@tpmjs/playground", + "version": "0.0.0", + "private": true, + "scripts": { + "dev": "next dev --port 3001", + "build": "next build", + "start": "next start", + "lint": "eslint .", + "type-check": "tsc --noEmit", + "clean": "rm -rf .next .turbo" + }, + "dependencies": { + "@ai-sdk/openai": "3.0.1", + "@ai-sdk/react": "3.0.3", + "@tpmjs/db": "workspace:*", + "@tpmjs/env": "workspace:*", + "@tpmjs/hello": "workspace:*", + "@tpmjs/search-registry": "workspace:*", + "@tpmjs/types": "workspace:*", + "@tpmjs/ui": "workspace:*", + "@tpmjs/utils": "workspace:*", + "@vercel/analytics": "^1.6.1", + "ai": "6.0.3", + "firecrawl-aisdk": "^0.7.2", + "nanoid": "^5.1.6", + "next": "^16.0.8", + "next-themes": "^0.4.6", + "openai": "^6.9.1", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "streamdown": "^1.6.9", + "zod": "^4.0.0" + }, + "devDependencies": { + "@tailwindcss/typography": "^0.5.19", + "@tpmjs/eslint-config": "workspace:*", + "@tpmjs/tailwind-config": "workspace:*", + "@tpmjs/tsconfig": "workspace:*", + "@types/node": "^22.10.2", + "@types/react": "^19.0.2", + "@types/react-dom": "^19.0.2", + "autoprefixer": "^10.4.20", + "eslint": "^9.39.1", + "eslint-config-next": "^16.0.4", + "postcss": "^8.5.1", + "tailwindcss": "^3.4.17", + "typescript": "^5.9.3" + } +} diff --git a/apps/playground/postcss.config.mjs b/apps/playground/postcss.config.mjs new file mode 100644 index 0000000..a982c64 --- /dev/null +++ b/apps/playground/postcss.config.mjs @@ -0,0 +1,8 @@ +const config = { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +}; + +export default config; diff --git a/apps/playground/src/app/api/chat/route.ts b/apps/playground/src/app/api/chat/route.ts new file mode 100644 index 0000000..243368e --- /dev/null +++ b/apps/playground/src/app/api/chat/route.ts @@ -0,0 +1,252 @@ +import { createOpenAI } from '@ai-sdk/openai'; +import { searchTpmjsToolsTool } from '@tpmjs/search-registry'; +import { type UIMessage, convertToModelMessages, stepCountIs, streamText } from 'ai'; +import type { NextRequest } from 'next/server'; +import { env } from '~/env'; +import { + addConversationTools, + loadToolsBatch, + setConversationEnv, +} from '~/lib/dynamic-tool-loader'; +import { loadAllTools, sanitizeToolName } from '~/lib/tool-loader'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; +export const maxDuration = 300; // 5 minutes for complex tool loading + +// Add conversation state tracking (in-memory for MVP) +// biome-ignore lint/suspicious/noExplicitAny: Tool types from AI SDK are complex +const conversationStates = new Map }>(); + +/** + * POST /api/chat + * Chat with AI agent that can execute TPMJS tools + */ +export async function POST(request: NextRequest) { + try { + const body = await request.json(); + console.log('📥 Request body:', JSON.stringify(body, null, 2)); + + const messages: UIMessage[] = body.messages || []; + const conversationId: string = body.conversationId || 'default'; + const clientEnv: Record = body.env || {}; + + console.log(`🔑 Conversation ID: ${conversationId}`); + console.log( + `🔐 Client env vars: ${Object.keys(clientEnv).length} keys`, + Object.keys(clientEnv) + ); + + // Store env vars for this conversation (so cached tools can access them) + setConversationEnv(conversationId, clientEnv); + + // Initialize OpenAI with client-provided or server API key + const apiKey = clientEnv.OPENAI_API_KEY || env.OPENAI_API_KEY; + if (!apiKey) { + return new Response( + JSON.stringify({ + success: false, + error: 'OPENAI_API_KEY is required. Please add it in the Settings sidebar.', + }), + { + status: 400, + headers: { 'Content-Type': 'application/json' }, + } + ); + } + + const openai = createOpenAI({ + apiKey, + }); + + // Get or create conversation state + if (!conversationStates.has(conversationId)) { + console.log('✨ Creating new conversation state'); + conversationStates.set(conversationId, { loadedTools: {} }); + } + // biome-ignore lint/style/noNonNullAssertion: We just ensured the value exists above + const state = conversationStates.get(conversationId)!; + console.log( + `📊 Current loaded tools in conversation: ${Object.keys(state.loadedTools).length}` + ); + + // 1. Load static tools + search tool + const staticTools = await loadAllTools(); + console.log(`🔧 Loaded ${Object.keys(staticTools).length} static tools`); + + staticTools.searchTpmjsTools = searchTpmjsToolsTool; + console.log('✅ Added searchTpmjsTools to static tools'); + + // Debug: Check the search tool structure + console.log('🔍 Search tool structure:', { + description: searchTpmjsToolsTool.description, + inputSchema: typeof searchTpmjsToolsTool.inputSchema, + execute: typeof searchTpmjsToolsTool.execute, + }); + + // 2. Extract user query and last 3 user messages for tool search + const lastMessage = messages[messages.length - 1]; + let userQuery = ''; + if (lastMessage?.role === 'user') { + // Extract text from message parts + const parts = (lastMessage as any).parts || []; + for (const part of parts) { + if (part.type === 'text') { + userQuery = part.text; + break; + } + } + } + + // Get last 3 user messages for context + const recentUserMessages = messages + .filter((msg) => msg.role === 'user') + .slice(-3) + .map((msg) => { + // Extract text from parts + const parts = (msg as any).parts || []; + for (const part of parts) { + if (part.type === 'text') { + return part.text; + } + } + return ''; + }) + .filter(Boolean); + + console.log(`💬 User query: "${userQuery}"`); + console.log(`📝 Recent messages: ${recentUserMessages.length}`); + + // 3. Automatically search for relevant tools based on the user's message + if (userQuery && userQuery.trim().length > 0) { + console.log('🔎 Searching for relevant tools...'); + + try { + // biome-ignore lint/style/noNonNullAssertion: Tool created with tool() always has execute + const result = await searchTpmjsToolsTool.execute!( + { + query: userQuery, + limit: 5, // Get top 5 relevant tools + recentMessages: recentUserMessages, + }, + {} as any + ); + + // Type assertion: searchTpmjsToolsTool returns direct result, not AsyncIterable + const searchResult = result as { + query: string; + matchCount: number; + tools: any[]; + }; + + console.log(`📦 Found ${searchResult.matchCount} matching tools`); + + if (searchResult.tools && searchResult.tools.length > 0) { + console.log( + '🔧 Tools found:', + searchResult.tools.map((t: any) => `${t.packageName}/${t.name}`) + ); + + // Dynamically load tools from esm.sh + console.log(`📥 Loading ${searchResult.tools.length} tools dynamically...`); + + const toolsToLoad = searchResult.tools.map((meta: any) => ({ + packageName: meta.packageName, + name: meta.name, + version: meta.version, + importUrl: meta.importUrl, + })); + + try { + const loadedTools = await loadToolsBatch(toolsToLoad, conversationId, clientEnv); + console.log(`✅ Successfully loaded ${Object.keys(loadedTools).length} tools`); + + // Add sanitized tools to conversation state + for (const [key, tool] of Object.entries(loadedTools)) { + const [pkg, exp] = key.split('::'); + const sanitizedKey = sanitizeToolName(`${pkg}-${exp}`); + state.loadedTools[sanitizedKey] = tool; + console.log(`✅ Added to conversation: ${sanitizedKey}`); + } + + // Track for this conversation + addConversationTools(conversationId, Object.keys(state.loadedTools)); + } catch (error) { + console.error('❌ Error loading tools:', error); + } + } else { + console.log('ℹ️ No matching tools found for this query'); + } + } catch (error) { + console.error('❌ Error searching for tools:', error); + } + } + + // 4. Merge with conversation's dynamically loaded tools + // biome-ignore lint/suspicious/noExplicitAny: Tool types from AI SDK are complex + const allTools: Record = { ...staticTools, ...state.loadedTools }; + + // 5. Build system prompt with available tools + const toolsList = Object.keys(allTools) + .map((name) => { + const tool = allTools[name] as { description?: string } | undefined; + return `- ${name}: ${tool?.description || 'No description'}`; + }) + .join('\n'); + + const system = `You are an AI assistant with access to a dynamic tool registry containing thousands of tools. Your job is to EXECUTE tools to help users accomplish tasks. + +## Tool Execution Rules + +1. **When a user asks you to "call", "use", "run", or "execute" a tool** - you MUST invoke that tool immediately. Do not just describe it or search for it. + +2. **When a user asks a question that could be answered by a tool** - invoke the appropriate tool to get real data, don't make up answers. + +3. **searchTpmjsTools is for DISCOVERY only** - use it when you need to find tools you don't have loaded yet. Once a tool is loaded (listed below), call it directly. + +4. **Tool names are sanitized** - if user says "extractTool from @parallel-web/ai-sdk-tools", look for a loaded tool like "parallel-web_ai-sdk-tools-extractTool". + +5. **Always execute, then explain** - after calling a tool, summarize the results for the user. + +## Currently Loaded Tools +${toolsList} + +## Examples + +User: "call extractTool on https://example.com" +→ Invoke the extractTool with url parameter, then explain results + +User: "search for web scraping tools" +→ Use searchTpmjsTools to find tools, then tell user what's available + +User: "what's the weather in Tokyo" +→ Search for a weather tool, load it, then invoke it + +Remember: Your value is in EXECUTING tools to get real results, not just describing what tools could do.`; + + // 6. Stream response with all available tools + const result = streamText({ + model: openai('gpt-4o-mini'), + system, + messages: await convertToModelMessages(messages), + tools: allTools, + stopWhen: stepCountIs(5), // Allow model to call tools AND generate text response + }); + + // Return UI message stream with tool calls and text + return result.toUIMessageStreamResponse(); + } catch (error) { + console.error('Chat API error:', error); + + return new Response( + JSON.stringify({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }), + { + status: 500, + headers: { 'Content-Type': 'application/json' }, + } + ); + } +} diff --git a/apps/playground/src/app/api/tools/route.ts b/apps/playground/src/app/api/tools/route.ts new file mode 100644 index 0000000..b892796 --- /dev/null +++ b/apps/playground/src/app/api/tools/route.ts @@ -0,0 +1,85 @@ +import { NextResponse } from 'next/server'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +interface RawTool { + id: string; + name: string; + description: string; + qualityScore: number; + importHealth: 'HEALTHY' | 'BROKEN' | 'UNKNOWN'; + executionHealth: 'HEALTHY' | 'BROKEN' | 'UNKNOWN'; + healthCheckError: string | null; + lastHealthCheck: string | null; + package?: { + npmPackageName: string; + npmVersion: string; + category: string; + frameworks: string[]; + env: Array<{ name: string; description: string; required?: boolean; default?: string }>; + }; +} + +function transformTool(tool: RawTool) { + return { + toolId: tool.id, + packageName: tool.package?.npmPackageName, + name: tool.name, + description: tool.description, + category: tool.package?.category, + version: tool.package?.npmVersion, + qualityScore: tool.qualityScore, + frameworks: tool.package?.frameworks, + env: tool.package?.env, + importUrl: `https://esm.sh/${tool.package?.npmPackageName}@${tool.package?.npmVersion}`, + importHealth: tool.importHealth, + executionHealth: tool.executionHealth, + healthCheckError: tool.healthCheckError, + lastHealthCheck: tool.lastHealthCheck, + }; +} + +export async function GET() { + try { + const baseUrl = process.env.TPMJS_API_URL || 'https://tpmjs.com'; + const allTools: RawTool[] = []; + let offset = 0; + const limit = 50; // Max allowed by the API + let hasMore = true; + + // Paginate through all tools + while (hasMore) { + const response = await fetch(`${baseUrl}/api/tools?limit=${limit}&offset=${offset}`); + + if (!response.ok) { + throw new Error(`Failed to fetch tools: ${response.statusText}`); + } + + const data = await response.json(); + const tools = data.data || []; + allTools.push(...tools); + + hasMore = data.pagination?.hasMore ?? false; + offset += limit; + + // Safety limit to prevent infinite loops + if (offset > 1000) break; + } + + return NextResponse.json({ + success: true, + tools: allTools.map(transformTool), + total: allTools.length, + }); + } catch (error) { + console.error('Failed to fetch tools:', error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Failed to fetch tools', + }, + { status: 500 } + ); + } +} diff --git a/apps/playground/src/app/globals.css b/apps/playground/src/app/globals.css new file mode 100644 index 0000000..13600ad --- /dev/null +++ b/apps/playground/src/app/globals.css @@ -0,0 +1,30 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + +@layer base { + /* Light mode (default) */ + :root { + /* Status Colors */ + --error: 0 65% 51%; /* Red */ + --error-foreground: 0 0% 100%; /* White text */ + --warning: 36 100% 50%; /* Amber */ + --warning-foreground: 0 0% 100%; + --success: 152 57% 45%; /* Green */ + --success-foreground: 0 0% 100%; + --info: 210 100% 56%; /* Blue */ + --info-foreground: 0 0% 100%; + } + + /* Dark mode */ + .dark { + --error: 0 65% 58%; /* Brighter red for dark mode */ + --error-foreground: 0 0% 100%; + --warning: 36 100% 55%; + --warning-foreground: 0 0% 100%; + --success: 152 57% 50%; + --success-foreground: 0 0% 100%; + --info: 210 100% 60%; + --info-foreground: 0 0% 100%; + } +} diff --git a/apps/playground/src/app/layout.tsx b/apps/playground/src/app/layout.tsx new file mode 100644 index 0000000..cac4e42 --- /dev/null +++ b/apps/playground/src/app/layout.tsx @@ -0,0 +1,44 @@ +import { Analytics } from '@vercel/analytics/next'; +import type { Metadata } from 'next'; +import { ThemeProvider } from 'next-themes'; +import { Space_Grotesk, Space_Mono } from 'next/font/google'; +import './globals.css'; + +const spaceGrotesk = Space_Grotesk({ + subsets: ['latin'], + variable: '--font-sans', + display: 'swap', +}); + +const spaceMono = Space_Mono({ + subsets: ['latin'], + weight: ['400', '700'], + variable: '--font-mono', + display: 'swap', +}); + +export const metadata: Metadata = { + title: 'TPMJS Playground - Test AI Tools', + description: 'Interactive playground for testing TPMJS tools with AI agents', +}; + +export default function RootLayout({ + children, +}: { + children: React.ReactNode; +}): React.ReactElement { + return ( + + + + {children} + + + + + ); +} diff --git a/apps/playground/src/app/page.tsx b/apps/playground/src/app/page.tsx new file mode 100644 index 0000000..f616fbc --- /dev/null +++ b/apps/playground/src/app/page.tsx @@ -0,0 +1,24 @@ +'use client'; + +import { ChatHeader } from '~/components/chat/ChatHeader'; +import { ChatInterface } from '~/components/chat/ChatInterface'; +import { SettingsSidebar } from '~/components/sidebar/SettingsSidebar'; +import { ToolsSidebar } from '~/components/sidebar/ToolsSidebar'; + +export default function PlaygroundPage(): React.ReactElement { + const handleClearChat = () => { + // Refresh the page to clear chat + window.location.reload(); + }; + + return ( +
+ +
+ + + +
+
+ ); +} diff --git a/apps/playground/src/components/chat/ChatHeader.tsx b/apps/playground/src/components/chat/ChatHeader.tsx new file mode 100644 index 0000000..2f6a3bd --- /dev/null +++ b/apps/playground/src/components/chat/ChatHeader.tsx @@ -0,0 +1,52 @@ +'use client'; + +import { Button } from '@tpmjs/ui/Button/Button'; +import { useTheme } from 'next-themes'; +import Link from 'next/link'; +import { useEffect, useState } from 'react'; + +interface ChatHeaderProps { + onClear: () => void; +} + +export function ChatHeader({ onClear }: ChatHeaderProps): React.ReactElement { + const { theme, setTheme } = useTheme(); + const [mounted, setMounted] = useState(false); + + // Avoid hydration mismatch + useEffect(() => { + setMounted(true); + }, []); + + const toggleTheme = () => { + setTheme(theme === 'dark' ? 'light' : 'dark'); + }; + + return ( +
+
+
+

TPMJS Playground

+ + View Registry → + +
+
+ {mounted && ( + + )} + +
+
+
+ ); +} diff --git a/apps/playground/src/components/chat/ChatInput.tsx b/apps/playground/src/components/chat/ChatInput.tsx new file mode 100644 index 0000000..a9e38d5 --- /dev/null +++ b/apps/playground/src/components/chat/ChatInput.tsx @@ -0,0 +1,63 @@ +'use client'; + +import { Button } from '@tpmjs/ui/Button/Button'; +import { Textarea } from '@tpmjs/ui/Textarea/Textarea'; +import type { FormEvent } from 'react'; + +interface ChatInputProps { + input: string; + isLoading: boolean; + onInputChange: (e: React.ChangeEvent) => void; + onSubmit: (e: FormEvent) => void; + setInput: (value: string) => void; +} + +export function ChatInput({ + input, + isLoading, + onInputChange, + onSubmit, +}: ChatInputProps): React.ReactElement { + const handleKeyDown = (e: React.KeyboardEvent) => { + if (e.key === 'Enter' && !e.shiftKey) { + e.preventDefault(); + if (input.trim() && !isLoading) { + // Trigger form submission + const form = e.currentTarget.form; + if (form) { + form.requestSubmit(); + } + } + } + }; + + return ( +
+
+
+