diff --git a/.dependency-cruiser.js b/.dependency-cruiser.js index 2ffd268..c928de9 100644 --- a/.dependency-cruiser.js +++ b/.dependency-cruiser.js @@ -105,8 +105,8 @@ export default { from: {}, to: { couldNotResolve: true, - // Allow TypeScript path aliases that are resolved by the TS compiler - pathNot: ['^~/'], + // Allow TypeScript path aliases and workspace packages that are resolved by the TS compiler + pathNot: ['^~/', '^@/', '^@tpmjs/'], }, }, { diff --git a/.env.vercel.production b/.env.vercel.production new file mode 100644 index 0000000..bf0ac19 --- /dev/null +++ b/.env.vercel.production @@ -0,0 +1,41 @@ +# Created by Vercel CLI +CRON_SECRET="CRON_SECRET=6c806d35cf6212f489c76414d38d2b6acbc44590ac78bb08aadea28dd04a29d0\n" +DATABASE_URL="postgresql://neondb_owner:npg_euvYo4OTi1lX@ep-broad-darkness-a4lml85k-pooler.us-east-1.aws.neon.tech/neondb?sslmode=require" +DATABASE_URL_UNPOOLED="postgresql://neondb_owner:npg_euvYo4OTi1lX@ep-broad-darkness-a4lml85k.us-east-1.aws.neon.tech/neondb?sslmode=require" +NEXT_PUBLIC_STACK_PROJECT_ID="d786bd3a-a31d-4c6b-9497-5d6803dd9d86" +NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY="pck_hafmpkaj047z331x5azv8bk5zggfnbgdedbj9pfqh1rn0" +NX_DAEMON="false" +PGDATABASE="neondb" +PGHOST="ep-broad-darkness-a4lml85k-pooler.us-east-1.aws.neon.tech" +PGHOST_UNPOOLED="ep-broad-darkness-a4lml85k.us-east-1.aws.neon.tech" +PGPASSWORD="npg_euvYo4OTi1lX" +PGUSER="neondb_owner" +POSTGRES_DATABASE="neondb" +POSTGRES_HOST="ep-broad-darkness-a4lml85k-pooler.us-east-1.aws.neon.tech" +POSTGRES_PASSWORD="npg_euvYo4OTi1lX" +POSTGRES_PRISMA_URL="postgresql://neondb_owner:npg_euvYo4OTi1lX@ep-broad-darkness-a4lml85k-pooler.us-east-1.aws.neon.tech/neondb?connect_timeout=15&sslmode=require" +POSTGRES_URL="postgresql://neondb_owner:npg_euvYo4OTi1lX@ep-broad-darkness-a4lml85k-pooler.us-east-1.aws.neon.tech/neondb?sslmode=require" +POSTGRES_URL_NON_POOLING="postgresql://neondb_owner:npg_euvYo4OTi1lX@ep-broad-darkness-a4lml85k.us-east-1.aws.neon.tech/neondb?sslmode=require" +POSTGRES_URL_NO_SSL="postgresql://neondb_owner:npg_euvYo4OTi1lX@ep-broad-darkness-a4lml85k-pooler.us-east-1.aws.neon.tech/neondb" +POSTGRES_USER="neondb_owner" +STACK_SECRET_SERVER_KEY="ssk_p05kwe938wx13rpera9xf1fewc816dwkbq658xcsbwj1g" +TURBO_CACHE="remote:rw" +TURBO_DOWNLOAD_LOCAL_ENABLED="true" +TURBO_REMOTE_ONLY="true" +TURBO_RUN_SUMMARY="true" +VERCEL="1" +VERCEL_ENV="production" +VERCEL_GIT_COMMIT_AUTHOR_LOGIN="" +VERCEL_GIT_COMMIT_AUTHOR_NAME="" +VERCEL_GIT_COMMIT_MESSAGE="" +VERCEL_GIT_COMMIT_REF="" +VERCEL_GIT_COMMIT_SHA="" +VERCEL_GIT_PREVIOUS_SHA="" +VERCEL_GIT_PROVIDER="" +VERCEL_GIT_PULL_REQUEST_ID="" +VERCEL_GIT_REPO_ID="" +VERCEL_GIT_REPO_OWNER="" +VERCEL_GIT_REPO_SLUG="" +VERCEL_OIDC_TOKEN="eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6Im1yay00MzAyZWMxYjY3MGY0OGE5OGFkNjFkYWRlNGEyM2JlNyJ9.eyJpc3MiOiJodHRwczovL29pZGMudmVyY2VsLmNvbS90cG1qcyIsInN1YiI6Im93bmVyOnRwbWpzOnByb2plY3Q6dHBtanMtd2ViOmVudmlyb25tZW50OmRldmVsb3BtZW50Iiwic2NvcGUiOiJvd25lcjp0cG1qczpwcm9qZWN0OnRwbWpzLXdlYjplbnZpcm9ubWVudDpkZXZlbG9wbWVudCIsImF1ZCI6Imh0dHBzOi8vdmVyY2VsLmNvbS90cG1qcyIsIm93bmVyIjoidHBtanMiLCJvd25lcl9pZCI6InRlYW1femtHV0NXYjdWakhvbmk2VmJ5ZmQyc3c4IiwicHJvamVjdCI6InRwbWpzLXdlYiIsInByb2plY3RfaWQiOiJwcmpfNWd1MEkwVzFjUFhkQ3ozd1RjQ0ZIejQzNUJ0MCIsImVudmlyb25tZW50IjoiZGV2ZWxvcG1lbnQiLCJwbGFuIjoicHJvIiwidXNlcl9pZCI6IkxKZk05VzdIdlljb2gyclVCaXRWd283ViIsIm5iZiI6MTc2NDM4OTAxMiwiaWF0IjoxNzY0Mzg5MDEyLCJleHAiOjE3NjQ0MzIyMTJ9.OF4IHrcmteA2lU1tkqHO1a9ITGGrCjCo29G8jI991q8_SQjgHZHVqcBj3AYVKZJDh6BjHib4HyNKdjO8nwUblF2dCFbYDv6y4hwB6jHNpsz32BE1JDKcXEJOKPtg_tBOFUDKtzMkPk7VOPWDVYw8Tz4_HZ_MR3SNoy1Pk9AFL-hEl3E-zR3bAYMDB8tKrIm9y9K4sZF6efMU7BR_J6Bf-i3IsbbrH-Axgq5dewlpogf-xHWmWaTXoUp6UFejNKhSMXqg3sAWTnizYeSGc2Ut6zNuAYPumUPBdQ37Kk7vuRNwS1h7RJz3vtEg6aOuw0-Ld0LdF-tWkfDGsVqanR_sxw" +VERCEL_TARGET_ENV="production" +VERCEL_URL="" diff --git a/.github/workflows/health-check.yml b/.github/workflows/health-check.yml new file mode 100644 index 0000000..ddf2a15 --- /dev/null +++ b/.github/workflows/health-check.yml @@ -0,0 +1,18 @@ +name: Daily Health Check + +on: + schedule: + # Run daily at 2am UTC + - cron: '0 2 * * *' + workflow_dispatch: + +jobs: + health-check: + runs-on: ubuntu-latest + steps: + - name: Trigger health check sync + run: | + curl -X POST "${{ secrets.VERCEL_PRODUCTION_URL }}/api/sync/health-check" \ + -H "Authorization: Bearer ${{ secrets.CRON_SECRET }}" \ + -H "Content-Type: application/json" \ + -f -s -S -w "\nHTTP Status: %{http_code}\n" diff --git a/.github/workflows/sync-changes.yml b/.github/workflows/sync-changes.yml new file mode 100644 index 0000000..fa2886e --- /dev/null +++ b/.github/workflows/sync-changes.yml @@ -0,0 +1,18 @@ +name: Sync NPM Changes Feed + +on: + schedule: + # Run every 2 minutes + - cron: '*/2 * * * *' + workflow_dispatch: + +jobs: + sync-changes: + runs-on: ubuntu-latest + steps: + - name: Trigger changes feed sync + run: | + curl -X POST "${{ secrets.VERCEL_PRODUCTION_URL }}/api/sync/changes" \ + -H "Authorization: Bearer ${{ secrets.CRON_SECRET }}" \ + -H "Content-Type: application/json" \ + -f -s -S -w "\nHTTP Status: %{http_code}\n" diff --git a/.github/workflows/sync-keyword.yml b/.github/workflows/sync-keyword.yml new file mode 100644 index 0000000..55c6399 --- /dev/null +++ b/.github/workflows/sync-keyword.yml @@ -0,0 +1,131 @@ +name: Sync NPM Keyword Search + +on: + schedule: + # Run every 15 minutes + - cron: '*/15 * * * *' + workflow_dispatch: + +jobs: + sync-keyword: + runs-on: ubuntu-latest + steps: + - name: Trigger keyword search sync + id: sync + run: | + # Call the sync API and capture response + response=$(curl -X POST "${{ secrets.VERCEL_PRODUCTION_URL }}/api/sync/keyword" \ + -H "Authorization: Bearer ${{ secrets.CRON_SECRET }}" \ + -H "Content-Type: application/json" \ + -f -s -S) + + echo "Response: $response" + + # Extract data using jq + processed=$(echo "$response" | jq -r '.data.processed') + skipped=$(echo "$response" | jq -r '.data.skipped') + errors=$(echo "$response" | jq -r '.data.errors') + packagesFound=$(echo "$response" | jq -r '.data.packagesFound') + durationMs=$(echo "$response" | jq -r '.data.durationMs') + + # Extract and display error messages + errorMessages=$(echo "$response" | jq -r '.data.errorMessages[]?' 2>/dev/null || echo "") + + if [ -n "$errorMessages" ]; then + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "⚠️ SYNC ERRORS ($errors total):" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "$response" | jq -r '.data.errorMessages[]?' | while IFS= read -r error; do + echo " • $error" + done + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + fi + + # Set outputs for Discord notification + echo "processed=$processed" >> $GITHUB_OUTPUT + echo "skipped=$skipped" >> $GITHUB_OUTPUT + echo "errors=$errors" >> $GITHUB_OUTPUT + echo "packagesFound=$packagesFound" >> $GITHUB_OUTPUT + echo "durationMs=$durationMs" >> $GITHUB_OUTPUT + + # Store error messages for Discord (first 3, truncated) + if [ "$errors" -gt 0 ]; then + errorSummary=$(echo "$response" | jq -r '.data.errorMessages[0:3]? | join("\n• ")' 2>/dev/null || echo "") + if [ -n "$errorSummary" ]; then + # Save to file to preserve newlines + echo "• $errorSummary" > /tmp/error_summary.txt + fi + fi + + # Store skipped packages for Discord + if [ "$skipped" -gt 0 ]; then + skippedList=$(echo "$response" | jq -r '.data.skippedPackages[]? | "\(.name) (by \(.author)) - \(.reason)"' 2>/dev/null | paste -sd "\n" - || echo "") + if [ -n "$skippedList" ]; then + echo "$skippedList" > /tmp/skipped_packages.txt + fi + fi + + # Determine status emoji + if [ "$errors" -gt 0 ]; then + echo "status_emoji=⚠️" >> $GITHUB_OUTPUT + echo "status_color=16776960" >> $GITHUB_OUTPUT # Yellow + else + echo "status_emoji=✅" >> $GITHUB_OUTPUT + echo "status_color=5763719" >> $GITHUB_OUTPUT # Green + fi + + - name: Send Discord notification + if: always() + run: | + # Format duration + duration_sec=$(echo "scale=2; ${{ steps.sync.outputs.durationMs }} / 1000" | bc) + + # Build Discord payload using jq for proper JSON escaping + # Read optional data + error_text="" + skipped_text="" + + if [ -f /tmp/error_summary.txt ] && [ ${{ steps.sync.outputs.errors }} -gt 0 ]; then + error_text=$(cat /tmp/error_summary.txt | head -c 800) + fi + + if [ -f /tmp/skipped_packages.txt ] && [ ${{ steps.sync.outputs.skipped }} -gt 0 ]; then + skipped_text=$(cat /tmp/skipped_packages.txt) + fi + + # Build fields array dynamically + base_fields='[ + { "name": "📦 Packages Found", "value": "${{ steps.sync.outputs.packagesFound }}", "inline": true }, + { "name": "✨ Processed", "value": "${{ steps.sync.outputs.processed }}", "inline": true }, + { "name": "⏭️ Skipped", "value": "${{ steps.sync.outputs.skipped }}", "inline": true }, + { "name": "❌ Errors", "value": "${{ steps.sync.outputs.errors }}", "inline": true }, + { "name": "⏱️ Duration", "value": "'"${duration_sec}s"'", "inline": true }, + { "name": "🔗 Run", "value": "[View Logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})", "inline": true } + ]' + + # Create payload with dynamic fields + payload=$(jq -n \ + --arg title "${{ steps.sync.outputs.status_emoji }} NPM Keyword Search Sync" \ + --argjson color ${{ steps.sync.outputs.status_color }} \ + --argjson baseFields "$base_fields" \ + --arg error_text "$error_text" \ + --arg skipped_text "$skipped_text" \ + --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%S.000Z)" \ + ' + { + embeds: [{ + title: $title, + color: $color, + fields: ( + $baseFields + + (if $skipped_text != "" then [{ name: "📋 Skipped Packages", value: $skipped_text, inline: false }] else [] end) + + (if $error_text != "" then [{ name: "🔍 Error Details", value: ("```\n" + $error_text + "\n```"), inline: false }] else [] end) + ), + timestamp: $timestamp + }] + }') + + # Send to Discord + curl -X POST "${{ secrets.DISCORD_WEBHOOK }}" \ + -H "Content-Type: application/json" \ + -d "$payload" diff --git a/.github/workflows/sync-manual.yml b/.github/workflows/sync-manual.yml new file mode 100644 index 0000000..135f5af --- /dev/null +++ b/.github/workflows/sync-manual.yml @@ -0,0 +1,97 @@ +name: Sync Manual Tools + +on: + schedule: + # Run daily at midnight UTC + - cron: '0 0 * * *' + workflow_dispatch: + # Run on pushes to main that modify manual-tools.ts + push: + branches: + - main + paths: + - 'manual-tools.ts' + - 'sync-manual-tools.ts' + +jobs: + sync-manual: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup pnpm + uses: pnpm/action-setup@v2 + with: + version: 8 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Generate Prisma Client + run: pnpm --filter=@tpmjs/db db:generate + + - name: Run manual tools sync + id: sync + run: | + # Run the sync script and capture output + output=$(pnpm tsx sync-manual-tools.ts 2>&1) + echo "$output" + + # Extract statistics from output + processed=$(echo "$output" | grep "Processed:" | awk '{print $2}') + skipped=$(echo "$output" | grep "Skipped:" | awk '{print $2}') + errors=$(echo "$output" | grep "Errors:" | awk '{print $2}') + total=$(echo "$output" | grep "Total manual tools:" | awk '{print $4}') + + # Set outputs for Discord notification + echo "processed=${processed:-0}" >> $GITHUB_OUTPUT + echo "skipped=${skipped:-0}" >> $GITHUB_OUTPUT + echo "errors=${errors:-0}" >> $GITHUB_OUTPUT + echo "total=${total:-0}" >> $GITHUB_OUTPUT + + # Determine status + if [ "${errors:-0}" -gt 0 ]; then + echo "status_emoji=⚠️" >> $GITHUB_OUTPUT + echo "status_color=16776960" >> $GITHUB_OUTPUT # Yellow + else + echo "status_emoji=✅" >> $GITHUB_OUTPUT + echo "status_color=5763719" >> $GITHUB_OUTPUT # Green + fi + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + + - name: Send Discord notification + if: always() + run: | + # Build Discord payload + payload=$(jq -n \ + --arg title "${{ steps.sync.outputs.status_emoji }} Manual Tools Sync" \ + --argjson color ${{ steps.sync.outputs.status_color }} \ + --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%S.000Z)" \ + ' + { + embeds: [{ + title: $title, + color: $color, + fields: [ + { name: "📦 Total Tools", value: "${{ steps.sync.outputs.total }}", inline: true }, + { name: "✨ Processed", value: "${{ steps.sync.outputs.processed }}", inline: true }, + { name: "⏭️ Skipped", value: "${{ steps.sync.outputs.skipped }}", inline: true }, + { name: "❌ Errors", value: "${{ steps.sync.outputs.errors }}", inline: true }, + { name: "🔗 Run", value: "[View Logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})", inline: true } + ], + timestamp: $timestamp + }] + }') + + # Send to Discord + curl -X POST "${{ secrets.DISCORD_WEBHOOK }}" \ + -H "Content-Type: application/json" \ + -d "$payload" diff --git a/.github/workflows/sync-metrics.yml b/.github/workflows/sync-metrics.yml new file mode 100644 index 0000000..7bd5c40 --- /dev/null +++ b/.github/workflows/sync-metrics.yml @@ -0,0 +1,18 @@ +name: Sync NPM Metrics + +on: + schedule: + # Run every hour + - cron: '0 * * * *' + workflow_dispatch: + +jobs: + sync-metrics: + runs-on: ubuntu-latest + steps: + - name: Trigger metrics sync + run: | + curl -X POST "${{ secrets.VERCEL_PRODUCTION_URL }}/api/sync/metrics" \ + -H "Authorization: Bearer ${{ secrets.CRON_SECRET }}" \ + -H "Content-Type: application/json" \ + -f -s -S -w "\nHTTP Status: %{http_code}\n" diff --git a/.github/workflows/sync-vercel-registry.yml b/.github/workflows/sync-vercel-registry.yml new file mode 100644 index 0000000..ad3bf15 --- /dev/null +++ b/.github/workflows/sync-vercel-registry.yml @@ -0,0 +1,264 @@ +name: Sync Vercel AI Registry + +on: + schedule: + # Run every hour + - cron: '0 * * * *' + workflow_dispatch: + # Run on pushes to main that modify the sync script + push: + branches: + - main + paths: + - 'sync-vercel-registry.ts' + +jobs: + sync-vercel: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.14.0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'pnpm' + + - name: Install dependencies + run: | + echo "📦 Installing dependencies..." + pnpm install --frozen-lockfile + echo "✅ Dependencies installed" + + - name: Run Vercel registry sync + id: sync + run: | + echo "════════════════════════════════════════" + echo "🚀 Starting Vercel AI Registry Sync" + echo "════════════════════════════════════════" + echo "" + echo "📅 Time: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" + echo "🔑 OpenAI API Key: ${OPENAI_API_KEY:0:8}..." + echo "" + + # Run the sync script and capture output + output=$(pnpm tsx sync-vercel-registry.ts 2>&1) + exit_code=$? + + echo "$output" + echo "" + + # Extract statistics from output + processed=$(echo "$output" | grep "Processed:" | tail -1 | awk '{print $2}') + skipped=$(echo "$output" | grep "Skipped:" | tail -1 | awk '{print $2}') + errors=$(echo "$output" | grep "Errors:" | tail -1 | awk '{print $2}') + total=$(echo "$output" | grep "Total:" | tail -1 | awk '{print $2}') + + # Set default values if extraction failed + processed=${processed:-0} + skipped=${skipped:-0} + errors=${errors:-0} + total=${total:-0} + + echo "════════════════════════════════════════" + echo "📊 Sync Statistics" + echo "════════════════════════════════════════" + echo "✨ Processed: $processed" + echo "⏭️ Skipped: $skipped" + echo "❌ Errors: $errors" + echo "📦 Total: $total" + echo "════════════════════════════════════════" + echo "" + + # Set outputs for later steps + echo "processed=$processed" >> $GITHUB_OUTPUT + echo "skipped=$skipped" >> $GITHUB_OUTPUT + echo "errors=$errors" >> $GITHUB_OUTPUT + echo "total=$total" >> $GITHUB_OUTPUT + echo "exit_code=$exit_code" >> $GITHUB_OUTPUT + + # Check if manual-tools.ts was modified + if git diff --quiet manual-tools.ts; then + echo "has_changes=false" >> $GITHUB_OUTPUT + echo "ℹ️ No changes to manual-tools.ts" + else + echo "has_changes=true" >> $GITHUB_OUTPUT + echo "✅ manual-tools.ts was modified" + echo "" + echo "📝 Changes preview:" + git diff --stat manual-tools.ts + echo "" + git diff manual-tools.ts | head -50 + fi + + # Determine status for notifications + if [ "$exit_code" -ne 0 ]; then + echo "status_emoji=❌" >> $GITHUB_OUTPUT + echo "status_color=15158332" >> $GITHUB_OUTPUT # Red + echo "status_text=Failed" >> $GITHUB_OUTPUT + elif [ "$errors" -gt 0 ]; then + echo "status_emoji=⚠️" >> $GITHUB_OUTPUT + echo "status_color=16776960" >> $GITHUB_OUTPUT # Yellow + echo "status_text=Completed with errors" >> $GITHUB_OUTPUT + else + echo "status_emoji=✅" >> $GITHUB_OUTPUT + echo "status_color=5763719" >> $GITHUB_OUTPUT # Green + echo "status_text=Success" >> $GITHUB_OUTPUT + fi + + # Exit with the original exit code + exit $exit_code + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + - name: Commit and push changes + if: steps.sync.outputs.has_changes == 'true' + run: | + echo "════════════════════════════════════════" + echo "📝 Committing changes to manual-tools.ts" + echo "════════════════════════════════════════" + echo "" + + # Configure git + git config --local user.email "github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + + # Show what's being committed + echo "📋 Files to commit:" + git status --short + echo "" + + # Commit changes + git add manual-tools.ts + + # Create commit message + COMMIT_MSG="chore: sync ${{ steps.sync.outputs.processed }} new tools from Vercel AI registry + + Added ${{ steps.sync.outputs.processed }} tools from Vercel AI SDK registry: + - Total tools in registry: ${{ steps.sync.outputs.total }} + - Already synced: ${{ steps.sync.outputs.skipped }} + - Newly added: ${{ steps.sync.outputs.processed }} + - Errors: ${{ steps.sync.outputs.errors }} + + 🤖 Automated by GitHub Actions + Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + + git commit -m "$COMMIT_MSG" + + echo "✅ Changes committed" + echo "" + + # Push changes + echo "📤 Pushing to remote..." + git push + + echo "✅ Changes pushed successfully" + echo "════════════════════════════════════════" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Send Discord notification + if: always() + run: | + echo "════════════════════════════════════════" + echo "📢 Sending Discord notification" + echo "════════════════════════════════════════" + + # Build fields array + base_fields='[ + { "name": "📦 Total Tools", "value": "${{ steps.sync.outputs.total }}", "inline": true }, + { "name": "✨ Processed", "value": "${{ steps.sync.outputs.processed }}", "inline": true }, + { "name": "⏭️ Skipped", "value": "${{ steps.sync.outputs.skipped }}", "inline": true }, + { "name": "❌ Errors", "value": "${{ steps.sync.outputs.errors }}", "inline": true }, + { "name": "📝 Changes", "value": "${{ steps.sync.outputs.has_changes }}", "inline": true }, + { "name": "🔗 Run", "value": "[View Logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})", "inline": true } + ]' + + # Add commit info if changes were made + if [ "${{ steps.sync.outputs.has_changes }}" = "true" ]; then + commit_sha=$(git rev-parse HEAD) + commit_url="https://github.com/${{ github.repository }}/commit/${commit_sha}" + additional_fields='[ + { "name": "💾 Commit", "value": "['"${commit_sha:0:7}"']('"$commit_url"')", "inline": false } + ]' + + # Merge fields + all_fields=$(jq -n --argjson base "$base_fields" --argjson additional "$additional_fields" '$base + $additional') + else + all_fields="$base_fields" + fi + + # Create Discord embed + payload=$(jq -n \ + --arg title "${{ steps.sync.outputs.status_emoji }} Vercel AI Registry Sync - ${{ steps.sync.outputs.status_text }}" \ + --argjson color ${{ steps.sync.outputs.status_color }} \ + --argjson fields "$all_fields" \ + --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%S.000Z)" \ + --arg description "Synced Vercel AI SDK tools registry with TPMJS manual tools" \ + ' + { + embeds: [{ + title: $title, + description: $description, + color: $color, + fields: $fields, + timestamp: $timestamp, + footer: { + text: "Vercel AI Registry Sync" + } + }] + }') + + echo "📤 Sending payload to Discord..." + + # Send to Discord + response=$(curl -X POST "${{ secrets.DISCORD_WEBHOOK }}" \ + -H "Content-Type: application/json" \ + -d "$payload" \ + -w "\nHTTP Status: %{http_code}\n" \ + -s) + + echo "$response" + + if echo "$response" | grep -q "HTTP Status: 2"; then + echo "✅ Discord notification sent successfully" + else + echo "⚠️ Discord notification may have failed" + fi + + echo "════════════════════════════════════════" + env: + DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} + + - name: Summary + if: always() + run: | + echo "" + echo "════════════════════════════════════════" + echo "📊 Workflow Summary" + echo "════════════════════════════════════════" + echo "" + echo "Status: ${{ steps.sync.outputs.status_text }}" + echo "Tools Processed: ${{ steps.sync.outputs.processed }}" + echo "Tools Skipped: ${{ steps.sync.outputs.skipped }}" + echo "Errors: ${{ steps.sync.outputs.errors }}" + echo "Total in Registry: ${{ steps.sync.outputs.total }}" + echo "Changes Made: ${{ steps.sync.outputs.has_changes }}" + echo "" + + if [ "${{ steps.sync.outputs.has_changes }}" = "true" ]; then + echo "✅ New tools added to manual-tools.ts and committed" + else + echo "ℹ️ No new tools found - manual-tools.ts is up to date" + fi + + echo "" + echo "════════════════════════════════════════" diff --git a/CLAUDE.md b/CLAUDE.md index 3c9063c..089f0a4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -678,4 +678,423 @@ API timeouts in serverless environments often stem from build configuration issu 4. Use `vercel inspect` to verify lambda deployment 5. Test database performance locally before deploying -The full working implementation is live at [tpmjs.com](https://tpmjs.com). \ No newline at end of file +The full working implementation is live at [tpmjs.com](https://tpmjs.com). + +--- + +## NPM Package Syncing System + +TPMJS.com automatically mirrors npm packages with the `tpmjs-tool` keyword to keep the tool registry up-to-date. This section documents how the syncing system works. + +### Overview + +The sync system uses three automated strategies running on Vercel Cron to discover and update TPMJS tools: + +1. **Changes Feed** - Monitors npm's real-time changes feed for all package updates +2. **Keyword Search** - Actively searches npm for packages with the `tpmjs-tool` keyword +3. **Metrics Sync** - Updates download stats and calculates quality scores + +### Sync Endpoints + +All sync endpoints are located in `apps/web/src/app/api/sync/`: + +#### 1. Changes Feed Sync (`/api/sync/changes`) + +**Purpose:** Monitors npm's changes feed to catch new packages and updates in real-time. + +**Schedule:** Every 2 minutes (`*/2 * * * *`) + +**How it works:** +1. Fetches the last checkpoint sequence number from the database +2. Calls npm's `/_changes` endpoint with `since=` (limit 100 per run) +3. For each changed package, fetches full metadata with `fetchLatestPackageWithMetadata()` +4. Validates that the package has a valid `tpmjs` field using `validateTpmjsField()` +5. Upserts the tool to the database with `discoveryMethod: 'changes-feed'` +6. Updates the checkpoint with the new sequence number for next run + +**Key Features:** +- Uses checkpoints to track progress and avoid reprocessing +- Processes up to 100 changes per run to avoid timeouts +- Logs all sync operations to `syncLog` table +- Requires `Authorization: Bearer ` header + +**Example Response:** +```json +{ + "success": true, + "data": { + "processed": 5, + "skipped": 93, + "errors": 0, + "lastSeq": "12345678", + "pending": 1250, + "durationMs": 2834 + } +} +``` + +#### 2. Keyword Search Sync (`/api/sync/keyword`) + +**Purpose:** Actively searches npm for packages with the `tpmjs-tool` keyword. + +**Schedule:** Every 15 minutes (`*/15 * * * *`) + +**How it works:** +1. Searches npm registry for packages with keyword `tpmjs-tool` (up to 250 results) +2. Fetches full metadata for each package +3. Validates the `tpmjs` field +4. Upserts tools with `discoveryMethod: 'keyword'` +5. Updates checkpoint with last run timestamp + +**Key Features:** +- Catches packages that might be missed by changes feed +- Useful for backfilling existing packages +- Processes up to 250 packages per run + +**Example Response:** +```json +{ + "success": true, + "data": { + "processed": 12, + "skipped": 3, + "errors": 0, + "packagesFound": 15, + "durationMs": 4521 + } +} +``` + +#### 3. Metrics Sync (`/api/sync/metrics`) + +**Purpose:** Updates download statistics and calculates quality scores for all tools. + +**Schedule:** Every hour (`0 * * * *`) + +**How it works:** +1. Fetches all tools from the database +2. For each tool, calls `fetchDownloadStats()` to get last 30 days of downloads +3. Calculates quality score based on: + - Tier (rich = 0.6, minimal = 0.4) + - Downloads (logarithmic scale, max 0.3) + - GitHub stars (logarithmic scale, max 0.1) +4. Updates `npmDownloadsLastMonth` and `qualityScore` fields + +**Quality Score Formula:** +```typescript +function calculateQualityScore(params: { + tier: string; + downloads: number; + githubStars: number; +}): number { + const tierScore = tier === 'rich' ? 0.6 : 0.4; + const downloadsScore = Math.min(0.3, Math.log10(downloads + 1) / 10); + const starsScore = Math.min(0.1, Math.log10(githubStars + 1) / 10); + return Math.min(1.0, tierScore + downloadsScore + starsScore); +} +``` + +**Example Response:** +```json +{ + "success": true, + "data": { + "processed": 25, + "skipped": 0, + "errors": 0, + "totalTools": 25, + "durationMs": 8234 + } +} +``` + +### Automated Sync Configuration + +The sync system can run via two methods: + +#### Option 1: Vercel Cron (Primary) + +Cron jobs are configured in `vercel.json` at the repository root: + +```json +{ + "crons": [ + { + "path": "/api/sync/changes", + "schedule": "*/2 * * * *" + }, + { + "path": "/api/sync/keyword", + "schedule": "*/15 * * * *" + }, + { + "path": "/api/sync/metrics", + "schedule": "0 * * * *" + } + ] +} +``` + +**Pros:** +- Native Vercel integration +- Automatic authentication with `CRON_SECRET` +- Same infrastructure as the app +- No setup required (works automatically on deploy) + +#### Option 2: GitHub Actions (Backup) + +A GitHub Actions workflow (`.github/workflows/sync.yml`) provides redundancy: + +```yaml +name: NPM Package Sync + +on: + schedule: + - cron: '*/2 * * * *' # Changes feed + - cron: '*/15 * * * *' # Keyword search + - cron: '0 * * * *' # Metrics + workflow_dispatch: # Manual trigger +``` + +**Pros:** +- Redundancy if Vercel Cron fails +- Manual trigger via GitHub UI +- Free on GitHub (included in free tier) +- Runs from GitHub's infrastructure + +**Setup:** + +1. Add secrets to GitHub repository settings: + - `VERCEL_PRODUCTION_URL` - Your production URL (e.g., `https://tpmjs.com`) + - `CRON_SECRET` - Same secret used in Vercel environment variables + +2. Enable GitHub Actions in repository settings + +3. The workflow will run automatically on schedule OR manually via: + - GitHub Actions tab → NPM Package Sync → Run workflow → Select sync type + +**Schedule Breakdown:** +- Changes feed: Every 2 minutes (30 times per hour) +- Keyword search: Every 15 minutes (4 times per hour) +- Metrics: Every hour (once per hour) + +**Recommendation:** Use Vercel Cron as primary and GitHub Actions as backup. Both can run simultaneously - the sync endpoints are idempotent. + +### Database Schema + +The sync system uses these Prisma models: + +**`Tool` - The main tool registry:** +```prisma +model Tool { + id String @id @default(cuid()) + npmPackageName String @unique + npmVersion String + npmDownloadsLastMonth Int @default(0) + qualityScore Float? + discoveryMethod String // 'changes-feed' | 'keyword' + tier String // 'minimal' | 'rich' + // ... other fields + + @@index([qualityScore]) + @@index([npmDownloadsLastMonth]) +} +``` + +**`SyncCheckpoint` - Tracks sync progress:** +```prisma +model SyncCheckpoint { + id String @id @default(cuid()) + source String @unique // 'changes-feed' | 'keyword-search' | 'metrics' + checkpoint Json // { lastSeq: string, lastRun: string, ... } +} +``` + +**`SyncLog` - Records all sync operations:** +```prisma +model SyncLog { + id String @id @default(cuid()) + source String + status String // 'success' | 'partial' | 'error' + processed Int + skipped Int + errors Int + message String? + metadata Json? + createdAt DateTime @default(now()) +} +``` + +### Manual Sync Triggers + +To manually trigger a sync (useful for testing or debugging): + +```bash +# Trigger changes feed sync +curl -X POST https://tpmjs.com/api/sync/changes \ + -H "Authorization: Bearer $CRON_SECRET" + +# Trigger keyword search +curl -X POST https://tpmjs.com/api/sync/keyword \ + -H "Authorization: Bearer $CRON_SECRET" + +# Trigger metrics update +curl -X POST https://tpmjs.com/api/sync/metrics \ + -H "Authorization: Bearer $CRON_SECRET" +``` + +**Note:** You need the `CRON_SECRET` environment variable set in Vercel. The endpoints return 401 Unauthorized without it. + +### Monitoring Sync Health + +Check sync logs in the database: + +```typescript +// Get recent sync operations +const recentSyncs = await prisma.syncLog.findMany({ + orderBy: { createdAt: 'desc' }, + take: 20, +}); + +// Check last successful sync for each source +const checkpoints = await prisma.syncCheckpoint.findMany(); +``` + +**Sync Log Example:** +```json +{ + "id": "clx...", + "source": "changes-feed", + "status": "success", + "processed": 5, + "skipped": 93, + "errors": 0, + "message": "Successfully processed 5 packages", + "metadata": { + "durationMs": 2834, + "lastSeq": "12345678", + "pending": 1250 + }, + "createdAt": "2025-11-30T12:00:00Z" +} +``` + +### Error Handling + +All sync endpoints follow this error handling pattern: + +1. **Partial Success:** If some packages fail but others succeed, status is `partial` +2. **Complete Failure:** If the entire sync fails, status is `error` +3. **Error Messages:** First 3 errors are included in the response +4. **Logging:** All operations are logged to `syncLog` regardless of success + +**Example Partial Failure:** +```json +{ + "success": true, + "data": { + "processed": 5, + "skipped": 2, + "errors": 3, + "durationMs": 5234 + } +} +``` + +The sync log will contain: +```json +{ + "status": "partial", + "message": "Processed with errors: Failed to process pkg1: Network timeout; Failed to process pkg2: Invalid tpmjs field; ..." +} +``` + +### Configuration + +Required environment variables in Vercel: + +```bash +# Database connection +DATABASE_URL="postgresql://..." + +# Cron job authentication +CRON_SECRET="your-secret-key" +``` + +**Important:** Vercel Cron automatically adds the `Authorization: Bearer $CRON_SECRET` header when calling the endpoints. No manual configuration needed. + +### Performance Considerations + +**Timeouts:** +- All sync routes have `maxDuration: 300` (5 minutes) +- Changes feed processes max 100 packages per run to avoid timeouts +- Keyword search processes max 250 packages per run +- Metrics sync processes all tools but runs only once per hour + +**Rate Limiting:** +- npm API has rate limits - be cautious when testing manually +- Vercel Cron jobs run from Vercel's infrastructure (different IP than dev) +- Consider implementing exponential backoff for npm API errors + +**Cold Starts:** +- First request to each sync endpoint may be slow due to Prisma initialization +- Subsequent requests are faster with warm Prisma Client +- This is acceptable for background cron jobs + +### Debugging Sync Issues + +**Check if cron jobs are running:** + +```bash +# View recent deployments +vercel ls + +# Check logs for a specific deployment +vercel logs + +# Filter for sync-related logs +vercel logs | grep sync +``` + +**Common issues:** + +1. **"Unauthorized" errors:** Check that `CRON_SECRET` is set in Vercel environment variables +2. **Timeouts:** Reduce batch size in changes feed (currently 100) +3. **Missing packages:** Check `syncLog` for errors during processing +4. **Stale data:** Verify metrics sync is running every hour + +**Test sync locally:** + +```bash +# Start dev server +pnpm dev --filter=@tpmjs/web + +# Trigger sync (requires CRON_SECRET in .env.local) +curl -X POST http://localhost:3000/api/sync/changes \ + -H "Authorization: Bearer $CRON_SECRET" +``` + +### Package Discovery Flow + +Here's how a new TPMJS tool gets discovered: + +1. **Developer publishes package to npm** with `tpmjs-tool` keyword and `tpmjs` field in package.json +2. **Within 2 minutes:** Changes feed sync picks it up from npm's `/_changes` endpoint +3. **Validation:** `validateTpmjsField()` checks that the `tpmjs` field meets requirements +4. **Database Insert:** Tool is upserted with initial data +5. **Within 1 hour:** Metrics sync updates download stats and calculates quality score +6. **Visible on tpmjs.com:** Tool appears in search results and category pages + +**Backup Discovery:** If changes feed misses a package, the keyword search (every 15 minutes) will catch it. + +### Future Improvements + +Potential enhancements to the sync system: + +- [ ] Add webhook endpoint for instant npm package notifications +- [ ] Implement exponential backoff for npm API rate limits +- [ ] Add Slack/Discord notifications for sync failures +- [ ] Create admin dashboard to monitor sync health +- [ ] Support GitHub stars syncing (requires GitHub API integration) +- [ ] Add sync metrics to Vercel Analytics +- [ ] Implement differential sync to reduce database writes \ No newline at end of file diff --git a/DENO_NODE_PACKAGE_ISSUE.md b/DENO_NODE_PACKAGE_ISSUE.md new file mode 100644 index 0000000..f8fbc5f --- /dev/null +++ b/DENO_NODE_PACKAGE_ISSUE.md @@ -0,0 +1,306 @@ +# Running `ai-sdk-tool-code-execution` in Deno - Compatibility Issue + +## Problem Summary + +We need to run the npm package `ai-sdk-tool-code-execution` in a Deno runtime environment on Railway. The package requires Node.js built-ins (`node:sqlite`, `undici`) that don't exist in Deno, and we're looking for a solution to make it work. + +## Environment + +- **Runtime:** Deno 1.39.0 on Railway +- **Package:** `ai-sdk-tool-code-execution@0.0.2` +- **Import Method:** Dynamic imports via esm.sh CDN +- **Use Case:** Remote code execution for AI SDK tools + +## What We're Trying to Do + +We have a Deno server that dynamically imports npm packages at runtime to provide AI SDK tools. The workflow is: + +1. User requests a tool (e.g., `executeCode`) +2. Deno server fetches the package from esm.sh or npm +3. Server loads the tool's schema and execution function +4. Server executes the tool with user-provided parameters + +## The Package We Need + +**Package:** `ai-sdk-tool-code-execution` +**Version:** `0.0.2` +**Description:** Execute Python code in a sandboxed environment using Vercel Sandbox +**npm URL:** https://www.npmjs.com/package/ai-sdk-tool-code-execution +**CDN URLs:** +- esm.sh: `https://esm.sh/ai-sdk-tool-code-execution@0.0.2` +- jsdelivr: `https://cdn.jsdelivr.net/npm/ai-sdk-tool-code-execution@0.0.2/+esm` + +**Dependencies (from package.json):** +```json +{ + "dependencies": { + "ai": "^4.0.18", + "better-sqlite3": "^11.8.1", + "undici": "^7.16.0" + } +} +``` + +**Key Issue:** The package depends on: +- `better-sqlite3` → which requires `node:sqlite` (Node.js built-in) +- `undici` → HTTP client that uses Node.js internals + +## What We've Tried + +### Attempt 1: Deno npm: Specifier (Node.js Compatibility Mode) + +**Code:** +```typescript +const npmUrl = `npm:ai-sdk-tool-code-execution@0.0.2`; +const module = await import(npmUrl); +``` + +**Error:** +``` +Loading unprepared module: npm:ai-sdk-tool-code-execution@0.0.2 +``` + +**Why it failed:** Deno's npm compatibility requires the package to be "prepared" (downloaded/cached) before import. Dynamic imports of unprepared npm packages fail. + +### Attempt 2: esm.sh with Node.js Target + +**Code:** +```typescript +const esmUrl = `https://esm.sh/ai-sdk-tool-code-execution@0.0.2?target=esnext`; +const module = await import(esmUrl); +``` + +**Error:** +``` +Module not found "https://esm.sh/node:sqlite?target=esnext" + at https://esm.sh/undici@^7.16.0?target=esnext:25:8 +``` + +**Why it failed:** The package code imports `node:sqlite` which esm.sh tries to load from `https://esm.sh/node:sqlite?target=esnext`, but `node:sqlite` is a Node.js built-in, not an npm package. + +### Attempt 3: Multi-Strategy with Fallback + +**Code:** +```typescript +let module; +let importError; + +// Strategy 1: npm: specifier +try { + const npmUrl = `npm:${packageName}@${version}`; + module = await import(npmUrl); +} catch (error) { + importError = error; + + // Strategy 2: esm.sh with esnext target + try { + const esmUrl = `https://esm.sh/${packageName}@${version}?target=esnext`; + module = await import(esmUrl); + } catch (esmError) { + return { success: false, error: esmError.message }; + } +} +``` + +**Result:** Both strategies fail with the same errors as above. + +## Current Deno Configuration + +**`deno.json`:** +```json +{ + "compilerOptions": { + "allowJs": true, + "lib": ["deno.window"], + "strict": true + }, + "nodeModulesDir": true, + "unstable": ["byonm"], + "imports": { + "zod-to-json-schema": "https://esm.sh/zod-to-json-schema@3.25.0" + } +} +``` + +**Key Settings:** +- `nodeModulesDir: true` - Creates `node_modules` directory for npm packages +- `unstable: ["byonm"]` - Enables "Bring Your Own Node Modules" mode + +## Full Error Details + +### npm: Strategy Error +```json +{ + "success": false, + "error": "Failed to import package: ...", + "details": { + "npmError": "Loading unprepared module: npm:ai-sdk-tool-code-execution@0.0.2, imported from: file:///app/server.ts" + } +} +``` + +### esm.sh Strategy Error +```json +{ + "success": false, + "error": "Failed to import package: Module not found \"https://esm.sh/node:sqlite?target=esnext\"", + "details": { + "esmError": "Module not found \"https://esm.sh/node:sqlite?target=esnext\".\n at https://esm.sh/undici@^7.16.0?target=esnext:25:8" + } +} +``` + +## Technical Deep Dive + +### Why This Package Needs Node.js + +1. **better-sqlite3** - Native Node.js addon for SQLite + - Uses `node:sqlite` built-in + - Compiled C++ bindings + - Not available in Deno without Node compatibility layer + +2. **undici** - Modern HTTP client for Node.js + - Uses Node.js streams and buffer APIs + - Optimized for Node.js internals + - May work in Deno with polyfills, but blocked by sqlite dependency + +### Deno's Node.js Compatibility + +Deno supports many Node.js built-ins via `node:*` imports: +- `node:fs`, `node:path`, `node:http`, `node:crypto`, etc. + +**BUT** it does NOT support: +- `node:sqlite` (not a standard Node.js built-in) +- Native addons (`.node` files) +- Some advanced internal APIs + +### The Import Flow + +1. **Deno tries to import** `npm:ai-sdk-tool-code-execution@0.0.2` +2. **Package resolves to** esm.sh or npm registry +3. **Package imports** `better-sqlite3` +4. **better-sqlite3 imports** `node:sqlite` +5. **FAILURE:** `node:sqlite` doesn't exist in Deno or esm.sh + +## Questions for ChatGPT + +1. **Can Deno's npm compatibility layer handle `better-sqlite3` or `node:sqlite`?** + - Is there a Deno-compatible SQLite library we could alias? + - Can we use import maps to redirect `node:sqlite` to a Deno polyfill? + +2. **Can we "prepare" the npm module in Deno before dynamic import?** + - Is there a way to pre-cache npm packages in Deno? + - Can we use `deno vendor` or similar to prepare the package? + +3. **Can esm.sh or other CDNs provide Node.js built-in polyfills?** + - Does esm.sh have a mode that bundles Node.js built-ins? + - Are there CDN parameters we're missing? + +4. **Could we use Deno's `--node-modules-dir` flag differently?** + - Should we install the package via npm/pnpm first? + - Can we point Deno to pre-installed node_modules? + +5. **Is there a way to patch/bundle the package to remove Node.js dependencies?** + - Could we create a Deno-compatible fork? + - Are there tools to transpile Node.js packages to Deno? + +6. **Alternative: Different code execution package?** + - Are there Deno-native code execution tools? + - Could we use WebAssembly or browser-based sandboxing? + +## What Would Success Look Like + +**Ideal outcome:** +```typescript +// This should work in Deno: +const module = await import('npm:ai-sdk-tool-code-execution@0.0.2'); +const { executeCode } = module; + +// And this should execute: +const result = await executeCode.execute({ + code: 'print(fibonacci(10))', + language: 'python' +}); +``` + +**Acceptable outcome:** +```typescript +// Some preparation step, then: +const module = await import('https://esm.sh/ai-sdk-tool-code-execution@0.0.2'); +// Works without errors +``` + +## Repository Context + +**Project:** TPMJS - Tool Package Manager for AI SDK +**Server:** `apps/railway-executor/server.ts` +**Config:** `apps/railway-executor/deno.json` +**Deployment:** Railway with Deno runtime + +**Server Code (Simplified):** +```typescript +async function loadAndDescribe(req: Request): Promise { + const { packageName, exportName, version, importUrl } = await req.json(); + + // Try npm: specifier first + try { + const npmUrl = `npm:${packageName}@${version}`; + const module = await import(npmUrl); + const tool = module[exportName]; + return Response.json({ success: true, tool }); + } catch (error) { + // Try esm.sh fallback + const esmUrl = `https://esm.sh/${packageName}@${version}?target=esnext`; + const module = await import(esmUrl); + const tool = module[exportName]; + return Response.json({ success: true, tool }); + } +} + +Deno.serve({ port: 3001 }, handler); +``` + +## Live Error Logs + +**Request:** +```bash +curl -X POST https://endearing-commitment-production.up.railway.app/load-and-describe \ + -H "Content-Type: application/json" \ + -d '{ + "packageName": "ai-sdk-tool-code-execution", + "exportName": "executeCode", + "version": "0.0.2", + "importUrl": "https://esm.sh/ai-sdk-tool-code-execution@0.0.2" + }' +``` + +**Response:** +```json +{ + "success": false, + "error": "Failed to import package: Module not found \"https://esm.sh/node:sqlite?target=esnext\"", + "details": { + "npmError": "Loading unprepared module: npm:ai-sdk-tool-code-execution@0.0.2", + "esmError": "Module not found \"https://esm.sh/node:sqlite?target=esnext\"" + } +} +``` + +## Additional Context + +- We successfully load other packages (e.g., `@tpmjs/hello`, `zod-to-json-schema`) +- Only packages with Node.js built-in dependencies fail +- Switching to Node.js would work, but we prefer Deno's security model +- This is for a production tool registry serving AI SDK tools to users + +## Related Resources + +- **Deno npm compatibility:** https://deno.com/manual/node/npm_specifiers +- **Deno Node built-ins:** https://deno.com/manual/node/node_specifiers +- **esm.sh documentation:** https://esm.sh/ +- **Package source:** https://www.npmjs.com/package/ai-sdk-tool-code-execution +- **Deno SQLite libraries:** https://deno.land/x/sqlite@v3.8 + +--- + +**Question for ChatGPT:** Is there any way to make `ai-sdk-tool-code-execution` work in Deno, given these constraints? If not, what's the closest alternative that would work in Deno's runtime? diff --git a/DYNAMIC_IMPORT_ISSUE.md b/DYNAMIC_IMPORT_ISSUE.md new file mode 100644 index 0000000..9523485 --- /dev/null +++ b/DYNAMIC_IMPORT_ISSUE.md @@ -0,0 +1,864 @@ +# Dynamic Import Issue: Cannot Import ESM Modules from CDN in Next.js Server-Side API Route + +## Executive Summary + +We're building a dynamic tool loading system where AI agents can discover and load tools at runtime from npm packages via esm.sh CDN. The system successfully searches and finds relevant tools, but fails when trying to dynamically import them using `import()` in a Next.js App Router API route. + +**Error**: `Error: Cannot find module 'unknown'` with code `MODULE_NOT_FOUND` + +**Critical Question**: How can we dynamically import ESM modules from external URLs (like esm.sh) in Next.js 16 App Router API routes running in Node.js runtime? + +--- + +## System Architecture + +### High-Level Flow + +``` +1. User sends message → "use firecrawl to search for ajax davis" +2. Chat API extracts query → "use firecrawl to search for ajax davis" +3. Pre-flight search → Calls searchTpmjsToolsTool.execute({ query, limit: 5 }) +4. Search API returns → Top 5 matching tools from database (BM25-like scoring) +5. Dynamic loading → Tries to import tools from esm.sh URLs ❌ FAILS HERE +6. Agent uses tools → Would pass loaded tools to AI model +``` + +### Tech Stack + +- **Framework**: Next.js 16.0.4 +- **Build Tool**: Turbopack (default in Next.js 15+) +- **Runtime**: Node.js (not edge) +- **Package Manager**: pnpm (monorepo with workspaces) +- **Deployment Target**: Vercel (eventually, currently local dev) +- **AI SDK**: Vercel AI SDK v6.0.0-beta.124 +- **Model**: OpenAI GPT-4o-mini via `streamText()` + +### Monorepo Structure + +``` +tpmjs/ +├── apps/ +│ ├── playground/ # Next.js app with chat interface +│ │ └── src/ +│ │ ├── app/api/chat/route.ts # Where dynamic import fails +│ │ └── lib/dynamic-tool-loader.ts +│ └── web/ # Tool registry website +│ └── src/app/api/tools/search/route.ts +└── packages/ + └── tools/ + ├── hello/ # Static tool (works fine) + └── search-registry/ # Meta-tool for searching registry +``` + +--- + +## Detailed Code Implementation + +### File 1: `apps/playground/src/lib/dynamic-tool-loader.ts` + +**Purpose**: Load tools dynamically from esm.sh CDN + +```typescript +// Cache for imported tool modules (process-level) +const moduleCache = new Map(); + +// Cache for per-conversation active tools +const conversationTools = new Map>(); + +/** + * Generate cache key for a tool + */ +function getCacheKey(packageName: string, exportName: string): string { + return `${packageName}::${exportName}`; +} + +/** + * Validate that an import is a valid AI SDK tool + */ +function isValidTool(value: any): boolean { + return ( + value && + typeof value === 'object' && + typeof value.description === 'string' && + typeof value.execute === 'function' + ); +} + +/** + * Dynamically import a tool from ESM CDN + * + * THIS IS WHERE IT FAILS ❌ + */ +export async function loadToolDynamically( + packageName: string, + exportName: string, + version: string, + importUrl?: string +): Promise { + const cacheKey = getCacheKey(packageName, exportName); + + // Check cache first + if (moduleCache.has(cacheKey)) { + console.log(`✅ Cache hit: ${cacheKey}`); + return moduleCache.get(cacheKey); + } + + // Build import URL + const url = importUrl || `https://esm.sh/${packageName}@${version}`; + + try { + console.log(`📦 Importing: ${url}`); + // Example: https://esm.sh/firecrawl-aisdk@0.7.2 + + // Dynamic import with @vite-ignore to bypass bundler + const module = await import(/* @vite-ignore */ url); + + console.log(`🔍 Module imported successfully`); + console.log(`🔍 Module type: ${typeof module}`); + console.log(`🔍 Module keys: ${Object.keys(module).join(', ')}`); + console.log(`🔍 Looking for export: "${exportName}"`); + console.log(`🔍 Export exists: ${exportName in module}`); + console.log(`🔍 Export type: ${typeof module[exportName]}`); + + // Get the specific export + const tool = module[exportName]; + + if (!tool) { + console.error(`❌ Export "${exportName}" not found in module. Available exports:`, Object.keys(module)); + return null; + } + + console.log(`🔍 Tool structure:`, { + hasDescription: 'description' in tool, + hasExecute: 'execute' in tool, + hasInputSchema: 'inputSchema' in tool, + keys: Object.keys(tool), + }); + + if (!isValidTool(tool)) { + console.error(`❌ Invalid tool structure: ${exportName} from ${packageName}`); + console.error(` Tool:`, tool); + return null; + } + + // Cache successful import + moduleCache.set(cacheKey, tool); + console.log(`✅ Loaded: ${cacheKey}`); + + return tool; + } catch (error) { + console.error(`❌ Failed to load ${packageName}#${exportName}:`, error); + console.error(` URL: ${url}`); + console.error(` Stack:`, error instanceof Error ? error.stack : 'No stack trace'); + return null; + } +} + +/** + * Load multiple tools in parallel + */ +export async function loadToolsBatch( + toolMetadata: Array<{ + packageName: string; + exportName: string; + version: string; + importUrl?: string; + }> +): Promise> { + const promises = toolMetadata.map((meta) => + loadToolDynamically( + meta.packageName, + meta.exportName, + meta.version, + meta.importUrl + ).then((tool) => ({ + key: getCacheKey(meta.packageName, meta.exportName), + tool, + })) + ); + + const results = await Promise.all(promises); + + const tools: Record = {}; + for (const { key, tool } of results) { + if (tool) { + tools[key] = tool; + } + } + + return tools; +} +``` + +### File 2: `apps/playground/src/app/api/chat/route.ts` + +**Purpose**: Main chat API that orchestrates tool discovery and loading + +```typescript +import { createOpenAI } from '@ai-sdk/openai'; +import { type UIMessage, convertToModelMessages, stepCountIs, streamText } from 'ai'; +import type { NextRequest } from 'next/server'; +import { NextResponse } from 'next/server'; +import { env } from '~/env'; +import { loadAllTools, sanitizeToolName } from '~/lib/tool-loader'; +import { searchTpmjsToolsTool } from '@tpmjs/search-registry'; +import { + loadToolsBatch, + addConversationTools, +} from '~/lib/dynamic-tool-loader'; + +export const runtime = 'nodejs'; // ⚠️ Important: We're using Node.js runtime, not edge +export const dynamic = 'force-dynamic'; +export const maxDuration = 60; + +// Initialize OpenAI provider +const openai = createOpenAI({ + apiKey: env.OPENAI_API_KEY, +}); + +// Add conversation state tracking (in-memory for MVP) +const conversationStates = new Map }>(); + +/** + * POST /api/chat + * Chat with AI agent that can execute TPMJS tools + */ +export async function POST(request: NextRequest) { + try { + const body = await request.json(); + console.log('📥 Request body:', JSON.stringify(body, null, 2)); + + const messages: UIMessage[] = body.messages || []; + const conversationId: string = body.conversationId || 'default'; + + console.log(`🔑 Conversation ID: ${conversationId}`); + + // Get or create conversation state + if (!conversationStates.has(conversationId)) { + console.log('✨ Creating new conversation state'); + conversationStates.set(conversationId, { loadedTools: {} }); + } + const state = conversationStates.get(conversationId)!; + console.log(`📊 Current loaded tools in conversation: ${Object.keys(state.loadedTools).length}`); + + // 1. Load static tools + search tool + const staticTools = await loadAllTools(); + console.log(`🔧 Loaded ${Object.keys(staticTools).length} static tools`); + + staticTools.searchTpmjsTools = searchTpmjsToolsTool; + console.log('✅ Added searchTpmjsTools to static tools'); + + // 2. Extract user query from last message for tool search + const lastMessage = messages[messages.length - 1]; + let userQuery = ''; + if (lastMessage?.role === 'user') { + const parts = (lastMessage as any).parts || []; + for (const part of parts) { + if (part.type === 'text') { + userQuery = part.text; + break; + } + } + } + + console.log(`💬 User query: "${userQuery}"`); + + // 3. Automatically search for relevant tools based on the user's message + if (userQuery && userQuery.trim().length > 0) { + console.log('🔎 Searching for relevant tools...'); + + try { + const searchResult = await searchTpmjsToolsTool.execute({ + query: userQuery, + limit: 5, // Get top 5 relevant tools + }, {} as any); + + console.log(`📦 Found ${searchResult.matchCount} matching tools`); + + if (searchResult.tools && searchResult.tools.length > 0) { + console.log(`🔧 Tools found:`, searchResult.tools.map((t: any) => `${t.packageName}/${t.exportName}`)); + + // Dynamically load tools from esm.sh + console.log(`📥 Loading ${searchResult.tools.length} tools dynamically...`); + + const toolsToLoad = searchResult.tools.map((meta: any) => ({ + packageName: meta.packageName, + exportName: meta.exportName, + version: meta.version, + importUrl: meta.importUrl, + })); + + try { + // ❌ THIS IS WHERE IT FAILS + const loadedTools = await loadToolsBatch(toolsToLoad); + console.log(`✅ Successfully loaded ${Object.keys(loadedTools).length} tools`); + + // Add sanitized tools to conversation state + for (const [key, tool] of Object.entries(loadedTools)) { + const [pkg, exp] = key.split('::'); + const sanitizedKey = sanitizeToolName(`${pkg}-${exp}`); + state.loadedTools[sanitizedKey] = tool; + console.log(`✅ Added to conversation: ${sanitizedKey}`); + } + + // Track for this conversation + addConversationTools(conversationId, Object.keys(state.loadedTools)); + } catch (error) { + console.error('❌ Error loading tools:', error); + } + } else { + console.log('ℹ️ No matching tools found for this query'); + } + } catch (error) { + console.error('❌ Error searching for tools:', error); + } + } + + // 4. Merge with conversation's dynamically loaded tools + const allTools: Record = { ...staticTools, ...state.loadedTools }; + + // 5. Build system prompt with available tools + const toolsList = Object.keys(allTools) + .map((name) => { + const tool = allTools[name] as { description?: string } | undefined; + return `- ${name}: ${tool?.description || 'No description'}`; + }) + .join('\n'); + + const system = `You are a helpful AI assistant that can use TPMJS tools to help users. + +Available tools: +${toolsList} + +When you use a tool, you MUST always follow up with a natural language answer to the user summarizing the result.`; + + // 6. Stream response with all available tools + const result = streamText({ + model: openai('gpt-4o-mini'), + system, + messages: convertToModelMessages(messages), + tools: allTools, + stopWhen: stepCountIs(5), + }); + + return result.toUIMessageStreamResponse(); + } catch (error) { + console.error('Chat API error:', error); + + return new Response( + JSON.stringify({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }), + { + status: 500, + headers: { 'Content-Type': 'application/json' }, + } + ); + } +} +``` + +### File 3: Example Tool Metadata (from search API) + +When we search for "firecrawl", the search API returns: + +```json +{ + "success": true, + "query": "firecrawl ajax davis", + "results": { + "total": 29, + "returned": 5, + "tools": [ + { + "id": "cm4abc123", + "exportName": "searchTool", + "description": "Search the web using Firecrawl's search API", + "qualityScore": 0.85, + "package": { + "npmPackageName": "firecrawl-aisdk", + "npmVersion": "0.7.2", + "category": "web-scraping", + "frameworks": ["vercel-ai"], + "env": "server" + }, + "importUrl": "https://esm.sh/firecrawl-aisdk@0.7.2", + "cdnUrl": "https://cdn.jsdelivr.net/npm/firecrawl-aisdk@0.7.2/+esm" + } + ] + } +} +``` + +So we're trying to: +```typescript +const module = await import('https://esm.sh/firecrawl-aisdk@0.7.2'); +const tool = module.searchTool; // Get the exported tool +``` + +--- + +## The Error + +### Console Output + +``` +📥 Loading 5 tools dynamically... +📦 Importing: https://esm.sh/firecrawl-aisdk@0.7.2 +❌ Failed to load firecrawl-aisdk#searchTool: Error: Cannot find module 'unknown' + at (.next/dev/server/chunks/[root-of-the-server]__746deca2._.js:357:23) + at loadToolDynamically (.next/dev/server/chunks/[root-of-the-server]__746deca2._.js:360:11) + at (src/lib/dynamic-tool-loader.ts:108:5) + at Array.map () + at loadToolsBatch (src/lib/dynamic-tool-loader.ts:107:33) + at POST (src/app/api/chat/route.ts:104:53) + { + code: 'MODULE_NOT_FOUND' + } + URL: https://esm.sh/firecrawl-aisdk@0.7.2 + Stack: Error: Cannot find module 'unknown' + at /Users/ajaxdavis/repos/tpmjs/tpmjs/apps/playground/.next/dev/server/chunks/[root-of-the-server]__746deca2._.js:357:23 + at loadToolDynamically (/Users/ajaxdavis/repos/tpmjs/tpmjs/apps/playground/.next/dev/server/chunks/[root-of-the-server]__746deca2._.js:360:11) +``` + +### Key Observations + +1. **Error happens immediately** - Never gets to our debug logs after `await import()` +2. **Error is MODULE_NOT_FOUND** - Treating URL as a module path +3. **Error says "unknown"** - Not even using the actual module name +4. **Code is in .next/dev/server/chunks/** - Next.js/Turbopack transformed our code +5. **Same error for all packages** - firecrawl-aisdk, @exalabs/ai-sdk, etc. + +--- + +## Verification: The URL Works + +### Manual Test 1: Browser + +``` +Visit: https://esm.sh/firecrawl-aisdk@0.7.2 +``` + +Returns valid ESM module: +```javascript +/* esm.sh - firecrawl-aisdk@0.7.2 */ +import * as __1$ from "/v135/@ai-sdk/provider-utils@2.0.8/..."; +// ... rest of module code +export { searchTool, scrapeTool, crawlTool }; +``` + +### Manual Test 2: Plain Node.js Script + +Create `test-import.mjs`: +```javascript +const module = await import('https://esm.sh/firecrawl-aisdk@0.7.2'); +console.log('Module:', module); +console.log('Exports:', Object.keys(module)); +``` + +Run: `node test-import.mjs` + +**Expected**: Would work in plain Node.js with `--experimental-network-imports` flag +**In Next.js**: Can't even get this far + +--- + +## What We've Tried + +### Attempt 1: `/* @vite-ignore */` Comment +```typescript +const module = await import(/* @vite-ignore */ url); +``` +**Result**: Still fails with MODULE_NOT_FOUND + +### Attempt 2: `/* webpackIgnore: true */` Comment +```typescript +const module = await import(/* webpackIgnore: true */ url); +``` +**Result**: Still fails with MODULE_NOT_FOUND + +### Attempt 3: Force Dynamic Runtime +```typescript +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; +``` +**Result**: Still fails (we're already using this) + +### Attempt 4: Verify esm.sh Works +- Tested URLs in browser: ✅ Works +- All packages return valid ESM: ✅ Valid +- esm.sh is accessible: ✅ Reachable + +### Attempt 5: Check Static Imports +```typescript +import { helloWorldTool } from '@tpmjs/hello'; +``` +**Result**: Works perfectly (but bundled at build time) + +--- + +## Configuration Files + +### `apps/playground/next.config.ts` + +```typescript +import type { NextConfig } from 'next'; + +const config: NextConfig = { + reactStrictMode: true, + transpilePackages: ['@tpmjs/ui'], + experimental: { + turbo: { + // Using Turbopack (Next.js 15+ default) + }, + }, +}; + +export default config; +``` + +### `apps/playground/package.json` (relevant parts) + +```json +{ + "name": "@tpmjs/playground", + "version": "0.0.0", + "private": true, + "scripts": { + "dev": "next dev --port 3001", + "build": "next build", + "start": "next start" + }, + "dependencies": { + "@ai-sdk/openai": "^1.0.15", + "@tpmjs/hello": "workspace:*", + "@tpmjs/search-registry": "workspace:*", + "ai": "6.0.0-beta.124", + "next": "16.0.4", + "react": "19.0.0" + } +} +``` + +### `turbo.json` (monorepo config) + +```json +{ + "$schema": "https://turbo.build/schema.json", + "tasks": { + "dev": { + "cache": false, + "persistent": true + }, + "build": { + "dependsOn": ["^build"], + "outputs": [".next/**", "dist/**"] + } + } +} +``` + +--- + +## Why This Matters + +### The Bigger Picture + +We're building a **self-referential tool discovery system**: + +1. **Tool Registry** (tpmjs.com) - Indexes all TPMJS-compatible tools from npm +2. **Search Tool** - AI SDK tool that searches the registry +3. **Dynamic Loader** - Loads found tools at runtime +4. **AI Agent** - Uses dynamically loaded tools + +This creates infinite extensibility: +- No need to bundle all possible tools +- Tools can be published to npm independently +- System discovers and loads tools as needed +- Bundle size stays small + +### Use Case Example + +``` +User: "Search Wikipedia for quantum computing" + ↓ +System searches registry: Finds "wikipedia-aisdk" tool + ↓ +System loads tool: import('https://esm.sh/wikipedia-aisdk@1.0.0') + ↓ +AI uses tool: wikipediaSearchTool.execute({ query: "quantum computing" }) + ↓ +User gets answer with Wikipedia citations +``` + +--- + +## Possible Root Causes + +### Hypothesis 1: Turbopack Doesn't Support Dynamic Import URLs +- Turbopack intercepts all `import()` calls +- Transforms them to module resolution +- Doesn't handle external URLs + +### Hypothesis 2: Next.js Security Restriction +- Next.js blocks dynamic imports from external URLs for security +- Prevents arbitrary code execution +- No way to whitelist esm.sh + +### Hypothesis 3: Dev Mode Only Issue +- Turbopack dev mode has more restrictions +- Production webpack build might work +- But we need dev mode to work too + +### Hypothesis 4: Node.js Runtime Limitation in Next.js +- Next.js Node.js runtime is sandboxed +- Dynamic imports are intercepted before reaching Node.js +- Plain Node.js would work with --experimental-network-imports + +--- + +## Alternative Approaches We're Considering + +### Option A: Fetch + VM Module +```typescript +import { SourceTextModule } from 'vm'; + +const response = await fetch(url); +const code = await response.text(); +const module = new SourceTextModule(code); +await module.link(() => {}); +await module.evaluate(); +const exports = module.namespace; +``` + +**Pros**: Bypasses import() entirely +**Cons**: Complex, security concerns, might not work in Next.js + +### Option B: Separate Microservice +```typescript +// New service: tool-loader-service (Express or Fastify) +POST /load-tool +Body: { packageName, exportName, version } +Response: { tool: } +``` + +**Pros**: Full control, definitely works +**Cons**: Extra infrastructure, latency, complexity + +### Option C: Switch to Edge Runtime +```typescript +export const runtime = 'edge'; // Instead of 'nodejs' +``` + +**Pros**: Edge might have different import behavior +**Cons**: Edge has limitations (no Node.js APIs), might still not work + +### Option D: Pre-bundle Common Tools +```typescript +// Generate static imports for top 100 tools +import { tool1 } from 'package1'; +import { tool2 } from 'package2'; +// ... etc +``` + +**Pros**: Definitely works +**Cons**: Defeats the purpose, huge bundle size + +### Option E: Use unpkg or jsdelivr with Different Strategy +```typescript +// Fetch raw code, eval in isolated context +const response = await fetch(`https://unpkg.com/${pkg}@${ver}/dist/index.mjs`); +const code = await response.text(); +const exports = evalInContext(code); +``` + +**Pros**: More control +**Cons**: Same security/execution issues + +--- + +## Specific Questions for ChatGPT + +### Question 1: Is This Possible? +**Can Next.js 16 App Router API routes (Node.js runtime) dynamically import ESM modules from external URLs using `import()`?** + +If yes: +- What configuration is needed? +- Are there security allowlists? +- Does it work in both dev and production? + +If no: +- Why not? +- What's the recommended alternative? +- Is this a Turbopack limitation or Next.js design? + +### Question 2: Turbopack Behavior +**Does Turbopack intercept all `import()` calls, even with magic comments?** + +We've tried: +- `/* @vite-ignore */` +- `/* webpackIgnore: true */` + +None work. Is there a Turbopack-specific comment or config? + +### Question 3: Edge vs Node Runtime +**Would switching to edge runtime change import behavior?** + +```typescript +export const runtime = 'edge'; // vs 'nodejs' +``` + +Does edge runtime allow dynamic imports from URLs? + +### Question 4: Best Practice +**What's the recommended way to implement dynamic tool loading in Next.js?** + +Given constraints: +- Need to load arbitrary npm packages at runtime +- Packages are ESM modules from CDN +- Can't pre-bundle all possibilities +- Need to work in production on Vercel + +### Question 5: Security Model +**Is Next.js intentionally blocking this for security?** + +- Is there a whitelist for allowed CDNs? +- Can we configure allowed import sources? +- Is this related to CSP or other security headers? + +--- + +## Environment Details + +### Versions +```json +{ + "next": "16.0.4", + "react": "19.0.0", + "turbo": "2.6.1", + "pnpm": "9.15.0", + "node": "v20.11.0", + "ai": "6.0.0-beta.124" +} +``` + +### Operating System +- **OS**: macOS (Darwin 23.5.0) +- **Architecture**: arm64 (Apple Silicon) + +### Development Commands +```bash +# Start dev server +pnpm dev --filter=@tpmjs/playground + +# Output +▲ Next.js 16.0.4 (Turbopack) +- Local: http://localhost:3001 +- Network: http://192.168.0.25:3001 +✓ Ready in 2.5s +``` + +### Build Output Structure +``` +apps/playground/.next/ +├── dev/ +│ └── server/ +│ └── chunks/ +│ └── [root-of-the-server]__746deca2._.js # ← Error originates here +``` + +--- + +## Success Criteria + +### What We Need Working + +```typescript +// In Next.js API route (Node.js runtime) +const url = 'https://esm.sh/firecrawl-aisdk@0.7.2'; +const module = await import(url); +const tool = module.searchTool; + +console.log(tool.description); // "Search the web using Firecrawl's search API" +console.log(typeof tool.execute); // "function" + +// Tool is ready to use with AI SDK +const result = await tool.execute({ query: "test" }, context); +``` + +### Acceptable Outcomes + +1. ✅ **Best**: Dynamic `import()` works with configuration change +2. ✅ **Good**: Alternative approach that doesn't require microservice +3. ✅ **Acceptable**: Workaround that works in production even if dev is tricky +4. ❌ **Unacceptable**: "You can't do this in Next.js" without alternative + +--- + +## Additional Context + +### Why Not Just Bundle Everything? + +Currently have ~30 tools in registry, growing to 100s or 1000s: +- Bundle size would be massive (10+ MB) +- Most tools won't be used in most conversations +- Tools are published independently by community +- Want instant availability of new tools without redeploying + +### Why esm.sh Specifically? + +- ✅ Converts any npm package to ESM +- ✅ Handles dependencies automatically +- ✅ Fast CDN with caching +- ✅ No build step required +- ✅ Version pinning built-in + +But we're flexible - if jsdelivr, unpkg, or another approach works better, we'll use it. + +### Static Imports Work Fine + +This works perfectly (but defeats the purpose): +```typescript +import { searchTool } from 'firecrawl-aisdk'; +``` + +The tools themselves are fine. We just can't load them dynamically. + +--- + +## What We're Hoping For + +### Ideal Answer Format + +1. **Root cause**: Why it's failing +2. **Solution**: How to fix it (with code example) +3. **Configuration**: Any Next.js config needed +4. **Limitations**: What won't work / tradeoffs +5. **Alternatives**: If dynamic import truly impossible + +### We're Happy to Try + +- Different CDN (unpkg, jsdelivr, etc.) +- Different import strategy (fetch + eval, vm module, etc.) +- Different runtime (edge if it works) +- Different Next.js version (if specific version supports this) +- Webpack instead of Turbopack (if webpack handles this better) + +We just need a path forward that enables runtime tool loading in a production Next.js app on Vercel. + +--- + +## Files to Reference + +All code is in this monorepo: +- `apps/playground/src/lib/dynamic-tool-loader.ts` - Import logic +- `apps/playground/src/app/api/chat/route.ts` - API route +- `apps/playground/next.config.ts` - Next.js config +- `DYNAMIC_IMPORT_ISSUE.md` - This document + +--- + +## Thank You + +This is a critical blocker for our dynamic tool loading system. Any insights, workarounds, or alternative approaches would be immensely helpful! diff --git a/DYNAMIC_TOOL_LOADING_PRD.md b/DYNAMIC_TOOL_LOADING_PRD.md new file mode 100644 index 0000000..f777951 --- /dev/null +++ b/DYNAMIC_TOOL_LOADING_PRD.md @@ -0,0 +1,988 @@ +# Dynamic Tool Loading System - Product Requirements Document + +## Executive Summary + +Build a self-referential tool discovery system where AI agents can search the TPMJS registry, find relevant tools, and dynamically import them during conversation. This creates a "meta-tool" that makes the entire TPMJS ecosystem available to any agent at runtime. + +**Core Innovation:** An AI agent can discover and load tools on-demand by searching the registry, rather than having all tools pre-loaded. This enables infinite tool extensibility without bundle size concerns. + +--- + +## Problem Statement + +### Current Limitations + +1. **Static Tool Loading**: Playground requires all tools to be hardcoded in `tool-loader.ts` +2. **Bundle Size**: Loading many tools increases bundle size and initialization time +3. **Discovery Gap**: Agents can't discover new tools that match their current task +4. **Manual Updates**: Adding tools requires code changes and redeployment + +### User Pain Points + +- Users want agents to access the full TPMJS registry without manual configuration +- Developers want to publish tools that are immediately available to all agents +- Agents need context-aware tool selection based on the conversation + +--- + +## Solution Overview + +### The Meta-Tool: `searchTpmjsTools` + +A TPMJS tool that searches the TPMJS registry and returns tool metadata needed for dynamic import. + +**Flow:** +``` +User: "Search Wikipedia for quantum computing" + ↓ +Agent: Calls searchTpmjsTools("wikipedia search") + ↓ +API: Returns tools matching "wikipedia" (BM25 search) + ↓ +Playground: Dynamically imports matching tools + ↓ +Agent: Now has Wikipedia tools available, uses them +``` + +### Key Components + +1. **`@tpmjs/search-registry`** - NPM package exporting `searchTpmjsToolsTool` +2. **`/api/tools/search`** - New API endpoint with BM25 full-text search +3. **Playground Dynamic Loader** - Runtime tool import system +4. **Tool Import Strategy** - ESM CDN imports or bundled approach + +--- + +## Technical Architecture + +### Component 1: Search Tool Package + +**Package:** `packages/tools/search-registry/` + +```typescript +// packages/tools/search-registry/src/index.ts +import { tool } from 'ai'; +import { z } from 'zod'; + +export const searchTpmjsToolsTool = tool({ + description: 'Search the TPMJS tool registry to find AI SDK tools. Use this when you need a tool that isn\'t currently available. Returns tool metadata including package names and descriptions.', + parameters: z.object({ + query: z.string().describe('Search query (e.g., "weather", "database", "wikipedia")'), + category: z.enum([ + 'text-analysis', + 'code-generation', + 'data-processing', + 'image-generation', + 'audio-processing', + 'search', + 'integration', + 'other' + ]).optional().describe('Filter by tool category'), + limit: z.number().min(1).max(20).default(10).describe('Max number of tools to return'), + }), + execute: async ({ query, category, limit }) => { + // Call TPMJS search API + const params = new URLSearchParams({ + q: query, + limit: String(limit), + ...(category && { category }), + }); + + const response = await fetch( + `https://tpmjs.com/api/tools/search?${params}` + ); + + if (!response.ok) { + throw new Error(`Search failed: ${response.statusText}`); + } + + const data = await response.json(); + + // Return structured tool metadata + return { + query, + matchCount: data.tools.length, + tools: data.tools.map((tool: any) => ({ + packageName: tool.package.npmPackageName, + exportName: tool.exportName, + description: tool.description, + category: tool.package.category, + qualityScore: tool.qualityScore, + frameworks: tool.package.frameworks, + env: tool.package.env, + })), + }; + }, +}); +``` + +**Package Metadata:** + +```json +{ + "name": "@tpmjs/search-registry", + "version": "0.1.0", + "description": "AI SDK tool for searching the TPMJS tool registry", + "keywords": ["tpmjs-tool", "ai", "search"], + "tpmjs": { + "category": "search", + "frameworks": ["vercel-ai"], + "tools": [ + { + "exportName": "searchTpmjsToolsTool", + "description": "Search the TPMJS tool registry to find AI SDK tools by keyword, category, or description. Returns tool metadata for dynamic loading.", + "parameters": [ + { + "name": "query", + "type": "string", + "description": "Search query (keywords, tool names, descriptions)", + "required": true + }, + { + "name": "category", + "type": "string", + "description": "Filter by category (text-analysis, search, etc.)", + "required": false + }, + { + "name": "limit", + "type": "number", + "description": "Maximum number of results (1-20, default 10)", + "required": false + } + ], + "returns": { + "type": "object", + "description": "Search results with tool metadata for dynamic import" + }, + "aiAgent": { + "useCase": "Use this tool when you need a tool that isn't currently available. For example, if asked to search Wikipedia but you don't have a Wikipedia tool, search for 'wikipedia' to find and load it.", + "examples": [ + "Search for 'weather' tools when asked about weather", + "Search for 'database' tools when working with data", + "Search for 'code' tools when generating code" + ], + "limitations": "Returns metadata only - the playground handles actual tool loading" + } + } + ] + } +} +``` + +--- + +### Component 2: BM25 Search API Endpoint + +**File:** `apps/web/src/app/api/tools/search/route.ts` + +**Requirements:** + +1. **Full-Text Search with BM25** + - Search across: tool description, package name, npm description, npm keywords + - BM25 scoring for relevance ranking + - Category filtering + - Quality score boosting (rich tier tools rank higher) + +2. **Search Implementation Options** + + **Option A: PostgreSQL Full-Text Search** + ```sql + -- Add tsvector column to tools table + ALTER TABLE tools ADD COLUMN search_vector tsvector; + + -- Create GIN index for fast full-text search + CREATE INDEX tools_search_idx ON tools USING GIN(search_vector); + + -- Update search vector on insert/update + CREATE TRIGGER tools_search_update + BEFORE INSERT OR UPDATE ON tools + FOR EACH ROW EXECUTE FUNCTION + tsvector_update_trigger(search_vector, 'pg_catalog.english', + description); + ``` + + **Option B: JavaScript BM25 Library** + ```typescript + import { BM25 } from 'bm25'; + + // Load all tools into memory (cached) + const tools = await prisma.tool.findMany({ + include: { package: true }, + }); + + // Build BM25 index + const documents = tools.map(tool => ({ + id: tool.id, + text: `${tool.description} ${tool.package.npmPackageName} ${tool.package.npmDescription} ${tool.package.npmKeywords.join(' ')}`, + })); + + const bm25 = new BM25(documents); + const results = bm25.search(query); + ``` + + **Option C: Hybrid Approach** + - Use PostgreSQL `LIKE` for exact matches (fastest) + - Fall back to BM25 for fuzzy/semantic search + - Cache search results in Redis + +3. **API Response Format** + +```typescript +// GET /api/tools/search?q=weather&category=integration&limit=10 + +{ + "success": true, + "query": "weather", + "filters": { + "category": "integration" + }, + "results": { + "total": 23, + "returned": 10, + "tools": [ + { + "id": "clx...", + "exportName": "getWeatherTool", + "description": "Get current weather data for any location using OpenWeatherMap API", + "qualityScore": 0.85, + "package": { + "npmPackageName": "@tpmjs/weather", + "npmVersion": "1.2.0", + "category": "integration", + "frameworks": ["vercel-ai"], + "env": [ + { + "name": "OPENWEATHER_API_KEY", + "description": "OpenWeatherMap API key", + "required": true + } + ], + "npmRepository": { + "type": "git", + "url": "https://github.com/user/weather-tool" + }, + "isOfficial": false + }, + // Include everything needed for dynamic import + "importUrl": "https://esm.sh/@tpmjs/weather@1.2.0", + "cdnUrl": "https://cdn.jsdelivr.net/npm/@tpmjs/weather@1.2.0/+esm" + } + // ... more tools + ] + } +} +``` + +--- + +### Component 3: Dynamic Tool Loader (Playground) + +**File:** `apps/playground/src/lib/dynamic-tool-loader.ts` + +**Requirements:** + +1. **Runtime ESM Import** + ```typescript + async function loadToolDynamically( + packageName: string, + exportName: string, + version: string + ) { + // Option 1: ESM CDN (esm.sh, unpkg, jsdelivr) + const cdnUrl = `https://esm.sh/${packageName}@${version}`; + + try { + const module = await import(/* @vite-ignore */ cdnUrl); + const tool = module[exportName]; + + if (!isValidTool(tool)) { + throw new Error(`Invalid tool: ${exportName}`); + } + + return tool; + } catch (error) { + console.error(`Failed to load ${packageName}:`, error); + return null; + } + } + ``` + +2. **Tool Caching Strategy** + ```typescript + // Cache loaded tools to avoid redundant imports + const toolCache = new Map(); + + function getCacheKey(packageName: string, exportName: string): string { + return `${packageName}::${exportName}`; + } + + async function loadToolWithCache( + packageName: string, + exportName: string, + version: string + ) { + const key = getCacheKey(packageName, exportName); + + if (toolCache.has(key)) { + return toolCache.get(key); + } + + const tool = await loadToolDynamically(packageName, exportName, version); + + if (tool) { + toolCache.set(key, tool); + } + + return tool; + } + ``` + +3. **Tool Registry Integration** + ```typescript + // Merge static tools + dynamically loaded tools + async function getAllAvailableTools( + staticTools: Record, + searchResults: SearchResult[] + ): Promise> { + const allTools = { ...staticTools }; + + // Load tools from search results + for (const result of searchResults) { + const tool = await loadToolWithCache( + result.package.npmPackageName, + result.exportName, + result.package.npmVersion + ); + + if (tool) { + const key = sanitizeToolName( + `${result.package.npmPackageName}-${result.exportName}` + ); + allTools[key] = tool; + } + } + + return allTools; + } + ``` + +--- + +### Component 4: Playground Chat Integration + +**File:** `apps/playground/src/app/api/chat/route.ts` + +**Flow:** + +1. **Initial Tool Set** + - Load static tools (hardcoded in tool-loader) + - Always include `searchTpmjsToolsTool` in initial set + +2. **Agent Invokes Search** + - Agent calls `searchTpmjsToolsTool` with query + - Search API returns matching tool metadata + - Response includes tool metadata + +3. **Dynamic Loading Trigger** + - Detect when agent successfully calls `searchTpmjsToolsTool` + - Extract tool metadata from response + - Load tools dynamically before next agent turn + +4. **Tool Availability Update** + - Merge dynamically loaded tools into available tool set + - Agent can now use newly loaded tools in subsequent turns + +**Implementation:** + +```typescript +// apps/playground/src/app/api/chat/route.ts +export async function POST(req: Request) { + const { messages } = await req.json(); + + // 1. Load static tools + search tool + let availableTools = await loadAllTools(); // static + availableTools['searchTpmjsTools'] = searchTpmjsToolsTool; // meta-tool + + // 2. Create streamText with current tools + const result = streamText({ + model: openai('gpt-4'), + messages, + tools: availableTools, + maxSteps: 10, // Allow multiple tool call rounds + + onStepFinish: async (step) => { + // 3. Check if agent called searchTpmjsToolsTool + for (const toolCall of step.toolCalls) { + if (toolCall.toolName === 'searchTpmjsTools') { + const searchResults = toolCall.result?.tools || []; + + // 4. Dynamically load tools from search results + console.log(`Loading ${searchResults.length} tools dynamically...`); + + for (const toolMeta of searchResults) { + const tool = await loadToolWithCache( + toolMeta.packageName, + toolMeta.exportName, + 'latest' // or toolMeta.version + ); + + if (tool) { + const key = sanitizeToolName( + `${toolMeta.packageName}-${toolMeta.exportName}` + ); + availableTools[key] = tool; + console.log(`✅ Loaded: ${key}`); + } + } + + // 5. Update tool registry for subsequent steps + // Note: This requires AI SDK to support dynamic tool updates + // May need to restart the streamText with updated tools + } + } + }, + }); + + return result.toDataStreamResponse(); +} +``` + +--- + +## Technical Challenges & Solutions + +### Challenge 1: AI SDK Doesn't Support Dynamic Tool Updates Mid-Stream + +**Problem:** Vercel AI SDK's `streamText` sets tools at initialization. Can't add tools after streaming starts. + +**Solutions:** + +**Option A: Multi-Turn Pattern** +```typescript +// Turn 1: Agent searches for tools +// Turn 2: Agent uses loaded tools + +// Detect search tool call, return early +if (hasSearchToolCall) { + return new Response(JSON.stringify({ + type: 'tools_loaded', + tools: searchResults, + message: 'Tools loaded. Please continue your request.', + })); +} +``` + +**Option B: Pre-Flight Search (Recommended)** +```typescript +// Before calling streamText, analyze user message +const needsTools = await analyzeMessageForToolNeeds(userMessage); + +if (needsTools.length > 0) { + // Pre-load tools based on intent + const searchResults = await searchTools(needsTools); + const dynamicTools = await loadToolsFromResults(searchResults); + availableTools = { ...staticTools, ...dynamicTools }; +} + +// Now call streamText with full tool set +const result = streamText({ + model, + messages, + tools: availableTools, +}); +``` + +**Option C: Agent-Driven Two-Phase** +```typescript +// Phase 1: Planning +const planResult = await generateText({ + model, + messages: [ + { role: 'system', content: 'Analyze this request and determine what tools are needed. Call searchTpmjsTools if needed.' }, + ...messages, + ], + tools: { searchTpmjsTools }, +}); + +// Phase 2: Execution with loaded tools +const executionResult = await streamText({ + model, + messages, + tools: { ...staticTools, ...loadedTools }, +}); +``` + +--- + +### Challenge 2: ESM Dynamic Import in Browser vs Node.js + +**Problem:** Dynamic `import()` works differently in browser vs server environments. + +**Solutions:** + +**Server-Side (Recommended):** +```typescript +// Use Node.js dynamic import +// Works with esm.sh CDN +const tool = await import(`https://esm.sh/${pkg}@${version}`); +``` + +**Client-Side (Avoid):** +```typescript +// Browser import() has CORS and CSP restrictions +// Would require: +// 1. CDN supports CORS +// 2. CSP allows script-src from CDN +// 3. Tools are browser-compatible (no Node.js APIs) +``` + +**Hybrid Approach:** +```typescript +// Load tools server-side, serialize to client +// Client displays available tools +// Server executes tool calls +``` + +--- + +### Challenge 3: Tool Dependencies & Environment Variables + +**Problem:** Dynamically loaded tools may require: +- Environment variables (API keys) +- npm dependencies not in bundle +- Node.js-specific APIs + +**Solutions:** + +**Option A: Require Pre-Configuration** +```typescript +// Before loading, check if tool requirements are met +async function canLoadTool(toolMeta: ToolMetadata): Promise { + // Check required env vars + for (const env of toolMeta.package.env || []) { + if (env.required && !process.env[env.name]) { + console.warn(`Missing required env: ${env.name}`); + return false; + } + } + + return true; +} +``` + +**Option B: Graceful Degradation** +```typescript +// Load tool, catch errors, inform agent +try { + const tool = await loadTool(packageName, exportName); + return tool; +} catch (error) { + return createStubTool(packageName, exportName, error); +} + +function createStubTool(pkg: string, exp: string, error: Error) { + return tool({ + description: `[UNAVAILABLE] ${exp} from ${pkg}: ${error.message}`, + parameters: z.object({}), + execute: async () => { + throw new Error(`Cannot execute ${exp}: ${error.message}`); + }, + }); +} +``` + +**Option C: Proxy Through Server** +```typescript +// All tools execute server-side where env vars exist +// Client just displays tool calls, server handles execution +``` + +--- + +### Challenge 4: Security & Sandboxing + +**Problem:** Dynamically importing arbitrary npm packages is a security risk. + +**Solutions:** + +**Option A: Allowlist Only** +```typescript +// Only load tools from TPMJS registry (already vetted) +const allowedPackages = await prisma.package.findMany({ + select: { npmPackageName: true } +}); + +if (!allowedPackages.includes(packageName)) { + throw new Error('Package not in TPMJS registry'); +} +``` + +**Option B: Version Pinning** +```typescript +// Only load specific versions from registry +// Don't use 'latest' to avoid supply chain attacks +const version = toolMeta.package.npmVersion; // e.g., "1.2.0" +const url = `https://esm.sh/${pkg}@${version}`; +``` + +**Option C: VM Sandbox (Advanced)** +```typescript +// Execute tools in isolated VM context +import { VM } from 'vm2'; + +const vm = new VM({ + timeout: 5000, + sandbox: { + fetch: safeFetch, // Wrapped fetch with rate limits + console: safeConsole, + }, +}); + +const tool = vm.run(toolCode); +``` + +--- + +### Challenge 5: Performance & Bundle Size + +**Problem:** Loading many tools dynamically could be slow. + +**Solutions:** + +**Option A: Lazy Loading** +```typescript +// Only load tools when agent decides to use them +// Not when they're discovered +``` + +**Option B: Parallel Loading** +```typescript +// Load multiple tools concurrently +const toolPromises = searchResults.map(result => + loadToolWithCache(result.package.npmPackageName, result.exportName, result.package.npmVersion) +); + +const tools = await Promise.all(toolPromises); +``` + +**Option C: CDN Caching** +```typescript +// Use CDN with aggressive caching +// esm.sh has built-in caching +const url = `https://esm.sh/${pkg}@${version}?target=es2022&bundle`; +``` + +--- + +## Implementation Plan + +### Phase 1: MVP (Week 1-2) + +**Goal:** Prove dynamic loading works with simple prototype + +1. **Create `@tpmjs/search-registry` package** + - Implement `searchTpmjsToolsTool` + - Publish to npm + - Add to manual-tools registry + +2. **Build `/api/tools/search` endpoint** + - Start with simple PostgreSQL `LIKE` search + - Return tool metadata with package info + - Test with curl + +3. **Implement basic dynamic loader** + - Use esm.sh CDN for imports + - Load tools server-side only + - Cache in memory + +4. **Playground integration - Two-Turn Pattern** + - User asks question + - Agent calls `searchTpmjsToolsTool` + - Backend loads tools + - Agent uses tools in next turn + +**Success Criteria:** +- Agent can search registry +- Agent can use dynamically loaded tools +- End-to-end flow works for 1-2 example tools + +--- + +### Phase 2: BM25 Search (Week 3) + +**Goal:** Improve search relevance with BM25 + +1. **Research BM25 implementation options** + - Test PostgreSQL full-text search + - Test JavaScript BM25 libraries + - Benchmark performance + +2. **Implement chosen approach** + - Add search vector column if using PostgreSQL + - Create search index + - Update search endpoint + +3. **Test search quality** + - Create test queries + - Measure precision/recall + - Compare to baseline `LIKE` search + +**Success Criteria:** +- BM25 search returns more relevant results than LIKE +- Search latency < 100ms for 95th percentile +- Agent can find tools for diverse queries + +--- + +### Phase 3: Production Hardening (Week 4) + +**Goal:** Make system production-ready + +1. **Error Handling** + - Handle import failures gracefully + - Validate tool schemas + - Return helpful error messages to agent + +2. **Security** + - Implement package allowlist + - Pin versions from registry + - Add rate limiting to search API + +3. **Performance** + - Implement Redis caching for search results + - Add CDN caching headers + - Optimize tool loading parallelism + +4. **Monitoring** + - Log all dynamic tool loads + - Track search queries and results + - Monitor import success/failure rates + +**Success Criteria:** +- System handles errors without crashing +- Security review passes +- Latency and reliability SLOs met + +--- + +### Phase 4: Advanced Features (Week 5+) + +**Goal:** Enhance UX and capabilities + +1. **Pre-flight Search** + - Analyze user message for intent + - Proactively load tools before agent call + - Reduce total turns needed + +2. **Tool Recommendations** + - "You might also need..." suggestions + - Based on tool co-occurrence data + - Help agent discover related tools + +3. **Client-Side Tool Display** + - Show which tools are available + - Indicate dynamically loaded tools + - Allow user to manually load tools + +4. **Tool Versioning** + - Support multiple versions of same tool + - Let agent choose version + - Handle breaking changes gracefully + +--- + +## Success Metrics + +### Technical Metrics + +1. **Search Quality** + - Precision@10 > 0.8 (80% of top 10 results are relevant) + - Mean Reciprocal Rank (MRR) > 0.7 + - Search latency p95 < 100ms + +2. **Tool Loading** + - Import success rate > 95% + - Tool load time p95 < 2 seconds + - Cache hit rate > 70% after warmup + +3. **End-to-End Performance** + - Total conversation latency < 5 seconds (including tool search + load + execution) + - Agent uses correct tools > 90% of time + +### User Metrics + +1. **Adoption** + - % of playground sessions using dynamic tools > 30% + - Number of unique tools loaded dynamically per week > 50 + +2. **Tool Coverage** + - % of user queries satisfied with available tools > 80% + - Tool search leading to successful task completion > 70% + +--- + +## Open Questions + +### 1. CDN Choice for ESM Imports + +**Options:** +- **esm.sh** - Purpose-built for ESM imports, fast, reliable +- **unpkg** - Popular, simple, but slower +- **jsdelivr** - Fast CDN, good for production +- **Custom bundler** - Pre-bundle tools, serve from our CDN + +**Recommendation:** Start with esm.sh for MVP, evaluate custom bundler for production. + +--- + +### 2. When to Load Tools? + +**Options:** +- **On-demand**: Load when agent calls search tool (current plan) +- **Pre-flight**: Analyze user message, load proactively +- **Lazy**: Load when agent tries to use tool (not when discovered) +- **Eager**: Load all tools from search results immediately + +**Recommendation:** Start with on-demand (Phase 1), add pre-flight in Phase 4. + +--- + +### 3. How to Handle Environment Variables? + +**Problem:** Dynamically loaded tools may need API keys (e.g., OpenWeather API). + +**Options:** +- **User provides**: UI for users to enter API keys (like playground settings) +- **Server-managed**: Admin pre-configures keys in .env +- **Graceful fail**: Load tool, but execution fails if env missing +- **Hybrid**: Some tools work without keys (free tier), others require keys + +**Recommendation:** Start with graceful fail (Phase 1), add user-provided keys (Phase 4). + +--- + +### 4. Should Tools Load Client-Side or Server-Side? + +**Client-Side Pros:** +- Reduces server load +- Faster for subsequent uses +- Better for browser-compatible tools + +**Client-Side Cons:** +- Requires CORS-enabled CDN +- CSP restrictions +- Many tools need Node.js APIs +- Exposing API keys in browser is insecure + +**Server-Side Pros:** +- Access to Node.js APIs +- Secure environment variable access +- No CORS issues +- Easier to implement + +**Server-Side Cons:** +- Requires server memory for caching +- Increases server load +- Cold starts for new tools + +**Recommendation:** Server-side for MVP (Phase 1), evaluate client-side for browser-compatible tools (Phase 4+). + +--- + +### 5. How to Handle Tool Dependencies? + +**Problem:** Some tools depend on other npm packages (e.g., `axios`, `cheerio`). + +**Options:** +- **Bundled**: CDN bundles dependencies (esm.sh does this) +- **Peer deps**: Require dependencies in playground package.json +- **Dynamic install**: npm install on-the-fly (slow, risky) +- **Pre-vetted**: Only allow tools with no/minimal dependencies + +**Recommendation:** Use esm.sh bundling (Phase 1), bundle size limits if issues arise. + +--- + +## Risk Assessment + +### High Risk + +1. **Security Vulnerability** + - **Risk**: Malicious package in registry executes code + - **Mitigation**: Allowlist registry packages, version pinning, VM sandboxing + - **Owner**: Security team + +2. **Performance Degradation** + - **Risk**: Loading many tools causes timeout/slow response + - **Mitigation**: Parallel loading, caching, lazy loading, timeouts + - **Owner**: Backend team + +### Medium Risk + +3. **Import Failures** + - **Risk**: CDN down, package incompatible, missing dependencies + - **Mitigation**: Fallback CDNs, error handling, stub tools + - **Owner**: Frontend team + +4. **AI SDK Limitations** + - **Risk**: Can't dynamically update tools mid-stream + - **Mitigation**: Two-turn pattern, pre-flight search + - **Owner**: AI team + +### Low Risk + +5. **Search Quality** + - **Risk**: BM25 doesn't return relevant tools + - **Mitigation**: A/B test search algorithms, collect feedback + - **Owner**: Search team + +--- + +## Future Enhancements + +### 1. Tool Composition +- Agent can combine multiple tools +- Example: `searchTool` + `summarizeTool` = search and summarize + +### 2. Tool Learning +- Track which tools are used together +- Recommend tool combinations +- "Users who used X also used Y" + +### 3. Custom Tool Registry +- Users can add private tools +- Organization-specific tool registry +- Access control and permissions + +### 4. Tool Marketplace +- Developers promote their tools +- Usage analytics and ratings +- Paid/premium tools + +### 5. Agent Templates +- Pre-configured agents with tool sets +- "Research Agent" has search + summarize tools +- "Code Agent" has code generation tools + +--- + +## Conclusion + +This dynamic tool loading system represents a paradigm shift in how AI agents discover and use tools. By making the TPMJS registry itself searchable, we enable infinite extensibility without the limitations of static bundling. + +**Key Innovation:** Self-referential tool discovery - a tool that searches for tools. + +**Next Steps:** +1. Review this PRD with team +2. Validate technical feasibility with ChatGPT/Claude +3. Spike on BM25 search implementation +4. Spike on dynamic ESM import +5. Begin Phase 1 implementation + +**Success Looks Like:** +- User: "Search Wikipedia for quantum computing" +- Agent: *searches registry, finds Wikipedia tool, loads it, uses it* +- User: Gets Wikipedia results without any manual tool configuration + +This is a novel approach that could define how AI agents discover and use tools. Let's build it. 🚀 diff --git a/ENV_VAR_TRANSPORT_ISSUE.md b/ENV_VAR_TRANSPORT_ISSUE.md new file mode 100644 index 0000000..3a04693 --- /dev/null +++ b/ENV_VAR_TRANSPORT_ISSUE.md @@ -0,0 +1,86 @@ +# Environment Variables Not Sent to API - Frontend Transport Issue + +## Problem + +Environment variables saved in localStorage are NOT being sent to `/api/chat` endpoint. + +**Evidence from logs:** +``` +📥 Request body: { + "conversationId": "7xur5hf1GDSOQMgFYF-l7", + "env": {}, // ❌ EMPTY - should have FIRECRAWL_API_KEY + ... +} +``` + +## Root Cause + +The issue is in `apps/playground/src/hooks/useChat.ts`: + +```typescript +export function useChat() { + const [conversationId] = useState(() => nanoid()); + const envVars = useEnvVars(); // ❌ Empty on first render (useEffect loads async) + + const envObject = envVars.reduce( + (acc, { key, value }) => { + acc[key] = value; + return acc; + }, + {} as Record + ); + + const chat = useAISDKChat({ + transport: new DefaultChatTransport({ // ❌ Created ONCE with empty envObject + api: '/api/chat', + body: { + conversationId, + env: envObject, // ❌ This is {} on first render, never updates + }, + }), + }); + + return { ...chat, conversationId }; +} +``` + +**Why it fails:** + +1. `useEnvVars()` loads from localStorage inside a `useEffect` (async) +2. On first render, `envVars = []`, so `envObject = {}` +3. `DefaultChatTransport` is created with `body: { env: {} }` +4. Even when `envVars` updates later, the transport is already created and doesn't re-create + +## Attempted Solutions That Don't Work + +❌ **Just updating state** - Transport is created once and cached +❌ **Using useEffect** - Transport is already created before effect runs + +## What We Need + +The `body` field in `DefaultChatTransport` needs to be **dynamic** and read the latest env vars on each request, not just once during component mount. + +## Questions for ChatGPT + +1. **How do we make `DefaultChatTransport` body dynamic?** Can we pass a function instead of an object? + +2. **Does AI SDK have a way to update transport body between messages?** The env vars might change while the chat is open. + +3. **Should we use a custom transport instead?** Can we implement our own transport that reads env vars fresh on each request? + +4. **Alternative: Can we manually add env to each message?** Is there a way to inject extra data per-request instead of per-transport? + +## Current Code Files + +- `apps/playground/src/hooks/useChat.ts` - The broken hook +- `apps/playground/src/components/sidebar/SettingsSidebar.tsx` - Where env vars are stored (works fine) +- `apps/playground/src/app/api/chat/route.ts` - Server expects `body.env` but receives `{}` + +## What We Know Works + +✅ Saving env vars to localStorage - working +✅ Reading env vars from localStorage - working +✅ Server accepting and using env vars - working +❌ **Sending env vars from client to server - BROKEN** + +The ONLY broken part is the transport not sending the latest env object. diff --git a/HOW_TO_PUBLISH_A_TOOL.md b/HOW_TO_PUBLISH_A_TOOL.md new file mode 100644 index 0000000..565cb8c --- /dev/null +++ b/HOW_TO_PUBLISH_A_TOOL.md @@ -0,0 +1,424 @@ +# How to Publish a TPMJS Tool + +This guide shows you how to create and publish an AI tool that will be automatically discovered and listed on tpmjs.com. + +## Quick Start + +1. Create a new NPM package +2. Add `"tpmjs-tool"` to the `keywords` array in package.json +3. Add a `tpmjs` field with your tool's metadata +4. Publish to NPM +5. Your tool will automatically appear on tpmjs.com within 15 minutes + +## Step-by-Step Guide + +### 1. Create Your NPM Package + +Create a standard NPM package with your tool implementation: + +```bash +mkdir my-awesome-tool +cd my-awesome-tool +npm init -y +``` + +### 2. Add the Required Keyword + +In your `package.json`, add `"tpmjs-tool"` to the keywords array: + +```json +{ + "name": "@yourname/my-awesome-tool", + "version": "1.0.0", + "keywords": ["tpmjs-tool", "ai", "other-keywords"], + ... +} +``` + +**Important:** The `"tpmjs-tool"` keyword is REQUIRED for automatic discovery! + +### 3. Add TPMJS Metadata + +Add a `tpmjs` field to your `package.json` with your tool's metadata. There are three tiers: + +#### Tier 1: Minimal (Required Fields Only) + +The bare minimum to get listed: + +```json +{ + "tpmjs": { + "category": "text-analysis", + "description": "A concise description of what your tool does" + } +} +``` + +**Required fields:** +- `category` - One of: `text-analysis`, `code-generation`, `data-processing`, `image-generation`, `audio-processing`, `search`, `integration`, `other` +- `description` - Clear description of what the tool does (1-3 sentences) + +#### Tier 2: Basic (Recommended) + +Add parameter and return type information: + +```json +{ + "tpmjs": { + "category": "text-analysis", + "description": "Analyzes sentiment in text and returns a score", + "parameters": [ + { + "name": "text", + "type": "string", + "description": "The text to analyze", + "required": true + }, + { + "name": "language", + "type": "string", + "description": "Language code (e.g., 'en', 'es')", + "required": false, + "default": "en" + } + ], + "returns": { + "type": "SentimentResult", + "description": "Object containing score (-1 to 1) and label (positive/negative/neutral)" + } + } +} +``` + +#### Tier 3: Rich (Full Documentation) + +Complete metadata for maximum visibility: + +```json +{ + "tpmjs": { + "category": "text-analysis", + "description": "Advanced sentiment analysis with emotion detection", + "parameters": [ + { + "name": "text", + "type": "string", + "description": "The text to analyze", + "required": true + }, + { + "name": "language", + "type": "string", + "description": "Language code", + "required": false, + "default": "en" + }, + { + "name": "includeEmotions", + "type": "boolean", + "description": "Whether to include emotion breakdown", + "required": false, + "default": false + } + ], + "returns": { + "type": "SentimentResult", + "description": "Object with score, label, and optional emotions array" + }, + "env": [ + { + "name": "SENTIMENT_API_KEY", + "description": "API key for sentiment analysis service", + "required": true + } + ], + "frameworks": ["vercel-ai", "langchain"], + "aiAgent": { + "useCase": "Use this tool when users need to analyze sentiment in text, detect emotions, or understand the tone of customer feedback, reviews, or social media posts.", + "limitations": "Only supports English and Spanish. Maximum 10,000 characters per request.", + "examples": [ + "Analyze customer review sentiment", + "Detect emotions in user feedback", + "Monitor social media sentiment" + ] + } + } +} +``` + +### 4. Implement Your Tool + +Write your tool's implementation. Here's the example from `@tpmjs/createblogpost`: + +```typescript +// src/index.ts +export interface BlogPostOptions { + title: string; + author: string; + content: string; + tags?: string[]; + format?: 'markdown' | 'mdx'; + excerpt?: string; +} + +export interface BlogPost { + frontmatter: { + title: string; + author: string; + date: string; + tags: string[]; + excerpt?: string; + slug: string; + wordCount: number; + readingTime: number; + }; + content: string; + formattedOutput: string; +} + +export async function createBlogPost(options: BlogPostOptions): Promise { + // Your implementation here + const { title, author, content, tags = [], format = 'markdown', excerpt } = options; + + // Validate inputs + if (!title || !author || !content) { + throw new Error('Title, author, and content are required'); + } + + // Process and return result + return { + frontmatter: { /* ... */ }, + content, + formattedOutput: '...' + }; +} + +export default createBlogPost; +``` + +### 5. Build and Publish + +Build your package and publish to NPM: + +```bash +# Build your package +npm run build + +# Publish to NPM +npm publish --access public +``` + +### 6. Verification + +Your tool will be automatically discovered through: + +1. **Keyword Search** - Runs every 15 minutes, searches NPM for `"tpmjs-tool"` +2. **Changes Feed** - Monitors NPM publishes in real-time (every 2 minutes) + +After publishing, your tool should appear on https://tpmjs.com within 15 minutes! + +You can verify by searching: https://tpmjs.com/api/tools?q=yourpackagename + +## Real Example: @tpmjs/createblogpost + +Here's the complete `package.json` from the published example: + +```json +{ + "name": "@tpmjs/createblogpost", + "version": "0.2.0", + "description": "A tool for creating structured blog posts with AI-generated content", + "type": "module", + "keywords": ["tpmjs-tool", "blog", "content", "ai", "writing"], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "files": ["dist"], + "scripts": { + "build": "tsup", + "dev": "tsup --watch", + "type-check": "tsc --noEmit" + }, + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "https://github.com/ajaxdavis/tpmjs.git", + "directory": "packages/tools/createBlogPost" + }, + "homepage": "https://tpmjs.com", + "license": "MIT", + "tpmjs": { + "category": "text-analysis", + "description": "Creates structured blog posts with customizable frontmatter, content sections, and SEO metadata. Supports multiple output formats including Markdown and MDX.", + "parameters": [ + { + "name": "title", + "type": "string", + "description": "The title of the blog post", + "required": true + }, + { + "name": "author", + "type": "string", + "description": "The author of the blog post", + "required": true + }, + { + "name": "content", + "type": "string", + "description": "The main content of the blog post", + "required": true + }, + { + "name": "tags", + "type": "string[]", + "description": "Array of tags for categorization", + "required": false, + "default": [] + }, + { + "name": "format", + "type": "'markdown' | 'mdx'", + "description": "Output format for the blog post", + "required": false, + "default": "markdown" + }, + { + "name": "excerpt", + "type": "string", + "description": "Short excerpt or summary of the post", + "required": false + } + ], + "returns": { + "type": "BlogPost", + "description": "A structured blog post object with frontmatter, content, and metadata including slug, wordCount, readingTime, and formattedOutput" + }, + "frameworks": ["vercel-ai", "langchain"], + "aiAgent": { + "useCase": "Use this tool when users need to generate blog posts, articles, or structured content with proper frontmatter and metadata. Ideal for content management systems, static site generators, and documentation sites.", + "limitations": "Does not include AI content generation - you must provide the content. Only formats and structures existing content.", + "examples": [ + "Create a blog post about TypeScript best practices", + "Generate a tutorial post with code examples", + "Format an article with SEO metadata" + ] + } + } +} +``` + +## Field Reference + +### Required Fields (Tier 1 - Minimal) + +| Field | Type | Description | +|-------|------|-------------| +| `category` | string | Tool category (see categories below) | +| `description` | string | Clear description (1-3 sentences) | + +### Optional Fields (Tier 2 - Basic) + +| Field | Type | Description | +|-------|------|-------------| +| `parameters` | array | Array of parameter objects | +| `returns` | object | Return type information | + +### Optional Fields (Tier 3 - Rich) + +| Field | Type | Description | +|-------|------|-------------| +| `env` | array | Required environment variables | +| `frameworks` | array | Compatible frameworks | +| `aiAgent` | object | AI agent integration info | + +### Categories + +Choose one of these for the `category` field: + +- `text-analysis` - NLP, sentiment, summarization +- `code-generation` - Code generation and transformation +- `data-processing` - Data manipulation and transformation +- `image-generation` - Image creation and editing +- `audio-processing` - Audio/speech processing +- `search` - Search and retrieval +- `integration` - Third-party integrations +- `other` - Anything else + +### Environment Variables + +If your tool requires environment variables: + +```json +"env": [ + { + "name": "OPENAI_API_KEY", + "description": "API key for OpenAI services", + "required": true + }, + { + "name": "API_ENDPOINT", + "description": "Custom API endpoint URL", + "required": false, + "default": "https://api.example.com" + } +] +``` + +## Quality Score + +Your tool gets a quality score based on: + +- **Tier**: Rich (1.0) > Basic (0.5) > Minimal (0.25) +- **Downloads**: Logarithmic scale based on monthly NPM downloads +- **GitHub Stars**: Logarithmic scale based on repository stars + +Higher scores = better visibility on tpmjs.com! + +## Tips for Success + +1. **Use descriptive names** - Make your package name clear and searchable +2. **Complete metadata** - Tier 3 (Rich) tools get 4x the base score +3. **Good documentation** - Add documentation URL to package.json homepage or repository fields +4. **Active maintenance** - Regular updates boost download counts +5. **AI-friendly descriptions** - Write the `aiAgent.useCase` field as guidance for AI agents + +## Testing Locally + +Before publishing, you can validate your `tpmjs` field using the validation schema: + +```bash +# In the tpmjs monorepo +pnpm --filter=@tpmjs/types test +``` + +Or manually check the structure matches the examples above. + +## Troubleshooting + +**Tool not appearing after 15 minutes?** +- Check that you added `"tpmjs-tool"` to keywords +- Verify your `tpmjs` field has required fields (category, description) +- Check the NPM package is public: `npm view yourpackage` + +**Tool showing as "minimal" tier?** +- Add `parameters` and `returns` fields for Basic tier +- Add all Rich tier fields for maximum visibility + +**Want to force a sync?** +You can manually trigger a sync (requires auth): +```bash +curl -X POST "https://tpmjs.com/api/sync/keyword" \ + -H "Authorization: Bearer YOUR_CRON_SECRET" +``` + +## Support + +Questions or issues? +- File an issue: https://github.com/ajaxdavis/tpmjs/issues +- Check the API: https://tpmjs.com/api/tools diff --git a/IMPLEMENTATION_STATUS.md b/IMPLEMENTATION_STATUS.md new file mode 100644 index 0000000..50bf6c1 --- /dev/null +++ b/IMPLEMENTATION_STATUS.md @@ -0,0 +1,220 @@ +# Dynamic Tool Loading - Implementation Status + +## ✅ Completed + +### 1. Search Tool Package (`@tpmjs/search-registry`) +- ✅ Created package with AI SDK v6 JSON Schema format +- ✅ Connects to search API endpoint +- ✅ Returns tool metadata (packageName, exportName, version, importUrl) +- ✅ Fixed schema format (was using Zod, now uses jsonSchema) +- ✅ Location: `packages/tools/search-registry/` + +### 2. Search API Endpoint (`/api/tools/search`) +- ✅ Implemented simple text-based search (BM25 had dependency issues) +- ✅ Searches by keywords in description, package name, keywords +- ✅ Returns tools with import URLs for esm.sh +- ✅ Location: `apps/web/src/app/api/tools/search/route.ts` + +### 3. Pre-flight Tool Loading in Playground +- ✅ Automatic search on every user message +- ✅ Extracts user query from last message +- ✅ Calls searchTpmjsTools automatically +- ✅ Attempts to load discovered tools dynamically +- ✅ Location: `apps/playground/src/app/api/chat/route.ts` + +### 4. Dynamic Tool Loader (Railway Service Approach) +- ✅ Updated to call Railway service instead of local imports +- ✅ Calls `/load-and-describe` endpoint to get tool schema +- ✅ Wraps tool with remote execution via `/execute-tool` endpoint +- ✅ Caches tool wrappers locally +- ✅ Location: `apps/playground/src/lib/dynamic-tool-loader.ts` + +### 5. Documentation +- ✅ DYNAMIC_IMPORT_ISSUE.md - Comprehensive problem analysis +- ✅ RAILWAY_DYNAMIC_TOOL_LOADER.md - Railway implementation guide +- ✅ This file - Implementation status + +## 🚧 Pending (Railway Service Implementation) + +### Railway Service Endpoints Needed + +You need to add these two endpoints to your existing Railway service: + +#### 1. `POST /load-and-describe` + +**Purpose**: Load a tool from esm.sh and return its schema + +**Request**: +```json +{ + "packageName": "firecrawl-aisdk", + "exportName": "webSearchTool", + "version": "0.7.2", + "importUrl": "https://esm.sh/firecrawl-aisdk@0.7.2" +} +``` + +**Response**: +```json +{ + "success": true, + "tool": { + "exportName": "webSearchTool", + "description": "Search the web using Firecrawl", + "inputSchema": { + "type": "object", + "properties": { + "query": { "type": "string" } + } + } + } +} +``` + +**Implementation Reference**: See `RAILWAY_DYNAMIC_TOOL_LOADER.md` for full code + +#### 2. `POST /execute-tool` + +**Purpose**: Execute a dynamically loaded tool with parameters + +**Request**: +```json +{ + "packageName": "firecrawl-aisdk", + "exportName": "webSearchTool", + "version": "0.7.2", + "importUrl": "https://esm.sh/firecrawl-aisdk@0.7.2", + "params": { + "query": "latest AI news" + } +} +``` + +**Response**: +```json +{ + "success": true, + "output": { "results": [...] }, + "executionTimeMs": 1234 +} +``` + +**Implementation Reference**: See `RAILWAY_DYNAMIC_TOOL_LOADER.md` for full code + +### Deployment Requirements + +1. **Railway Service**: + - Must run with `--experimental-network-imports` flag + - Add to start command: `node --experimental-network-imports server.js` + +2. **Environment Variables** (Vercel): + ```bash + RAILWAY_SERVICE_URL=https://your-railway-service.up.railway.app + # or reuse existing: + SANDBOX_EXECUTOR_URL=https://your-railway-service.up.railway.app + ``` + +3. **Local Testing** (Railway service on port 3001): + ```bash + RAILWAY_SERVICE_URL=http://localhost:3001 + ``` + +## 🎯 Testing Checklist + +Once Railway endpoints are deployed: + +- [ ] Test `/load-and-describe` endpoint directly with curl +- [ ] Test `/execute-tool` endpoint directly with curl +- [ ] Test full flow in playground: + - [ ] Ask: "search the web for latest AI news" + - [ ] Verify pre-flight search finds tools + - [ ] Verify tools load via Railway + - [ ] Verify tool execution works + - [ ] Check console logs for debugging info + +## 📊 Current Flow + +``` +User: "search the web for latest AI news" + ↓ + Chat API extracts query + ↓ + Automatically calls searchTpmjsTools + ↓ + Search API returns matching tools + (packageName, exportName, version) + ↓ + loadToolsBatch() called for each tool + ↓ + For each tool: + 1. Check local cache + 2. If not cached: + → POST to Railway: /load-and-describe + ← Get back: description + inputSchema + 3. Create wrapper tool with: + - description from Railway + - inputSchema from Railway + - execute() → calls Railway /execute-tool + 4. Cache wrapper locally + ↓ + All tools available to agent + ↓ + Agent calls tool (wrapper) + ↓ + Wrapper → POST to Railway: /execute-tool + ↓ + Railway imports from esm.sh and executes + ↓ + Result returned to agent + ↓ + Agent uses result to answer user +``` + +## 🔍 Debugging + +Check console logs for: +- `📦 Loading from Railway` - Tool loading initiated +- `✅ Tool loaded from Railway` - Tool schema received +- `🚀 Executing ... remotely` - Tool execution initiated +- `✅ Tool executed successfully` - Tool execution complete +- `❌ Railway service error` - Connection failed +- `❌ Failed to load tool` - Import failed + +## 📁 Files Modified + +1. `packages/tools/search-registry/src/index.ts` - Search tool +2. `packages/tools/search-registry/package.json` - AI SDK version +3. `apps/web/src/app/api/tools/search/route.ts` - Search endpoint +4. `apps/playground/src/app/api/chat/route.ts` - Pre-flight search +5. `apps/playground/src/lib/dynamic-tool-loader.ts` - Railway integration +6. `apps/playground/next.config.ts` - Added urlImports (unused) +7. `apps/playground/src/lib/tool-loader.ts` - Removed firecrawl + +## 🚀 Next Steps + +1. **Deploy Railway endpoints** using code from `RAILWAY_DYNAMIC_TOOL_LOADER.md` +2. **Set environment variables** in Vercel +3. **Test locally** with Railway service running on localhost:3001 +4. **Deploy to production** and test with real tools +5. **Monitor logs** for any issues + +## 💡 Key Insights + +- **Next.js Limitation**: Cannot do dynamic HTTP imports due to bundler +- **Railway Solution**: Plain Node.js with `--experimental-network-imports` +- **Caching Strategy**: Two-level cache (local wrapper + Railway module) +- **Execution Model**: Remote execution in Railway, not Next.js +- **Security**: Tools execute in Railway sandbox, not Vercel +- **Performance**: First load ~1-2s (import), cached loads <10ms + +## 📚 Related Documentation + +- `DYNAMIC_IMPORT_ISSUE.md` - Problem analysis and ChatGPT response +- `RAILWAY_DYNAMIC_TOOL_LOADER.md` - Full Railway implementation guide +- Plan file: `~/.claude/plans/jiggly-inventing-dragon.md` + +--- + +**Status**: Ready for Railway deployment +**Blocker**: Railway `/load-and-describe` and `/execute-tool` endpoints need implementation +**ETA**: 30-60 minutes to implement Railway endpoints + test diff --git a/MANUAL_TOOLS.md b/MANUAL_TOOLS.md new file mode 100644 index 0000000..fe27e30 --- /dev/null +++ b/MANUAL_TOOLS.md @@ -0,0 +1,293 @@ +# Manual Tools Registry + +## Overview + +This system allows TPMJS to include high-quality tools that don't follow the standard `tpmjs` field specification in their package.json. These tools are manually curated and synced to the database. + +## Why Manual Tools? + +Some excellent tools (like Vercel's code execution, Exa search, Firecrawl, etc.) don't include the `tpmjs` field in their package.json. Rather than wait for these package maintainers to adopt the spec, we manually curate metadata for these tools. + +## Architecture + +### Files + +1. **`manual-tools.ts`** - The registry of manually curated tools +2. **`sync-manual-tools.ts`** - Script to sync manual tools to database +3. **`MANUAL_TOOLS.md`** - This documentation + +### How It Works + +1. **Manual Tool Registry** (`manual-tools.ts`) + - Exports a `manualTools` array with metadata for each tool + - Each entry includes npm package name, export name, category, description, parameters, etc. + - Follows the same schema as the standard `tpmjs` field + +2. **Sync Script** (`sync-manual-tools.ts`) + - Fetches latest package metadata from npm + - Combines npm metadata with manual metadata + - Upserts Package + Tool records to database + - Marks tools with `discoveryMethod: 'manual'` + +3. **Database Storage** + - Manual tools stored in same `packages` and `tools` tables as auto-discovered tools + - No special handling needed in API or frontend + - `discoveryMethod: 'manual'` field distinguishes them + +## Adding a New Manual Tool + +### Step 1: Add to Registry + +Edit `manual-tools.ts` and add a new entry: + +```typescript +{ + npmPackageName: 'example-package', + category: 'search', + frameworks: ['vercel-ai'], + exportName: 'exampleTool', + description: 'A clear, concise description of what this tool does', + + // Optional but recommended for 'rich' tier + parameters: [ + { + name: 'query', + type: 'string', + description: 'The search query', + required: true, + }, + ], + + returns: { + type: 'array', + description: 'Array of search results', + }, + + aiAgent: { + useCase: 'Use when you need to search for X', + limitations: 'Rate limits apply', + examples: [ + 'Search for current news', + 'Find specific information', + ], + }, + + // Environment variables + env: [ + { + name: 'EXAMPLE_API_KEY', + description: 'API key for the service', + required: true, + }, + ], + + // Additional metadata + tags: ['search', 'web'], + docsUrl: 'https://example.com/docs', + apiKeyUrl: 'https://example.com/api-keys', + websiteUrl: 'https://example.com', +} +``` + +### Step 2: Run Sync Script + +```bash +# From repository root +pnpm tsx sync-manual-tools.ts +``` + +This will: +1. Fetch the package from npm +2. Create/update Package record +3. Create/update Tool record(s) +4. Set `discoveryMethod: 'manual'` + +### Step 3: Verify + +Check that the tool appears on tpmjs.com: + +```bash +# Start dev server +pnpm dev --filter=@tpmjs/web + +# Visit http://localhost:3000/tool/tool-search +# Search for your package name +``` + +## Multi-Tool Packages + +If a package exports multiple tools, add multiple entries with the same `npmPackageName` but different `exportName`: + +```typescript +{ + npmPackageName: 'firecrawl-aisdk', + exportName: 'scrapeTool', + description: 'Scrape websites...', + // ... +}, +{ + npmPackageName: 'firecrawl-aisdk', + exportName: 'searchTool', + description: 'Search the web...', + // ... +}, +{ + npmPackageName: 'firecrawl-aisdk', + exportName: 'crawlTool', + description: 'Crawl entire websites...', + // ... +}, +``` + +## Tier Calculation + +Tools are automatically assigned a tier: + +- **Rich tier**: Has `parameters` OR `returns` OR `aiAgent` fields +- **Minimal tier**: Only has basic metadata + +Rich tier tools get 4x quality score multiplier, so add detailed metadata when possible. + +## Maintenance + +### Updating Manual Tools + +1. Edit the entry in `manual-tools.ts` +2. Run `pnpm tsx sync-manual-tools.ts` +3. The upsert will update existing records + +### Removing Manual Tools + +1. Remove the entry from `manual-tools.ts` +2. Manually delete from database OR wait for metrics sync to mark as stale + +### Version Updates + +The sync script automatically fetches the latest version from npm unless you specify `npmVersion` in the manual tool entry. + +## Production Deployment + +### Option 1: Manual Sync on Deploy + +Add to your deployment workflow: + +```yaml +# .github/workflows/deploy.yml +- name: Sync manual tools + run: pnpm tsx sync-manual-tools.ts + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} +``` + +### Option 2: Scheduled Sync + +Create a cron job or GitHub Action to sync periodically: + +```yaml +# .github/workflows/sync-manual.yml +name: Sync Manual Tools + +on: + schedule: + - cron: '0 0 * * 0' # Weekly on Sunday + workflow_dispatch: # Manual trigger + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'pnpm' + - run: pnpm install + - run: pnpm tsx sync-manual-tools.ts + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} +``` + +### Option 3: API Endpoint + +Create a sync endpoint (similar to keyword/changes sync): + +```typescript +// apps/web/src/app/api/sync/manual/route.ts +import { manualTools } from '@/manual-tools'; +// ... sync logic + +export async function POST(request: Request) { + // Verify CRON_SECRET + // Run manual sync + // Return results +} +``` + +## Currently Included Manual Tools + +As of this documentation: + +- **ai-sdk-tool-code-execution** - Vercel Sandbox code execution +- **@exalabs/ai-sdk** - Exa web search +- **@parallel-web/ai-sdk-tools** - Parallel search and extraction (2 tools) +- **ctx-zip** - MCP + Vercel Sandbox integration +- **@perplexity-ai/ai-sdk** - Perplexity search +- **@tavily/ai-sdk** - Tavily web research +- **firecrawl-aisdk** - Firecrawl scraping, search, crawling (3 tools) +- **bedrock-agentcore** - AWS Bedrock code interpreter and browser (2 tools) +- **@superagent-ai/ai-sdk** - Superagent security tools (3 tools) +- **@valyu/ai-sdk** - Valyu domain-specific search tools (8 tools) + +**Total: 24 manually curated tools across 10 packages** + +## FAQ + +### Why not just ask package maintainers to add the tpmjs field? + +We should! But: +1. Some packages are from large companies (Vercel, AWS, etc.) with slow adoption cycles +2. We want these tools available on TPMJS now +3. Manual curation lets us provide better metadata than package authors might + +### Will manual tools be replaced by auto-discovered ones? + +Yes! If a package adds a proper `tpmjs` field, the auto-discovery sync will update it with `discoveryMethod: 'keyword'` or `'changes-feed'`. Manual entries can then be removed from `manual-tools.ts`. + +### Can I mix manual and auto-discovered tools from the same package? + +Yes. If a package has some tools in the `tpmjs` field but is missing others, you can manually add the missing ones. The sync scripts will coexist peacefully. + +### How do I know if a tool is manually curated? + +Check the `discoveryMethod` field in the database: +- `'manual'` = Manually curated +- `'keyword'` = Auto-discovered via keyword search +- `'changes-feed'` = Auto-discovered via npm changes feed + +## Best Practices + +1. **Complete Metadata** - Provide as much metadata as possible for rich tier +2. **Accurate Descriptions** - Tool descriptions should be clear and specific +3. **AI-Friendly** - Write `aiAgent.useCase` as guidance for LLMs +4. **Keep Updated** - Periodically check if packages have added native `tpmjs` support +5. **Link to Docs** - Always include `docsUrl` when available +6. **API Key URLs** - Include `apiKeyUrl` for tools requiring authentication + +## Contributing + +To contribute new manual tools: + +1. Fork the repository +2. Add your tool to `manual-tools.ts` +3. Test with `pnpm tsx sync-manual-tools.ts` +4. Open a pull request with: + - Why this tool should be included + - Link to the npm package + - Screenshot of it working in TPMJS + +## Related Documentation + +- [HOW_TO_PUBLISH_A_TOOL.md](./HOW_TO_PUBLISH_A_TOOL.md) - Standard tpmjs field spec +- [CLAUDE.md](./CLAUDE.md) - General project documentation +- [packages/types/src/tpmjs.ts](./packages/types/src/tpmjs.ts) - TypeScript schema definitions diff --git a/OPENAI_SCHEMA_ERROR.md b/OPENAI_SCHEMA_ERROR.md new file mode 100644 index 0000000..f7ef9e9 --- /dev/null +++ b/OPENAI_SCHEMA_ERROR.md @@ -0,0 +1,522 @@ +# OpenAI Schema Validation Error - AI SDK v6 + +## ✅ RESOLVED + +**Solution:** Use `tool()` and `jsonSchema()` from AI SDK instead of Zod for tool definitions. + +## Error Message + +``` +Error [AI_APICallError]: Invalid schema for function 'helloWorld': schema must be a JSON Schema of 'type: "object"', got 'type: "None"'. +``` + +## Context + +Building a Next.js playground app to test AI SDK v6 tool execution with OpenAI's GPT-4o-mini model. The error occurs when OpenAI validates the tool schema sent in the API request. + +## Root Cause + +Zod 4.0.0 generates JSON Schema with `allOf` + `$ref` at the root level instead of a direct `type: "object"`. OpenAI's API requires a JSON Schema with `type: "object"` at the root, so it rejects Zod 4 schemas with `type: "None"` error. + +## Environment + +- **AI SDK Version**: `ai@6.0.0-beta.124` +- **OpenAI Provider**: `@ai-sdk/openai@3.0.0-beta.74` +- **OpenAI Library**: `openai@^6.9.1` +- **Zod Version**: `zod@^4.0.0` +- **Next.js Version**: `next@^16.0.4` +- **Node.js**: Latest +- **TypeScript**: Strict mode enabled + +## Tool Definition + +Located at: `packages/tools/hello/src/index.ts` + +```typescript +import { z } from 'zod'; + +/** + * Hello World Tool + * Returns a simple "Hello, World!" greeting + * + * This is a proper AI SDK v6 tool that can be used with streamText() + */ +export const helloWorldTool = { + description: 'Returns a simple "Hello, World!" greeting message', + parameters: z.object({ + // OpenAI requires at least one optional parameter, can't be completely empty + includeTimestamp: z.boolean().optional().describe('Whether to include a timestamp in the response'), + }), + execute: async ({ includeTimestamp = true }: { includeTimestamp?: boolean }) => { + const response: any = { + message: 'Hello, World!', + }; + + if (includeTimestamp) { + response.timestamp = new Date().toISOString(); + } + + return response; + }, +}; + +/** + * Hello Name Tool + * Returns a personalized greeting with the provided name + * + * This is a proper AI SDK v6 tool that can be used with streamText() + */ +export const helloNameTool = { + description: 'Returns a personalized greeting with the provided name', + parameters: z.object({ + name: z.string().describe('The name of the person to greet'), + }), + execute: async ({ name }: { name: string }) => { + return { + message: `Hello, ${name}!`, + timestamp: new Date().toISOString(), + }; + }, +}; +``` + +## Tool Loading + +Located at: `apps/playground/src/lib/tool-loader.ts` + +```typescript +// Static imports for tools (required for Next.js/webpack) +import { helloWorldTool, helloNameTool } from '@tpmjs/hello'; +import { scrapeTool, crawlTool, searchTool } from 'firecrawl-aisdk'; + +/** + * Load a specific TPMJS tool by package name + */ +export async function loadTpmjsTool(packageName: string): Promise { + try { + // Map package names to their tool functions + switch (packageName) { + case '@tpmjs/hello': + // Hello has multiple tools, return all of them + return { + helloWorld: helloWorldTool, + helloName: helloNameTool, + }; + + case 'firecrawl-aisdk': + // Firecrawl has multiple tools, return all of them + return { + scrapeTool, + crawlTool, + searchTool, + }; + + default: + throw new Error(`Unknown tool package: ${packageName}`); + } + } catch (error) { + if (error instanceof Error) { + throw new Error(`Failed to load tool from package ${packageName}: ${error.message}`); + } + throw new Error(`Failed to load tool from package ${packageName}: Unknown error`); + } +} + +/** + * Load all installed TPMJS tools + */ +export async function loadAllTools(): Promise> { + const installedTools = ['@tpmjs/hello', 'firecrawl-aisdk']; + + const tools: Record = {}; + + for (const packageName of installedTools) { + try { + const tool = await loadTpmjsTool(packageName); + + // If the tool returns an object with multiple tools (like firecrawl), spread them + if (tool && typeof tool === 'object' && !tool.description) { + Object.assign(tools, tool); + } else { + // Single tool - use a cleaned name (remove hyphens, camelCase) + const toolName = packageName.replace(/-([a-z])/g, (_match, letter) => letter.toUpperCase()).replace(/-/g, ''); + tools[toolName] = tool; + } + } catch (error) { + console.error(`Failed to load tool ${packageName}:`, error); + // Continue loading other tools even if one fails + } + } + + return tools; +} +``` + +## API Route + +Located at: `apps/playground/src/app/api/chat/route.ts` + +```typescript +import { loadAllTools } from '~/lib/tool-loader'; +import { openai } from '@ai-sdk/openai'; +import { streamText } from 'ai'; +import { NextRequest } from 'next/server'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; +export const maxDuration = 60; + +export async function POST(request: NextRequest) { + try { + const body = await request.json(); + const { messages } = body; + + if (!messages || !Array.isArray(messages)) { + return new Response(JSON.stringify({ error: 'Invalid request: messages array required' }), { + status: 400, + headers: { 'Content-Type': 'application/json' }, + }); + } + + // Load all available tools + const tools = await loadAllTools(); + + console.log('Loaded tools:', Object.keys(tools)); + + // Create system message + const systemMessage = { + role: 'system' as const, + content: `You are a helpful AI assistant that can use TPMJS tools to help users. + +Available tools: +${Object.entries(tools) + .map(([name, tool]) => `- ${name}: ${tool.description}`) + .join('\n')} + +Call tools as needed to answer user questions. Execute tools directly.`, + }; + + // Stream the AI response with tools + const result = streamText({ + model: openai('gpt-4o-mini'), + messages: [systemMessage, ...messages], + tools, + maxSteps: 5, + }); + + return result.toTextStreamResponse(); + } catch (error) { + console.error('Chat API error:', error); + return new Response( + JSON.stringify({ + error: error instanceof Error ? error.message : 'Unknown error occurred', + }), + { + status: 500, + headers: { 'Content-Type': 'application/json' }, + } + ); + } +} +``` + +## Package Configuration + +Located at: `packages/tools/hello/package.json` + +```json +{ + "name": "@tpmjs/hello", + "version": "0.0.1", + "private": true, + "description": "Example TPMJS tools - Hello World and Hello Name", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "tsc", + "dev": "tsc --watch", + "clean": "rm -rf dist", + "type-check": "tsc --noEmit" + }, + "keywords": [ + "tpmjs-tool", + "ai-sdk", + "hello", + "example" + ], + "tpmjs": { + "category": "text-analysis", + "description": "Simple greeting tools - Hello World and personalized Hello Name greetings" + }, + "dependencies": { + "ai": "6.0.0-beta.124", + "zod": "^4.0.0" + }, + "devDependencies": { + "@tpmjs/tsconfig": "workspace:*", + "typescript": "^5.9.3" + }, + "files": [ + "dist", + "README.md" + ] +} +``` + +## TypeScript Configuration + +Located at: `packages/tools/hello/tsconfig.json` + +```json +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["ES2020"], + "outDir": "./dist", + "rootDir": "./src", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "node", + "resolveJsonModule": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} +``` + +## Compiled Output + +Located at: `packages/tools/hello/dist/index.js` + +```javascript +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.helloNameTool = exports.helloWorldTool = void 0; +const zod_1 = require("zod"); +/** + * Hello World Tool + * Returns a simple "Hello, World!" greeting + * + * This is a proper AI SDK v6 tool that can be used with streamText() + */ +exports.helloWorldTool = { + description: 'Returns a simple "Hello, World!" greeting message', + parameters: zod_1.z.object({ + // OpenAI requires at least one optional parameter, can't be completely empty + includeTimestamp: zod_1.z.boolean().optional().describe('Whether to include a timestamp in the response'), + }), + execute: async ({ includeTimestamp = true }) => { + const response = { + message: 'Hello, World!', + }; + if (includeTimestamp) { + response.timestamp = new Date().toISOString(); + } + return response; + }, +}; +/** + * Hello Name Tool + * Returns a personalized greeting with the provided name + * + * This is a proper AI SDK v6 tool that can be used with streamTime() + */ +exports.helloNameTool = { + description: 'Returns a personalized greeting with the provided name', + parameters: zod_1.z.object({ + name: zod_1.z.string().describe('The name of the person to greet'), + }), + execute: async ({ name }) => { + return { + message: `Hello, ${name}!`, + timestamp: new Date().toISOString(), + }; + }, +}; +``` + +## Full Error Response from OpenAI + +```json +{ + "error": { + "message": "Invalid schema for function 'helloWorld': schema must be a JSON Schema of 'type: \"object\"', got 'type: \"None\"'.", + "type": "invalid_request_error", + "param": "tools[0].parameters", + "code": "invalid_function_parameters" + } +} +``` + +API endpoint: `https://api.openai.com/v1/responses` +Status code: 400 + +## Problem Analysis + +1. **OpenAI expects JSON Schema format** - The `tools[0].parameters` field must be a valid JSON Schema object with `type: "object"` + +2. **AI SDK v6 should convert Zod to JSON Schema** - The AI SDK is supposed to automatically convert Zod schemas to JSON Schema when sending to OpenAI, but it's producing `type: "None"` instead + +3. **Potential causes**: + - Zod 4.0.0 compatibility issue with AI SDK v6 beta + - AI SDK not properly converting the Zod schema + - Issue with how the tool object is structured + - Problem with how tools are passed to `streamText()` + +4. **Already tried**: + - Added at least one parameter (even optional) to helloWorldTool + - Used proper Zod schema with `.describe()` for descriptions + - Followed AI SDK v6 tool definition format exactly + - Built the package successfully (dist folder exists) + +## AI SDK v6 Tool Format Reference + +According to AI SDK v6 documentation, a tool should be defined as: + +```typescript +{ + description: string, + parameters: ZodSchema, + execute: async (args) => Promise +} +``` + +This matches our implementation exactly. + +## Questions for ChatGPT + +1. Is there a known compatibility issue between AI SDK v6 Beta (6.0.0-beta.124) and Zod 4.0.0? + +2. Does the AI SDK v6 require a specific tool registration format when passing to `streamText()`? + +3. Should tools be wrapped in a different structure (e.g., using `tool()` helper function)? + +4. Is there a way to manually convert Zod schema to JSON Schema that OpenAI accepts? + +5. Are there any known issues with using workspace packages (`@tpmjs/hello`) in Next.js API routes with dynamic imports? + +6. Should we downgrade to Zod 3.x instead of Zod 4.0.0? + +7. Is there a debug mode to see what JSON Schema is being sent to OpenAI? + +## Additional Context + +- The `firecrawl-aisdk` package works correctly with the same setup +- Build process completes successfully with no TypeScript errors +- The tool is being loaded and passed to `streamText()` correctly +- Error only occurs when OpenAI validates the tool schema +- This is a monorepo using pnpm workspaces and Turborepo + +## Related Files + +- Tool definition: `packages/tools/hello/src/index.ts` +- Tool loader: `apps/playground/src/lib/tool-loader.ts` +- API route: `apps/playground/src/app/api/chat/route.ts` +- Package config: `packages/tools/hello/package.json` +- Compiled output: `packages/tools/hello/dist/index.js` + +## Expected Behavior + +Tools should be automatically converted from Zod schema to JSON Schema by AI SDK v6 and accepted by OpenAI's API. + +## Actual Behavior + +OpenAI rejects the tool schema with error: `got 'type: "None"'` instead of a valid JSON Schema object. + +--- + +## ✅ SOLUTION IMPLEMENTED + +### What We Changed + +Instead of using Zod schemas with `parameters`, we now use AI SDK's `tool()` helper with `jsonSchema()` for the input schema. This bypasses Zod's JSON Schema conversion entirely. + +### Before (Broken with Zod 4) + +```typescript +import { z } from 'zod'; + +export const helloWorldTool = { + description: 'Returns a simple "Hello, World!" greeting message', + parameters: z.object({ + includeTimestamp: z.boolean().optional().describe('Whether to include a timestamp'), + }), + execute: async ({ includeTimestamp = true }) => { + // ... + }, +}; +``` + +### After (Working with jsonSchema) + +```typescript +import { jsonSchema, tool } from 'ai'; + +type HelloWorldInput = { + includeTimestamp?: boolean; +}; + +export const helloWorldTool = tool({ + description: 'Returns a simple "Hello, World!" greeting message', + inputSchema: jsonSchema({ + type: 'object', + properties: { + includeTimestamp: { + type: 'boolean', + description: 'Whether to include a timestamp in the response', + }, + }, + additionalProperties: false, + }), + async execute({ includeTimestamp = true }) { + const response: any = { + message: 'Hello, World!', + }; + if (includeTimestamp) { + response.timestamp = new Date().toISOString(); + } + return response; + }, +}); +``` + +### Key Changes + +1. **Import from `ai`**: Added `jsonSchema` and `tool` imports +2. **Define TypeScript types**: Created `HelloWorldInput` type for type safety +3. **Use `tool()` wrapper**: Wraps the entire tool definition +4. **Use `jsonSchema()` for schema**: Provides explicit JSON Schema with `type: "object"` at root +5. **Removed Zod dependency**: No longer need `zod` in package.json + +### Benefits + +- ✅ Works with OpenAI's strict schema validation +- ✅ Explicit control over JSON Schema structure +- ✅ Full TypeScript type safety with generic types +- ✅ No dependency on Zod (one less package to maintain) +- ✅ Follows AI SDK v6 best practices +- ✅ Guaranteed `type: "object"` at root level + +### Updated Package Dependencies + +```json +{ + "dependencies": { + "ai": "6.0.0-beta.124" + } +} +``` + +Zod is no longer needed in tool packages that use `jsonSchema()`. + +### References + +- [AI SDK Core: tool](https://ai-sdk.dev/docs/reference/ai-sdk-core/tool) +- [AI SDK Core: jsonSchema](https://ai-sdk.dev/docs/reference/ai-sdk-core/json-schema) +- [GitHub Issue: Zod 4 JSON Schema compatibility](https://github.com/vercel/ai/issues/10240) diff --git a/RAILWAY_DEPLOYMENT_NOTE.md b/RAILWAY_DEPLOYMENT_NOTE.md new file mode 100644 index 0000000..2f192ce --- /dev/null +++ b/RAILWAY_DEPLOYMENT_NOTE.md @@ -0,0 +1,98 @@ +# Railway Executor - Deployment Status + +## Issue Discovered + +Node.js does not support HTTP(S) imports by default, even with `--experimental-network-imports` flag (that flag doesn't exist in current Node versions). + +## Solutions Considered + +1. **Custom ESM Loader** - Complex, requires Node.js 18.19+ with `--loader` flag +2. **fetch + eval** - Security concerns, doesn't handle ES modules properly +3. **Bundler approach** - Would defeat the purpose of dynamic imports +4. **Deno** - Supports HTTP imports natively, but different ecosystem + +## Recommended Solution + +Since the core issue is that we need truly dynamic runtime imports from HTTP URLs, and Node.js doesn't support this, we have **two viable paths**: + +### Option A: Use Deno on Railway (RECOMMENDED) + +Deno supports HTTP imports natively: + +```typescript +// server.ts (Deno) +import { serve } from "https://deno.land/std@0.208.0/http/server.ts"; + +const moduleCache = new Map(); + +async function loadTool(url: string, exportName: string) { + if (moduleCache.has(url)) { + return moduleCache.get(url); + } + + // Deno supports this natively! + const module = await import(url); + const tool = module[exportName]; + moduleCache.set(url, tool); + return tool; +} + +serve(async (req) => { + // ... handle requests +}, { port: 3002 }); +``` + +**Deploy to Railway:** +```bash +# In Railway dashboard: +# - Set Start Command: deno run --allow-net --allow-env server.ts +# - Or use railway.json with deno runtime +``` + +### Option B: Pre-build Bundle Approach + +Instead of truly dynamic imports, pre-fetch and cache tools: + +1. Playground searches for tools +2. Backend fetches tool code once and caches it +3. Use `vm2` or similar to execute in sandbox +4. Not truly "dynamic" but works with Node.js + +## Current Status + +The Railway executor service is **created** but **not deployed** because Node.js doesn't support the required HTTP imports. + +**Files created:** +- `apps/railway-executor/package.json` +- `apps/railway-executor/server.js` (incomplete - needs Deno or vm2 approach) +- `apps/railway-executor/README.md` + +## Next Steps + +**If using Deno (recommended):** +1. Rewrite server.js as server.ts for Deno +2. Deploy to Railway with Deno runtime +3. Test HTTP imports work +4. Update playground to use Railway URL + +**If sticking with Node.js:** +1. Install `vm2` package for sandboxed execution +2. Implement fetch + vm2 approach +3. Deploy to Railway +4. Accept limitations (less dynamic, more complex) + +## Alternative: Skip Railway, Use Different Architecture + +Since the original issue is Next.js bundler limitations, consider: + +**Web Workers in Browser** - Load tools client-side using native `import()` +- Pros: No server needed, truly dynamic +- Cons: Exposes API keys, security concerns + +**Serverless Functions with Pre-installed Tools** - Deploy each tool as separate function +- Pros: Works with Vercel/Next.js +- Cons: Not truly dynamic, requires redeployment for new tools + +--- + +**Recommendation**: Use Deno on Railway. It's designed for exactly this use case. diff --git a/RAILWAY_DYNAMIC_TOOL_LOADER.md b/RAILWAY_DYNAMIC_TOOL_LOADER.md new file mode 100644 index 0000000..e77b66a --- /dev/null +++ b/RAILWAY_DYNAMIC_TOOL_LOADER.md @@ -0,0 +1,376 @@ +# Railway Service - Dynamic Tool Loader Implementation + +## Overview + +This document describes the Railway service implementation needed to support dynamic tool loading from esm.sh in the TPMJS playground. + +## Why Railway Service? + +Next.js/Turbopack intercepts all `import()` calls and tries to resolve them through its module graph. HTTP URLs like `https://esm.sh/...` are not supported. + +**Solution**: Use a plain Node.js service on Railway that: +- Runs with `--experimental-network-imports` flag +- Can dynamically import from HTTP URLs (esm.sh) +- Executes tool functions and returns results +- Is already set up for existing ToolPlayground + +## New Endpoint Required + +### `POST /load-and-describe` + +**Purpose**: Dynamically import a tool package and return its AI SDK tool definition (description, schema) without executing it. + +**Request**: +```json +{ + "packageName": "firecrawl-aisdk", + "exportName": "webSearchTool", + "version": "0.7.2", + "importUrl": "https://esm.sh/firecrawl-aisdk@0.7.2" +} +``` + +**Response**: +```json +{ + "success": true, + "tool": { + "exportName": "webSearchTool", + "description": "Search the web using Firecrawl", + "inputSchema": { + "type": "object", + "properties": { + "query": { "type": "string", "description": "Search query" } + }, + "required": ["query"] + } + } +} +``` + +**Implementation** (pseudo-code for Railway service): + +```javascript +// server.js (Railway service) +import express from 'express'; + +const app = express(); +app.use(express.json()); + +// Cache for imported modules +const moduleCache = new Map(); + +app.post('/load-and-describe', async (req, res) => { + const { packageName, exportName, version, importUrl } = req.body; + + const cacheKey = `${packageName}::${exportName}`; + + try { + let toolModule; + + // Check cache first + if (moduleCache.has(cacheKey)) { + console.log(`✅ Cache hit: ${cacheKey}`); + toolModule = moduleCache.get(cacheKey); + } else { + // Dynamic import from esm.sh + const url = importUrl || `https://esm.sh/${packageName}@${version}`; + console.log(`📦 Importing: ${url}`); + + const module = await import(url); + toolModule = module[exportName]; + + if (!toolModule) { + return res.status(404).json({ + success: false, + error: `Export "${exportName}" not found in module` + }); + } + + // Validate it's an AI SDK tool + if (!toolModule.description || !toolModule.execute) { + return res.status(400).json({ + success: false, + error: `Invalid AI SDK tool structure` + }); + } + + // Cache it + moduleCache.set(cacheKey, toolModule); + } + + // Extract tool definition (description + schema) + // AI SDK v6 tools have: description, inputSchema, execute + res.json({ + success: true, + tool: { + exportName, + description: toolModule.description, + inputSchema: toolModule.inputSchema || toolModule.parameters?.shape || {}, + } + }); + } catch (error) { + console.error('Failed to load tool:', error); + res.status(500).json({ + success: false, + error: error.message + }); + } +}); + +// Start server +const PORT = process.env.PORT || 3000; +app.listen(PORT, () => { + console.log(`Railway tool loader running on port ${PORT}`); +}); +``` + +**Railway Deployment**: +```bash +# Start command in Railway settings: +node --experimental-network-imports server.js + +# Or in package.json: +{ + "scripts": { + "start": "node --experimental-network-imports server.js" + } +} +``` + +## Modified Endpoint: `POST /execute-tool` + +**Purpose**: Execute a dynamically loaded tool with parameters. + +**Request**: +```json +{ + "packageName": "firecrawl-aisdk", + "exportName": "webSearchTool", + "version": "0.7.2", + "importUrl": "https://esm.sh/firecrawl-aisdk@0.7.2", + "params": { + "query": "latest AI news" + } +} +``` + +**Response**: +```json +{ + "success": true, + "output": { + "results": [...] + }, + "executionTimeMs": 1234 +} +``` + +**Implementation** (pseudo-code): + +```javascript +app.post('/execute-tool', async (req, res) => { + const { packageName, exportName, version, importUrl, params } = req.body; + + const cacheKey = `${packageName}::${exportName}`; + const startTime = Date.now(); + + try { + let toolModule; + + // Check cache or import + if (moduleCache.has(cacheKey)) { + toolModule = moduleCache.get(cacheKey); + } else { + const url = importUrl || `https://esm.sh/${packageName}@${version}`; + const module = await import(url); + toolModule = module[exportName]; + + if (!toolModule || !toolModule.execute) { + return res.status(404).json({ + success: false, + error: 'Tool not found or invalid' + }); + } + + moduleCache.set(cacheKey, toolModule); + } + + // Execute the tool + const result = await toolModule.execute(params); + + res.json({ + success: true, + output: result, + executionTimeMs: Date.now() - startTime + }); + } catch (error) { + res.status(500).json({ + success: false, + error: error.message, + executionTimeMs: Date.now() - startTime + }); + } +}); +``` + +## Integration with Playground + +### 1. Update `dynamic-tool-loader.ts` + +Replace local dynamic imports with Railway service calls: + +```typescript +// apps/playground/src/lib/dynamic-tool-loader.ts + +const RAILWAY_SERVICE_URL = process.env.RAILWAY_SERVICE_URL || 'http://localhost:3001'; + +export async function loadToolDynamically( + packageName: string, + exportName: string, + version: string, + importUrl?: string +): Promise { + const cacheKey = getCacheKey(packageName, exportName); + + // Check local cache first + if (moduleCache.has(cacheKey)) { + console.log(`✅ Cache hit: ${cacheKey}`); + return moduleCache.get(cacheKey); + } + + try { + console.log(`📦 Loading from Railway: ${packageName}/${exportName}`); + + // Call Railway service to load and describe tool + const response = await fetch(`${RAILWAY_SERVICE_URL}/load-and-describe`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + packageName, + exportName, + version, + importUrl, + }), + }); + + if (!response.ok) { + console.error(`❌ Railway service error: ${response.status}`); + return null; + } + + const data = await response.json(); + + if (!data.success) { + console.error(`❌ Failed to load tool: ${data.error}`); + return null; + } + + // Create a tool wrapper that executes remotely + const tool = { + description: data.tool.description, + inputSchema: data.tool.inputSchema, + execute: async (params: any) => { + console.log(`🚀 Executing ${packageName}/${exportName} remotely`); + + const execResponse = await fetch(`${RAILWAY_SERVICE_URL}/execute-tool`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + packageName, + exportName, + version, + importUrl, + params, + }), + }); + + const result = await execResponse.json(); + + if (!result.success) { + throw new Error(result.error || 'Tool execution failed'); + } + + return result.output; + }, + }; + + // Cache the wrapper + moduleCache.set(cacheKey, tool); + console.log(`✅ Loaded and cached: ${cacheKey}`); + + return tool; + } catch (error) { + console.error(`❌ Failed to load ${packageName}#${exportName}:`, error); + return null; + } +} +``` + +### 2. Environment Variables + +Add to `.env.local`: +```bash +RAILWAY_SERVICE_URL=https://your-railway-service.up.railway.app +``` + +Or for local testing with Railway running locally: +```bash +RAILWAY_SERVICE_URL=http://localhost:3001 +``` + +## Testing Locally + +### Terminal 1: Run Railway service locally +```bash +cd railway-service +node --experimental-network-imports server.js +``` + +### Terminal 2: Run playground +```bash +cd tpmjs +pnpm dev --filter=@tpmjs/playground +``` + +### Test the flow: +```bash +# Test Railway service directly +curl -X POST http://localhost:3001/load-and-describe \ + -H "Content-Type: application/json" \ + -d '{ + "packageName": "firecrawl-aisdk", + "exportName": "webSearchTool", + "version": "0.7.2" + }' + +# Then test via playground UI +# Navigate to http://localhost:3000/playground +# Ask: "search the web for latest AI news" +``` + +## Deployment Checklist + +- [ ] Create Railway service with Node.js +- [ ] Add `--experimental-network-imports` flag to start command +- [ ] Deploy `/load-and-describe` endpoint +- [ ] Deploy `/execute-tool` endpoint (or modify existing `/execute`) +- [ ] Set `RAILWAY_SERVICE_URL` in Vercel environment variables +- [ ] Test with real tools from TPMJS registry +- [ ] Monitor Railway logs for import errors + +## Benefits + +1. ✅ **Works around Next.js limitations** - Imports happen in plain Node +2. ✅ **Reuses existing Railway infrastructure** - No new service needed +3. ✅ **Caching on both sides** - Local cache + Railway cache +4. ✅ **Security** - Tools execute in Railway sandbox, not Next.js +5. ✅ **Scalability** - Railway handles the heavy lifting + +## Next Steps + +1. Implement Railway service endpoints +2. Update `dynamic-tool-loader.ts` to use Railway +3. Test locally +4. Deploy to Railway + Vercel +5. Celebrate dynamic tool loading! 🎉 diff --git a/STREAMING_EMPTY_RESPONSE.md b/STREAMING_EMPTY_RESPONSE.md new file mode 100644 index 0000000..98383d7 --- /dev/null +++ b/STREAMING_EMPTY_RESPONSE.md @@ -0,0 +1,363 @@ +# AI SDK v6 Streaming Empty Response Issue + +## Problem + +Using AI SDK v6 Beta with OpenAI and `streamText()`, the API route returns a 200 OK response, but the streamed response body is **completely empty** when tools are involved. + +- **Normal chat** (without tool calls): Works fine, streams text back +- **Tool calls** (when user asks "say hello world"): Returns empty response body, no error messages + +## Environment + +- **AI SDK Version**: `ai@6.0.0-beta.124` +- **OpenAI Provider**: `@ai-sdk/openai@3.0.0-beta.74` +- **OpenAI Library**: `openai@^6.9.1` +- **Next.js Version**: `next@^16.0.4` (App Router) +- **Runtime**: Node.js (`runtime = 'nodejs'`) +- **Model**: `gpt-4o-mini` + +## API Route Implementation + +Located at: `apps/playground/src/app/api/chat/route.ts` + +```typescript +import { createOpenAI } from '@ai-sdk/openai'; +import { streamText, type CoreMessage, tool, jsonSchema } from 'ai'; +import { type NextRequest } from 'next/server'; +import { z } from 'zod'; +import { env } from '~/env'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; +export const maxDuration = 60; + +// Initialize OpenAI provider +const openai = createOpenAI({ + apiKey: env.OPENAI_API_KEY, +}); + +// Request schema +const RequestSchema = z.object({ + messages: z.array( + z.object({ + role: z.enum(['user', 'assistant', 'system']), + content: z.string(), + }) + ), +}); + +// Simple inline test tool to verify streaming works +const testHelloTool = tool({ + description: 'Returns a simple hello world greeting', + inputSchema: jsonSchema<{ includeTimestamp?: boolean }>({ + type: 'object', + properties: { + includeTimestamp: { + type: 'boolean', + description: 'Whether to include a timestamp', + }, + }, + additionalProperties: false, + }), + async execute({ includeTimestamp = true }) { + const response: any = { message: 'Hello, World!' }; + if (includeTimestamp) { + response.timestamp = new Date().toISOString(); + } + return response; + }, +}); + +/** + * POST /api/chat + * Chat with AI agent that can execute TPMJS tools + */ +export async function POST(request: NextRequest) { + try { + const body = await request.json(); + const { messages } = RequestSchema.parse(body); + + // Use simple inline tool for testing + const tools = { + testHello: testHelloTool, + }; + + // Create system prompt listing available tools + const toolsList = Object.keys(tools) + .map((name) => `- ${name}: ${tools[name]?.description}`) + .join('\n'); + + const systemMessage: CoreMessage = { + role: 'system', + content: `You are a helpful AI assistant that can use TPMJS tools to help users. + +Available tools: +${toolsList} + +Call tools as needed to answer user questions. When a user asks to say hello world or for a greeting, use the testHello tool.`, + }; + + // Stream the response + const result = streamText({ + model: openai('gpt-4o-mini'), + messages: [systemMessage, ...messages], + tools, + }); + + // Return the stream as SSE + return result.toTextStreamResponse(); + } catch (error) { + console.error('Chat API error:', error); + + if (error instanceof z.ZodError) { + return new Response( + JSON.stringify({ + success: false, + error: 'Invalid request format', + details: error.issues, + }), + { + status: 400, + headers: { 'Content-Type': 'application/json' }, + } + ); + } + + return new Response( + JSON.stringify({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }), + { + status: 500, + headers: { 'Content-Type': 'application/json' }, + } + ); + } +} +``` + +## Client-Side Hook + +Located at: `apps/playground/src/hooks/useChat.ts` + +```typescript +'use client'; + +import { useCallback, useState } from 'react'; + +export interface ChatMessage { + id: string; + role: 'user' | 'assistant' | 'system'; + content: string; + timestamp: Date; +} + +export function useChat() { + const [messages, setMessages] = useState([]); + const [isLoading, setIsLoading] = useState(false); + const [error, setError] = useState(null); + + const sendMessage = useCallback(async (content: string) => { + if (!content.trim()) return; + + // Add user message immediately + const userMessage: ChatMessage = { + id: crypto.randomUUID(), + role: 'user', + content, + timestamp: new Date(), + }; + + setMessages((prev) => [...prev, userMessage]); + setIsLoading(true); + setError(null); + + try { + // Create assistant message placeholder + const assistantMessageId = crypto.randomUUID(); + const assistantMessage: ChatMessage = { + id: assistantMessageId, + role: 'assistant', + content: '', + timestamp: new Date(), + }; + + setMessages((prev) => [...prev, assistantMessage]); + + // Send request to API + const response = await fetch('/api/chat', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + messages: [...messages, userMessage].map((m) => ({ + role: m.role, + content: m.content, + })), + }), + }); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + // Read the streaming response + const reader = response.body?.getReader(); + const decoder = new TextDecoder(); + + if (!reader) { + throw new Error('Response body is null'); + } + + let accumulatedContent = ''; + + while (true) { + const { done, value } = await reader.read(); + + if (done) { + break; + } + + // Decode the chunk + const chunk = decoder.decode(value, { stream: true }); + accumulatedContent += chunk; + + // Update the assistant message with accumulated content + setMessages((prev) => + prev.map((m) => + m.id === assistantMessageId + ? { ...m, content: accumulatedContent } + : m + ) + ); + } + } catch (err) { + console.error('Error sending message:', err); + setError(err instanceof Error ? err.message : 'Failed to send message'); + } finally { + setIsLoading(false); + } + }, [messages]); + + const clearChat = useCallback(() => { + setMessages([]); + setError(null); + }, []); + + return { + messages, + isLoading, + error, + sendMessage, + clearChat, + }; +} +``` + +## Observed Behavior + +### Working Case (Normal Chat) +- User types: "hi" +- API response: 200 OK +- Response body: Streams text chunks successfully +- UI shows: "Hi! How can I help you today?" + +### Broken Case (Tool Call) +- User types: "say hello world" +- API response: 200 OK ✅ +- Response body: **EMPTY** ❌ (no chunks, no data, nothing) +- UI shows: Empty message bubble +- Console: No errors logged + +## HTTP Response Details + +``` +Request Method: POST +Status Code: 200 OK +URL: http://localhost:3001/api/chat +Content-Type: text/plain; charset=utf-8 +Transfer-Encoding: chunked +``` + +The response headers look correct for a streaming response, but the body is completely empty. + +## What We've Tried + +1. ✅ Fixed OpenAI schema validation error (was `type: "None"`, now uses proper JSON Schema) +2. ✅ Using `tool()` and `jsonSchema()` from AI SDK +3. ✅ Simplified to a single inline test tool +4. ✅ Tool executes without errors (no schema validation issues) +5. ✅ Normal chat works fine (proves streaming infrastructure is correct) + +## Questions + +1. **Is `toTextStreamResponse()` the correct method for streaming with tools in AI SDK v6?** + - Should we use a different method like `toDataStreamResponse()` for tool calls? + +2. **Are we constructing the messages array correctly?** + - We're sending `{ role: 'user' | 'assistant' | 'system', content: string }[]` + - Do we need to include tool call messages or tool result messages? + +3. **Does AI SDK v6 require a specific message format for tool calls?** + - Should we be including `toolInvocations` or `tool_calls` in the message history? + - Are we missing required fields in the `CoreMessage` type? + +4. **Is the client-side streaming reader correct?** + - We're reading chunks with `response.body.getReader()` + - Should we be parsing SSE events differently for tool calls? + +5. **Does `streamText()` with tools require `maxSteps` parameter?** + - Do we need to set `maxSteps: 5` to allow multi-step reasoning? + +6. **Are we handling the conversation history correctly?** + - We're sending all previous messages on each request + - Should we be including assistant messages with tool call results? + +## AI SDK v6 Documentation References + +We're following these patterns from the official docs: + +- [streamText() API](https://ai-sdk.dev/docs/reference/ai-sdk-core/stream-text) +- [tool() API](https://ai-sdk.dev/docs/reference/ai-sdk-core/tool) +- [Tool Calling Guide](https://ai-sdk.dev/docs/ai-sdk-core/tools-and-tool-calling) + +But we might be missing something specific about: +- How to handle streaming when tools are executed +- What response format tool calls produce +- How to parse the stream when tools are involved + +## Suspected Issue + +**The message format might be wrong.** We're sending: + +```typescript +const systemMessage: CoreMessage = { + role: 'system', + content: `You are a helpful AI assistant...`, +}; + +const result = streamText({ + model: openai('gpt-4o-mini'), + messages: [systemMessage, ...messages], + tools, +}); +``` + +But `CoreMessage` might need additional fields when tools are involved, or we might need to handle tool call results differently in the conversation history. + +## What We Need + +1. Correct message format for `streamText()` with tools +2. How to properly stream responses that include tool calls +3. Whether we need different client-side parsing for tool call streams +4. Example of a working Next.js API route using AI SDK v6 with `streamText()` and tools + +## Repo Context + +- Monorepo using Turborepo + pnpm workspaces +- Next.js 16 App Router with Turbopack +- TypeScript strict mode +- All UI components from internal `@tpmjs/ui` package +- Tools are imported from workspace package `@tpmjs/hello` diff --git a/USECHAT_INPUT_UNDEFINED.md b/USECHAT_INPUT_UNDEFINED.md new file mode 100644 index 0000000..861026c --- /dev/null +++ b/USECHAT_INPUT_UNDEFINED.md @@ -0,0 +1,400 @@ +# useChat Hook Returns Undefined Input Property + +## Problem + +Using `@ai-sdk/react`'s `useChat` hook, the `input` property is returning `undefined`, causing the application to crash when trying to call `.trim()` on it. + +## Error + +``` +TypeError: Cannot read properties of undefined (reading 'trim') + +at ChatInput (src/components/chat/ChatInput.tsx:41:48) +``` + +**Code that fails:** +```typescript +