diff --git a/.github/workflows/sync-vercel-registry.yml b/.github/workflows/sync-vercel-registry.yml new file mode 100644 index 0000000..d7e94fa --- /dev/null +++ b/.github/workflows/sync-vercel-registry.yml @@ -0,0 +1,263 @@ +name: Sync Vercel AI Registry + +on: + schedule: + # Run every hour + - cron: '0 * * * *' + workflow_dispatch: + # Run on pushes to main that modify the sync script + push: + branches: + - main + paths: + - 'sync-vercel-registry.ts' + +jobs: + sync-vercel: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.14.0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'pnpm' + + - name: Install dependencies + run: | + echo "šŸ“¦ Installing dependencies..." + pnpm install --frozen-lockfile + echo "āœ… Dependencies installed" + + - name: Run Vercel registry sync + id: sync + run: | + echo "════════════════════════════════════════" + echo "šŸš€ Starting Vercel AI Registry Sync" + echo "════════════════════════════════════════" + echo "" + echo "šŸ“… Time: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" + echo "šŸ”‘ OpenAI API Key: ${OPENAI_API_KEY:0:8}..." + echo "" + + # Run the sync script and capture output + output=$(pnpm tsx sync-vercel-registry.ts 2>&1) + exit_code=$? + + echo "$output" + echo "" + + # Extract statistics from output + processed=$(echo "$output" | grep "Processed:" | tail -1 | awk '{print $2}') + skipped=$(echo "$output" | grep "Skipped:" | tail -1 | awk '{print $2}') + errors=$(echo "$output" | grep "Errors:" | tail -1 | awk '{print $2}') + total=$(echo "$output" | grep "Total:" | tail -1 | awk '{print $2}') + + # Set default values if extraction failed + processed=${processed:-0} + skipped=${skipped:-0} + errors=${errors:-0} + total=${total:-0} + + echo "════════════════════════════════════════" + echo "šŸ“Š Sync Statistics" + echo "════════════════════════════════════════" + echo "✨ Processed: $processed" + echo "ā­ļø Skipped: $skipped" + echo "āŒ Errors: $errors" + echo "šŸ“¦ Total: $total" + echo "════════════════════════════════════════" + echo "" + + # Set outputs for later steps + echo "processed=$processed" >> $GITHUB_OUTPUT + echo "skipped=$skipped" >> $GITHUB_OUTPUT + echo "errors=$errors" >> $GITHUB_OUTPUT + echo "total=$total" >> $GITHUB_OUTPUT + echo "exit_code=$exit_code" >> $GITHUB_OUTPUT + + # Check if manual-tools.ts was modified + if git diff --quiet manual-tools.ts; then + echo "has_changes=false" >> $GITHUB_OUTPUT + echo "ā„¹ļø No changes to manual-tools.ts" + else + echo "has_changes=true" >> $GITHUB_OUTPUT + echo "āœ… manual-tools.ts was modified" + echo "" + echo "šŸ“ Changes preview:" + git diff --stat manual-tools.ts + echo "" + git diff manual-tools.ts | head -50 + fi + + # Determine status for notifications + if [ "$exit_code" -ne 0 ]; then + echo "status_emoji=āŒ" >> $GITHUB_OUTPUT + echo "status_color=15158332" >> $GITHUB_OUTPUT # Red + echo "status_text=Failed" >> $GITHUB_OUTPUT + elif [ "$errors" -gt 0 ]; then + echo "status_emoji=āš ļø" >> $GITHUB_OUTPUT + echo "status_color=16776960" >> $GITHUB_OUTPUT # Yellow + echo "status_text=Completed with errors" >> $GITHUB_OUTPUT + else + echo "status_emoji=āœ…" >> $GITHUB_OUTPUT + echo "status_color=5763719" >> $GITHUB_OUTPUT # Green + echo "status_text=Success" >> $GITHUB_OUTPUT + fi + + # Exit with the original exit code + exit $exit_code + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + - name: Commit and push changes + if: steps.sync.outputs.has_changes == 'true' + run: | + echo "════════════════════════════════════════" + echo "šŸ“ Committing changes to manual-tools.ts" + echo "════════════════════════════════════════" + echo "" + + # Configure git + git config --local user.email "github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + + # Show what's being committed + echo "šŸ“‹ Files to commit:" + git status --short + echo "" + + # Commit changes + git add manual-tools.ts + + commit_message="chore: sync ${{ steps.sync.outputs.processed }} new tools from Vercel AI registry + +Added ${{ steps.sync.outputs.processed }} tools from Vercel AI SDK registry: +- Total tools in registry: ${{ steps.sync.outputs.total }} +- Already synced: ${{ steps.sync.outputs.skipped }} +- Newly added: ${{ steps.sync.outputs.processed }} +- Errors: ${{ steps.sync.outputs.errors }} + +šŸ¤– Automated by GitHub Actions +Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + + git commit -m "$commit_message" + + echo "āœ… Changes committed" + echo "" + + # Push changes + echo "šŸ“¤ Pushing to remote..." + git push + + echo "āœ… Changes pushed successfully" + echo "════════════════════════════════════════" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Send Discord notification + if: always() + run: | + echo "════════════════════════════════════════" + echo "šŸ“¢ Sending Discord notification" + echo "════════════════════════════════════════" + + # Build fields array + base_fields='[ + { "name": "šŸ“¦ Total Tools", "value": "${{ steps.sync.outputs.total }}", "inline": true }, + { "name": "✨ Processed", "value": "${{ steps.sync.outputs.processed }}", "inline": true }, + { "name": "ā­ļø Skipped", "value": "${{ steps.sync.outputs.skipped }}", "inline": true }, + { "name": "āŒ Errors", "value": "${{ steps.sync.outputs.errors }}", "inline": true }, + { "name": "šŸ“ Changes", "value": "${{ steps.sync.outputs.has_changes }}", "inline": true }, + { "name": "šŸ”— Run", "value": "[View Logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})", "inline": true } + ]' + + # Add commit info if changes were made + if [ "${{ steps.sync.outputs.has_changes }}" = "true" ]; then + commit_sha=$(git rev-parse HEAD) + commit_url="https://github.com/${{ github.repository }}/commit/${commit_sha}" + additional_fields='[ + { "name": "šŸ’¾ Commit", "value": "['"${commit_sha:0:7}"']('"$commit_url"')", "inline": false } + ]' + + # Merge fields + all_fields=$(jq -n --argjson base "$base_fields" --argjson additional "$additional_fields" '$base + $additional') + else + all_fields="$base_fields" + fi + + # Create Discord embed + payload=$(jq -n \ + --arg title "${{ steps.sync.outputs.status_emoji }} Vercel AI Registry Sync - ${{ steps.sync.outputs.status_text }}" \ + --argjson color ${{ steps.sync.outputs.status_color }} \ + --argjson fields "$all_fields" \ + --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%S.000Z)" \ + --arg description "Synced Vercel AI SDK tools registry with TPMJS manual tools" \ + ' + { + embeds: [{ + title: $title, + description: $description, + color: $color, + fields: $fields, + timestamp: $timestamp, + footer: { + text: "Vercel AI Registry Sync" + } + }] + }') + + echo "šŸ“¤ Sending payload to Discord..." + + # Send to Discord + response=$(curl -X POST "${{ secrets.DISCORD_WEBHOOK }}" \ + -H "Content-Type: application/json" \ + -d "$payload" \ + -w "\nHTTP Status: %{http_code}\n" \ + -s) + + echo "$response" + + if echo "$response" | grep -q "HTTP Status: 2"; then + echo "āœ… Discord notification sent successfully" + else + echo "āš ļø Discord notification may have failed" + fi + + echo "════════════════════════════════════════" + env: + DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} + + - name: Summary + if: always() + run: | + echo "" + echo "════════════════════════════════════════" + echo "šŸ“Š Workflow Summary" + echo "════════════════════════════════════════" + echo "" + echo "Status: ${{ steps.sync.outputs.status_text }}" + echo "Tools Processed: ${{ steps.sync.outputs.processed }}" + echo "Tools Skipped: ${{ steps.sync.outputs.skipped }}" + echo "Errors: ${{ steps.sync.outputs.errors }}" + echo "Total in Registry: ${{ steps.sync.outputs.total }}" + echo "Changes Made: ${{ steps.sync.outputs.has_changes }}" + echo "" + + if [ "${{ steps.sync.outputs.has_changes }}" = "true" ]; then + echo "āœ… New tools added to manual-tools.ts and committed" + else + echo "ā„¹ļø No new tools found - manual-tools.ts is up to date" + fi + + echo "" + echo "════════════════════════════════════════" diff --git a/docs/vercel-registry-sync.md b/docs/vercel-registry-sync.md new file mode 100644 index 0000000..06701a4 --- /dev/null +++ b/docs/vercel-registry-sync.md @@ -0,0 +1,281 @@ +# Vercel AI Registry Sync + +Automated system that syncs tools from Vercel's AI SDK registry to our manual-tools.ts file. + +## Overview + +The Vercel AI SDK maintains an official registry of tools at: +``` +https://github.com/vercel/ai/blob/main/content/tools-registry/registry.ts +``` + +This automation: +1. Fetches the latest registry every hour +2. Identifies new tools not yet in our manual-tools.ts +3. Uses OpenAI GPT-4 to intelligently convert tool metadata +4. Appends new tools to manual-tools.ts +5. Commits and pushes changes automatically +6. Sends Discord notifications + +## How It Works + +### 1. Script: `sync-vercel-registry.ts` + +Located at the repository root, this TypeScript script: + +**Step 1: Fetch Registry** +- Downloads `registry.ts` from Vercel's GitHub +- Parses TypeScript to extract tool definitions +- Converts to JSON array + +**Step 2: Find New Tools** +- Compares against existing `manual-tools.ts` +- Identifies tools by `npmPackageName` +- Returns list of new tools to add + +**Step 3: AI Conversion** +- For each new tool, calls OpenAI GPT-4 +- Provides Vercel tool metadata + our ManualTool interface +- AI extracts: + - Export names (handles multiple exports per package) + - Parameters from code examples + - Environment variables + - Categories and tags + - Use cases and limitations + +**Step 4: Append to File** +- Generates properly formatted TypeScript code +- Inserts before closing `];` of manualTools array +- Preserves existing formatting + +### 2. GitHub Action: `.github/workflows/sync-vercel-registry.yml` + +**Triggers:** +- **Schedule:** Every hour (`0 * * * *`) +- **Manual:** Via workflow_dispatch +- **Auto:** On push to main that modifies `sync-vercel-registry.ts` + +**Steps:** +1. Checkout repository with `GITHUB_TOKEN` for commits +2. Setup Node.js 22 and pnpm 10.14.0 +3. Install dependencies +4. Run sync script with `OPENAI_API_KEY` +5. Capture output and extract statistics +6. If changes detected: + - Commit with detailed message + - Push to main +7. Send Discord notification with results + +## Configuration + +### Required GitHub Secrets + +| Secret | Description | Where to Get | +|--------|-------------|--------------| +| `OPENAI_API_KEY` | OpenAI API key for GPT-4 | https://platform.openai.com/api-keys | +| `GITHUB_TOKEN` | Auto-provided by GitHub | (automatic) | +| `DISCORD_WEBHOOK` | Discord webhook URL | Discord Server Settings → Integrations → Webhooks | + +### Environment Variables + +The script uses: +- `OPENAI_API_KEY` - Required for AI conversion +- `GITHUB_TOKEN` - Required for committing changes + +## Monitoring + +### GitHub Actions + +View workflow runs: +```bash +gh run list --workflow=sync-vercel-registry.yml +gh run view --log +``` + +### Discord Notifications + +Each run sends a Discord embed with: +- āœ…/āš ļø/āŒ Status indicator +- Total tools in registry +- Number processed (new) +- Number skipped (existing) +- Number of errors +- Whether changes were committed +- Link to GitHub Actions run +- Link to commit (if changes made) + +### Logs + +The workflow includes extensive logging: +- Timestamps and API key masking +- Tool-by-tool processing +- OpenAI conversion details +- Git diff preview +- Statistics summary + +## Manual Testing + +Test the script locally: + +```bash +# Set your OpenAI API key +export OPENAI_API_KEY="sk-..." + +# Run the sync script +pnpm tsx sync-vercel-registry.ts +``` + +This will: +- Fetch the Vercel registry +- Find new tools +- Convert with AI +- Show what would be added (but won't commit) + +## Workflow Output Example + +``` +════════════════════════════════════════ +šŸš€ Starting Vercel AI Registry Sync +════════════════════════════════════════ + +šŸ“… Time: 2025-12-04 15:00:00 UTC +šŸ”‘ OpenAI API Key: sk-proj-... + +šŸ“„ Fetching Vercel AI registry... + URL: https://raw.githubusercontent.com/vercel/ai/refs/heads/main/content/tools-registry/registry.ts + +āœ… Fetched registry (15234 bytes) + +šŸ” Parsing TypeScript registry... + Found tools array (12456 chars) + +šŸ”§ Converting to JSON... +āœ… Parsed 12 tools from registry + +šŸ” Checking for new tools... + Existing manual tools: 25 + Vercel registry tools: 12 + +✨ Found 2 new tools: + + 1. Example Tool (@example/sdk) + 2. Another Tool (@another/tool) + +šŸ¤– Converting new tools with OpenAI... + +šŸ¤– Using OpenAI to convert: Example Tool + Package: @example/sdk + āœ… Received OpenAI response (1234 chars) + ✨ Converted to 2 ManualTool(s): + 1. searchTool - Search the web for current information... + 2. extractTool - Extract structured content from web pages... + +šŸ“Š Conversion Summary: + New Vercel tools: 2 + Converted ManualTools: 3 + Errors: 0 + +šŸ“ Adding 3 new tools to manual-tools.ts... + +āœ… Successfully updated manual-tools.ts + +════════════════════════════════════════ +āœ… Vercel AI Registry Sync Complete! + +šŸ“Š Final Results: + Processed: 3 + Skipped: 10 + Errors: 0 + Total: 12 +``` + +## Commit Message Format + +When changes are committed, the message includes: + +``` +chore: sync 3 new tools from Vercel AI registry + +Added 3 tools from Vercel AI SDK registry: +- Total tools in registry: 12 +- Already synced: 10 +- Newly added: 3 +- Errors: 0 + +šŸ¤– Automated by GitHub Actions +Run: https://github.com/org/repo/actions/runs/123456789 +``` + +## Troubleshooting + +### "Failed to parse JSON" + +The TypeScript-to-JSON conversion may fail if Vercel changes their registry format. + +**Fix:** Update the regex patterns in `fetchVercelRegistry()` function. + +### "Empty response from OpenAI" + +API key issue or rate limiting. + +**Fix:** +- Verify `OPENAI_API_KEY` is set correctly +- Check OpenAI account balance +- Check rate limits at https://platform.openai.com/usage + +### "Could not find closing bracket" + +The `manual-tools.ts` file structure changed. + +**Fix:** Ensure the file ends with `];` on its own line. + +### "Commit failed" + +Git permissions issue. + +**Fix:** Verify GitHub Actions has write permissions in repository settings: +- Settings → Actions → General → Workflow permissions +- Enable "Read and write permissions" + +## Maintenance + +### Updating the Conversion Prompt + +The AI prompt is in `convertToolWithAI()` function. Key sections: + +1. **Interface Definition** - Keep in sync with `ManualTool` interface +2. **Examples** - Show the AI how to handle multi-export packages +3. **Instructions** - Be explicit about response format + +### Changing Sync Frequency + +Edit `.github/workflows/sync-vercel-registry.yml`: + +```yaml +schedule: + - cron: '0 */6 * * *' # Every 6 hours instead of hourly +``` + +### Adding More Registries + +To sync from additional registries: + +1. Create new script: `sync-other-registry.ts` +2. Copy workflow: `sync-other-registry.yml` +3. Update fetch URL and parsing logic +4. Add to documentation + +## Statistics + +As of December 2025: +- Vercel registry: 12 tools +- TPMJS manual tools: 25+ tools +- Sync frequency: Every hour +- Average execution time: ~30 seconds +- OpenAI cost per run: ~$0.01 + +## Related Documentation + +- [Manual Tools](../manual-tools.ts) - The target file +- [Manual Sync](../sync-manual-tools.ts) - Syncs manual tools to database +- [GitHub Actions Overview](../CLAUDE.md#github-actions) - All workflows diff --git a/package.json b/package.json index 1ecac4e..1af9e20 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "dependency-cruiser": "^17.3.1", "knip": "^5.70.2", "lefthook": "^1.10.1", + "openai": "^6.9.1", "tsx": "^4.21.0", "turbo": "^2.6.1", "type-coverage": "^2.29.7", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9b254c5..6aae816 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,6 +29,9 @@ importers: lefthook: specifier: ^1.10.1 version: 1.13.6 + openai: + specifier: ^6.9.1 + version: 6.9.1(ws@8.18.3)(zod@4.1.13) tsx: specifier: ^4.21.0 version: 4.21.0 @@ -50,6 +53,9 @@ importers: '@ai-sdk/react': specifier: 3.0.0-beta.131 version: 3.0.0-beta.131(effect@3.18.4)(react@19.2.0)(zod@4.1.13) + '@tpmjs/db': + specifier: workspace:* + version: link:../../packages/db '@tpmjs/env': specifier: workspace:* version: link:../../packages/env diff --git a/sync-vercel-registry.ts b/sync-vercel-registry.ts new file mode 100644 index 0000000..f7f33d6 --- /dev/null +++ b/sync-vercel-registry.ts @@ -0,0 +1,452 @@ +/** + * Sync Vercel AI Registry Tools + * + * Fetches tools from Vercel's AI SDK registry and adds new ones to manual-tools.ts + * Uses OpenAI to intelligently convert Vercel's format to our ManualTool format. + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import OpenAI from 'openai'; +import { manualTools } from './manual-tools.js'; +import type { ManualTool } from './manual-tools.js'; + +// Vercel registry structure (based on their TypeScript interface) +interface VercelTool { + slug: string; + name: string; + description: string; + packageName: string; + tags?: string[]; + apiKeyEnvName?: string; + installCommand?: { + pnpm?: string; + npm?: string; + yarn?: string; + bun?: string; + }; + codeExample?: string; + docsUrl?: string; + apiKeyUrl?: string; + websiteUrl?: string; + npmUrl?: string; +} + +const VERCEL_REGISTRY_URL = + 'https://raw.githubusercontent.com/vercel/ai/refs/heads/main/content/tools-registry/registry.ts'; + +const openai = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY, +}); + +async function fetchVercelRegistry(): Promise { + console.log('šŸ“„ Fetching Vercel AI registry...'); + console.log(` URL: ${VERCEL_REGISTRY_URL}\n`); + + const response = await fetch(VERCEL_REGISTRY_URL); + + if (!response.ok) { + throw new Error(`Failed to fetch registry: ${response.status} ${response.statusText}`); + } + + const content = await response.text(); + console.log(`āœ… Fetched registry (${content.length} bytes)\n`); + + // Parse TypeScript content to extract tools array + // The registry.ts file exports: export const tools: Tool[] = [...] + console.log('šŸ” Parsing TypeScript registry...'); + + // Extract the tools array using regex + const toolsMatch = content.match(/export const tools[^=]*=\s*(\[[\s\S]*?\n\]);/); + + if (!toolsMatch) { + throw new Error('Could not find tools array in registry'); + } + + const toolsArrayString = toolsMatch[1]; + console.log(` Found tools array (${toolsArrayString.length} chars)\n`); + + // Convert TypeScript to JSON by: + // 1. Remove trailing commas + // 2. Quote unquoted keys + // 3. Remove template literals + const jsonString = toolsArrayString + // Remove single-line comments + .replace(/\/\/.*$/gm, '') + // Remove multi-line comments + .replace(/\/\*[\s\S]*?\*\//g, '') + // Handle template literals (convert to strings) + .replace(/`([^`]*)`/g, '"$1"') + // Quote unquoted object keys + .replace(/(\w+):/g, '"$1":') + // Remove trailing commas before closing braces/brackets + .replace(/,(\s*[}\]])/g, '$1'); + + console.log('šŸ”§ Converting to JSON...'); + + try { + const tools = JSON.parse(jsonString) as VercelTool[]; + console.log(`āœ… Parsed ${tools.length} tools from registry\n`); + return tools; + } catch (error) { + console.error('āŒ Failed to parse JSON. Error:', error); + console.error('Problematic JSON string preview:'); + console.error(jsonString.substring(0, 500)); + throw error; + } +} + +function findNewTools(vercelTools: VercelTool[]): VercelTool[] { + console.log('šŸ” Checking for new tools...'); + console.log(` Existing manual tools: ${manualTools.length}`); + console.log(` Vercel registry tools: ${vercelTools.length}\n`); + + const existingPackages = new Set(manualTools.map((t) => t.npmPackageName)); + + const newTools = vercelTools.filter((tool) => !existingPackages.has(tool.packageName)); + + console.log(`✨ Found ${newTools.length} new tools:\n`); + newTools.forEach((tool, idx) => { + console.log(` ${idx + 1}. ${tool.name} (${tool.packageName})`); + }); + console.log(); + + return newTools; +} + +async function convertToolWithAI(vercelTool: VercelTool): Promise { + console.log(`šŸ¤– Using OpenAI to convert: ${vercelTool.name}`); + console.log(` Package: ${vercelTool.packageName}`); + + const prompt = `You are converting a tool from Vercel's AI registry to TPMJS manual-tools format. + +VERCEL TOOL DATA: +${JSON.stringify(vercelTool, null, 2)} + +YOUR TASK: +Convert this Vercel tool to TPMJS ManualTool format(s). Important: + +1. A single npm package can have MULTIPLE exports (tools). Each export needs its own ManualTool entry. +2. Analyze the codeExample to identify ALL exported tools/functions. +3. For EACH tool/function, create a separate ManualTool object. + +MANUALTOOLS INTERFACE: +{ + npmPackageName: string; // Use vercelTool.packageName + npmVersion?: string; // Omit - will fetch latest + category: string; // Choose from: 'text-analysis', 'code-generation', 'data-processing', 'image-generation', 'audio-processing', 'search', 'integration', 'other' + frameworks: Array<'vercel-ai' | 'langchain' | 'llamaindex' | 'other'>; + exportName: string; // The actual export name (e.g., 'webSearch', 'executeCode') + description: string; // Tool-specific description + parameters?: Array<{ // Extract from codeExample if possible + name: string; + type: string; + description: string; + required: boolean; + default?: string; + }>; + returns?: { + type: string; + description: string; + }; + aiAgent?: { + useCase: string; // When should AI agents use this? + limitations?: string; + examples?: string[]; + }; + env?: Array<{ // Convert apiKeyEnvName to this format + name: string; + description: string; + required: boolean; + }>; + tags?: string[]; // Use vercelTool.tags + docsUrl?: string; + apiKeyUrl?: string; + websiteUrl?: string; +} + +EXAMPLES: + +If codeExample shows: +\`\`\`ts +import { webSearch, financeSearch } from '@valyu/ai-sdk'; +\`\`\` + +Create TWO ManualTool entries: +1. { exportName: 'webSearch', ... } +2. { exportName: 'financeSearch', ... } + +RESPONSE FORMAT: +Return ONLY a valid JSON array of ManualTool objects. No markdown, no explanation, just the JSON array. + +Example response: +[ + { + "npmPackageName": "@example/sdk", + "category": "search", + "frameworks": ["vercel-ai"], + "exportName": "webSearch", + "description": "...", + "env": [{"name": "EXAMPLE_API_KEY", "description": "...", "required": true}], + "tags": ["search"], + "docsUrl": "...", + "apiKeyUrl": "...", + "websiteUrl": "..." + } +]`; + + try { + const completion = await openai.chat.completions.create({ + model: 'gpt-4o', + messages: [ + { + role: 'system', + content: + 'You are a tool metadata converter. You convert tool definitions from one format to another. Always return valid JSON arrays.', + }, + { role: 'user', content: prompt }, + ], + temperature: 0.3, + response_format: { type: 'json_object' }, + }); + + const responseText = completion.choices[0]?.message?.content; + + if (!responseText) { + throw new Error('Empty response from OpenAI'); + } + + console.log(` āœ… Received OpenAI response (${responseText.length} chars)`); + + // Parse the response + let parsed: unknown; + try { + parsed = JSON.parse(responseText); + } catch (error) { + console.error(' āŒ Failed to parse OpenAI response as JSON'); + console.error(' Response:', responseText); + throw error; + } + + // Handle both array and object with array property + let tools: ManualTool[]; + if (Array.isArray(parsed)) { + tools = parsed; + } else if (parsed.tools && Array.isArray(parsed.tools)) { + tools = parsed.tools; + } else if (parsed.manualTools && Array.isArray(parsed.manualTools)) { + tools = parsed.manualTools; + } else { + // Assume single tool, wrap in array + tools = [parsed]; + } + + console.log(` ✨ Converted to ${tools.length} ManualTool(s):`); + tools.forEach((tool, idx) => { + console.log(` ${idx + 1}. ${tool.exportName} - ${tool.description.substring(0, 60)}...`); + }); + console.log(); + + return tools; + } catch (error) { + console.error(' āŒ OpenAI conversion failed:', error); + throw error; + } +} + +async function appendToManualTools(newTools: ManualTool[]): Promise { + if (newTools.length === 0) { + console.log('āœ… No new tools to add\n'); + return; + } + + console.log(`šŸ“ Adding ${newTools.length} new tools to manual-tools.ts...\n`); + + const filePath = path.join(process.cwd(), 'manual-tools.ts'); + + // Read the current file + const currentContent = await fs.readFile(filePath, 'utf-8'); + + // Find the closing bracket of the manualTools array + const lastBracketIndex = currentContent.lastIndexOf('];'); + + if (lastBracketIndex === -1) { + throw new Error('Could not find closing bracket of manualTools array'); + } + + // Generate TypeScript code for new tools + const newToolsCode = newTools + .map((tool) => { + // Convert to properly formatted TypeScript + const lines: string[] = [' {']; + + // Required fields + lines.push(` npmPackageName: '${tool.npmPackageName}',`); + lines.push(` category: '${tool.category}',`); + lines.push(` frameworks: [${tool.frameworks.map((f) => `'${f}'`).join(', ')}],`); + lines.push(` exportName: '${tool.exportName}',`); + lines.push(` description: '${tool.description.replace(/'/g, "\\'")}',`); + + // Optional fields + if (tool.tags && tool.tags.length > 0) { + lines.push(` tags: [${tool.tags.map((t) => `'${t}'`).join(', ')}],`); + } + + if (tool.env && tool.env.length > 0) { + lines.push(' env: ['); + tool.env.forEach((e, idx) => { + lines.push(' {'); + lines.push(` name: '${e.name}',`); + lines.push(` description: '${e.description.replace(/'/g, "\\'")}',`); + lines.push(` required: ${e.required},`); + lines.push(` }${idx < tool.env?.length - 1 ? ',' : ''}`); + }); + lines.push(' ],'); + } + + if (tool.parameters && tool.parameters.length > 0) { + lines.push(' parameters: ['); + tool.parameters.forEach((p, idx) => { + lines.push(' {'); + lines.push(` name: '${p.name}',`); + lines.push(` type: '${p.type}',`); + lines.push(` description: '${p.description.replace(/'/g, "\\'")}',`); + lines.push(` required: ${p.required},`); + if (p.default) { + lines.push(` default: '${p.default}',`); + } + lines.push(` }${idx < tool.parameters?.length - 1 ? ',' : ''}`); + }); + lines.push(' ],'); + } + + if (tool.returns) { + lines.push(' returns: {'); + lines.push(` type: '${tool.returns.type}',`); + lines.push(` description: '${tool.returns.description.replace(/'/g, "\\'")}',`); + lines.push(' },'); + } + + if (tool.aiAgent) { + lines.push(' aiAgent: {'); + lines.push(` useCase: '${tool.aiAgent.useCase.replace(/'/g, "\\'")}',`); + if (tool.aiAgent.limitations) { + lines.push(` limitations: '${tool.aiAgent.limitations.replace(/'/g, "\\'")}',`); + } + if (tool.aiAgent.examples && tool.aiAgent.examples.length > 0) { + lines.push( + ` examples: [${tool.aiAgent.examples.map((e) => `'${e.replace(/'/g, "\\'")}'`).join(', ')}],` + ); + } + lines.push(' },'); + } + + if (tool.docsUrl) { + lines.push(` docsUrl: '${tool.docsUrl}',`); + } + + if (tool.apiKeyUrl) { + lines.push(` apiKeyUrl: '${tool.apiKeyUrl}',`); + } + + if (tool.websiteUrl) { + lines.push(` websiteUrl: '${tool.websiteUrl}',`); + } + + lines.push(' },'); + + return lines.join('\n'); + }) + .join('\n'); + + // Insert new tools before the closing bracket + const updatedContent = `${ + currentContent.slice(0, lastBracketIndex) + newToolsCode + }\n${currentContent.slice(lastBracketIndex)}`; + + // Write back to file + await fs.writeFile(filePath, updatedContent, 'utf-8'); + + console.log('āœ… Successfully updated manual-tools.ts\n'); +} + +async function main() { + console.log('\nšŸš€ Starting Vercel AI Registry Sync\n'); + console.log('═══════════════════════════════════════\n'); + + try { + // Step 1: Fetch Vercel registry + const vercelTools = await fetchVercelRegistry(); + + // Step 2: Find new tools + const newVercelTools = findNewTools(vercelTools); + + if (newVercelTools.length === 0) { + console.log('āœ… No new tools found. Manual tools are up to date!\n'); + return { + processed: 0, + skipped: vercelTools.length, + errors: 0, + total: vercelTools.length, + }; + } + + // Step 3: Convert new tools using AI + console.log('šŸ¤– Converting new tools with OpenAI...\n'); + + const convertedTools: ManualTool[] = []; + let errors = 0; + + for (const vercelTool of newVercelTools) { + try { + const tools = await convertToolWithAI(vercelTool); + convertedTools.push(...tools); + } catch (error) { + console.error(`āŒ Failed to convert ${vercelTool.name}:`, error); + errors++; + } + } + + console.log('\nšŸ“Š Conversion Summary:'); + console.log(` New Vercel tools: ${newVercelTools.length}`); + console.log(` Converted ManualTools: ${convertedTools.length}`); + console.log(` Errors: ${errors}\n`); + + // Step 4: Append to manual-tools.ts + if (convertedTools.length > 0) { + await appendToManualTools(convertedTools); + } + + console.log('═══════════════════════════════════════\n'); + console.log('āœ… Vercel AI Registry Sync Complete!\n'); + + return { + processed: convertedTools.length, + skipped: vercelTools.length - newVercelTools.length, + errors, + total: vercelTools.length, + }; + } catch (error) { + console.error('\nāŒ Sync failed:', error); + throw error; + } +} + +// Run if called directly +if (import.meta.url === `file://${process.argv[1]}`) { + main() + .then((result) => { + console.log('\nšŸ“Š Final Results:'); + console.log(` Processed: ${result.processed}`); + console.log(` Skipped: ${result.skipped}`); + console.log(` Errors: ${result.errors}`); + console.log(` Total: ${result.total}\n`); + process.exit(0); + }) + .catch((error) => { + console.error('Fatal error:', error); + process.exit(1); + }); +} + +export { main as syncVercelRegistry };