feat: add automated Vercel AI registry sync with OpenAI
Add hourly GitHub Action that syncs tools from Vercel's AI SDK registry: Features: - Fetches Vercel AI registry from their GitHub - Uses OpenAI GPT-4 to intelligently convert tool metadata - Handles multi-export packages (multiple tools per npm package) - Automatically commits new tools to manual-tools.ts - Sends Discord notifications with detailed stats - Extensive logging at every step Files added: - sync-vercel-registry.ts - Main sync script with AI conversion - .github/workflows/sync-vercel-registry.yml - Hourly GitHub Action - docs/vercel-registry-sync.md - Complete documentation Requires OPENAI_API_KEY secret in GitHub repository settings.
This commit is contained in:
Ajax Davis
parent
dd8fceb0d5
commit
f9dfc67de4
5 changed files with 1003 additions and 0 deletions
263
.github/workflows/sync-vercel-registry.yml
vendored
Normal file
263
.github/workflows/sync-vercel-registry.yml
vendored
Normal file
|
|
@ -0,0 +1,263 @@
|
|||
name: Sync Vercel AI Registry
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Run every hour
|
||||
- cron: '0 * * * *'
|
||||
workflow_dispatch:
|
||||
# Run on pushes to main that modify the sync script
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- 'sync-vercel-registry.ts'
|
||||
|
||||
jobs:
|
||||
sync-vercel:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 10.14.0
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'pnpm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
echo "📦 Installing dependencies..."
|
||||
pnpm install --frozen-lockfile
|
||||
echo "✅ Dependencies installed"
|
||||
|
||||
- name: Run Vercel registry sync
|
||||
id: sync
|
||||
run: |
|
||||
echo "════════════════════════════════════════"
|
||||
echo "🚀 Starting Vercel AI Registry Sync"
|
||||
echo "════════════════════════════════════════"
|
||||
echo ""
|
||||
echo "📅 Time: $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
|
||||
echo "🔑 OpenAI API Key: ${OPENAI_API_KEY:0:8}..."
|
||||
echo ""
|
||||
|
||||
# Run the sync script and capture output
|
||||
output=$(pnpm tsx sync-vercel-registry.ts 2>&1)
|
||||
exit_code=$?
|
||||
|
||||
echo "$output"
|
||||
echo ""
|
||||
|
||||
# Extract statistics from output
|
||||
processed=$(echo "$output" | grep "Processed:" | tail -1 | awk '{print $2}')
|
||||
skipped=$(echo "$output" | grep "Skipped:" | tail -1 | awk '{print $2}')
|
||||
errors=$(echo "$output" | grep "Errors:" | tail -1 | awk '{print $2}')
|
||||
total=$(echo "$output" | grep "Total:" | tail -1 | awk '{print $2}')
|
||||
|
||||
# Set default values if extraction failed
|
||||
processed=${processed:-0}
|
||||
skipped=${skipped:-0}
|
||||
errors=${errors:-0}
|
||||
total=${total:-0}
|
||||
|
||||
echo "════════════════════════════════════════"
|
||||
echo "📊 Sync Statistics"
|
||||
echo "════════════════════════════════════════"
|
||||
echo "✨ Processed: $processed"
|
||||
echo "⏭️ Skipped: $skipped"
|
||||
echo "❌ Errors: $errors"
|
||||
echo "📦 Total: $total"
|
||||
echo "════════════════════════════════════════"
|
||||
echo ""
|
||||
|
||||
# Set outputs for later steps
|
||||
echo "processed=$processed" >> $GITHUB_OUTPUT
|
||||
echo "skipped=$skipped" >> $GITHUB_OUTPUT
|
||||
echo "errors=$errors" >> $GITHUB_OUTPUT
|
||||
echo "total=$total" >> $GITHUB_OUTPUT
|
||||
echo "exit_code=$exit_code" >> $GITHUB_OUTPUT
|
||||
|
||||
# Check if manual-tools.ts was modified
|
||||
if git diff --quiet manual-tools.ts; then
|
||||
echo "has_changes=false" >> $GITHUB_OUTPUT
|
||||
echo "ℹ️ No changes to manual-tools.ts"
|
||||
else
|
||||
echo "has_changes=true" >> $GITHUB_OUTPUT
|
||||
echo "✅ manual-tools.ts was modified"
|
||||
echo ""
|
||||
echo "📝 Changes preview:"
|
||||
git diff --stat manual-tools.ts
|
||||
echo ""
|
||||
git diff manual-tools.ts | head -50
|
||||
fi
|
||||
|
||||
# Determine status for notifications
|
||||
if [ "$exit_code" -ne 0 ]; then
|
||||
echo "status_emoji=❌" >> $GITHUB_OUTPUT
|
||||
echo "status_color=15158332" >> $GITHUB_OUTPUT # Red
|
||||
echo "status_text=Failed" >> $GITHUB_OUTPUT
|
||||
elif [ "$errors" -gt 0 ]; then
|
||||
echo "status_emoji=⚠️" >> $GITHUB_OUTPUT
|
||||
echo "status_color=16776960" >> $GITHUB_OUTPUT # Yellow
|
||||
echo "status_text=Completed with errors" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "status_emoji=✅" >> $GITHUB_OUTPUT
|
||||
echo "status_color=5763719" >> $GITHUB_OUTPUT # Green
|
||||
echo "status_text=Success" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
# Exit with the original exit code
|
||||
exit $exit_code
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
|
||||
- name: Commit and push changes
|
||||
if: steps.sync.outputs.has_changes == 'true'
|
||||
run: |
|
||||
echo "════════════════════════════════════════"
|
||||
echo "📝 Committing changes to manual-tools.ts"
|
||||
echo "════════════════════════════════════════"
|
||||
echo ""
|
||||
|
||||
# Configure git
|
||||
git config --local user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git config --local user.name "github-actions[bot]"
|
||||
|
||||
# Show what's being committed
|
||||
echo "📋 Files to commit:"
|
||||
git status --short
|
||||
echo ""
|
||||
|
||||
# Commit changes
|
||||
git add manual-tools.ts
|
||||
|
||||
commit_message="chore: sync ${{ steps.sync.outputs.processed }} new tools from Vercel AI registry
|
||||
|
||||
Added ${{ steps.sync.outputs.processed }} tools from Vercel AI SDK registry:
|
||||
- Total tools in registry: ${{ steps.sync.outputs.total }}
|
||||
- Already synced: ${{ steps.sync.outputs.skipped }}
|
||||
- Newly added: ${{ steps.sync.outputs.processed }}
|
||||
- Errors: ${{ steps.sync.outputs.errors }}
|
||||
|
||||
🤖 Automated by GitHub Actions
|
||||
Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
|
||||
git commit -m "$commit_message"
|
||||
|
||||
echo "✅ Changes committed"
|
||||
echo ""
|
||||
|
||||
# Push changes
|
||||
echo "📤 Pushing to remote..."
|
||||
git push
|
||||
|
||||
echo "✅ Changes pushed successfully"
|
||||
echo "════════════════════════════════════════"
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Send Discord notification
|
||||
if: always()
|
||||
run: |
|
||||
echo "════════════════════════════════════════"
|
||||
echo "📢 Sending Discord notification"
|
||||
echo "════════════════════════════════════════"
|
||||
|
||||
# Build fields array
|
||||
base_fields='[
|
||||
{ "name": "📦 Total Tools", "value": "${{ steps.sync.outputs.total }}", "inline": true },
|
||||
{ "name": "✨ Processed", "value": "${{ steps.sync.outputs.processed }}", "inline": true },
|
||||
{ "name": "⏭️ Skipped", "value": "${{ steps.sync.outputs.skipped }}", "inline": true },
|
||||
{ "name": "❌ Errors", "value": "${{ steps.sync.outputs.errors }}", "inline": true },
|
||||
{ "name": "📝 Changes", "value": "${{ steps.sync.outputs.has_changes }}", "inline": true },
|
||||
{ "name": "🔗 Run", "value": "[View Logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})", "inline": true }
|
||||
]'
|
||||
|
||||
# Add commit info if changes were made
|
||||
if [ "${{ steps.sync.outputs.has_changes }}" = "true" ]; then
|
||||
commit_sha=$(git rev-parse HEAD)
|
||||
commit_url="https://github.com/${{ github.repository }}/commit/${commit_sha}"
|
||||
additional_fields='[
|
||||
{ "name": "💾 Commit", "value": "['"${commit_sha:0:7}"']('"$commit_url"')", "inline": false }
|
||||
]'
|
||||
|
||||
# Merge fields
|
||||
all_fields=$(jq -n --argjson base "$base_fields" --argjson additional "$additional_fields" '$base + $additional')
|
||||
else
|
||||
all_fields="$base_fields"
|
||||
fi
|
||||
|
||||
# Create Discord embed
|
||||
payload=$(jq -n \
|
||||
--arg title "${{ steps.sync.outputs.status_emoji }} Vercel AI Registry Sync - ${{ steps.sync.outputs.status_text }}" \
|
||||
--argjson color ${{ steps.sync.outputs.status_color }} \
|
||||
--argjson fields "$all_fields" \
|
||||
--arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%S.000Z)" \
|
||||
--arg description "Synced Vercel AI SDK tools registry with TPMJS manual tools" \
|
||||
'
|
||||
{
|
||||
embeds: [{
|
||||
title: $title,
|
||||
description: $description,
|
||||
color: $color,
|
||||
fields: $fields,
|
||||
timestamp: $timestamp,
|
||||
footer: {
|
||||
text: "Vercel AI Registry Sync"
|
||||
}
|
||||
}]
|
||||
}')
|
||||
|
||||
echo "📤 Sending payload to Discord..."
|
||||
|
||||
# Send to Discord
|
||||
response=$(curl -X POST "${{ secrets.DISCORD_WEBHOOK }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$payload" \
|
||||
-w "\nHTTP Status: %{http_code}\n" \
|
||||
-s)
|
||||
|
||||
echo "$response"
|
||||
|
||||
if echo "$response" | grep -q "HTTP Status: 2"; then
|
||||
echo "✅ Discord notification sent successfully"
|
||||
else
|
||||
echo "⚠️ Discord notification may have failed"
|
||||
fi
|
||||
|
||||
echo "════════════════════════════════════════"
|
||||
env:
|
||||
DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
|
||||
|
||||
- name: Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo ""
|
||||
echo "════════════════════════════════════════"
|
||||
echo "📊 Workflow Summary"
|
||||
echo "════════════════════════════════════════"
|
||||
echo ""
|
||||
echo "Status: ${{ steps.sync.outputs.status_text }}"
|
||||
echo "Tools Processed: ${{ steps.sync.outputs.processed }}"
|
||||
echo "Tools Skipped: ${{ steps.sync.outputs.skipped }}"
|
||||
echo "Errors: ${{ steps.sync.outputs.errors }}"
|
||||
echo "Total in Registry: ${{ steps.sync.outputs.total }}"
|
||||
echo "Changes Made: ${{ steps.sync.outputs.has_changes }}"
|
||||
echo ""
|
||||
|
||||
if [ "${{ steps.sync.outputs.has_changes }}" = "true" ]; then
|
||||
echo "✅ New tools added to manual-tools.ts and committed"
|
||||
else
|
||||
echo "ℹ️ No new tools found - manual-tools.ts is up to date"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "════════════════════════════════════════"
|
||||
281
docs/vercel-registry-sync.md
Normal file
281
docs/vercel-registry-sync.md
Normal file
|
|
@ -0,0 +1,281 @@
|
|||
# Vercel AI Registry Sync
|
||||
|
||||
Automated system that syncs tools from Vercel's AI SDK registry to our manual-tools.ts file.
|
||||
|
||||
## Overview
|
||||
|
||||
The Vercel AI SDK maintains an official registry of tools at:
|
||||
```
|
||||
https://github.com/vercel/ai/blob/main/content/tools-registry/registry.ts
|
||||
```
|
||||
|
||||
This automation:
|
||||
1. Fetches the latest registry every hour
|
||||
2. Identifies new tools not yet in our manual-tools.ts
|
||||
3. Uses OpenAI GPT-4 to intelligently convert tool metadata
|
||||
4. Appends new tools to manual-tools.ts
|
||||
5. Commits and pushes changes automatically
|
||||
6. Sends Discord notifications
|
||||
|
||||
## How It Works
|
||||
|
||||
### 1. Script: `sync-vercel-registry.ts`
|
||||
|
||||
Located at the repository root, this TypeScript script:
|
||||
|
||||
**Step 1: Fetch Registry**
|
||||
- Downloads `registry.ts` from Vercel's GitHub
|
||||
- Parses TypeScript to extract tool definitions
|
||||
- Converts to JSON array
|
||||
|
||||
**Step 2: Find New Tools**
|
||||
- Compares against existing `manual-tools.ts`
|
||||
- Identifies tools by `npmPackageName`
|
||||
- Returns list of new tools to add
|
||||
|
||||
**Step 3: AI Conversion**
|
||||
- For each new tool, calls OpenAI GPT-4
|
||||
- Provides Vercel tool metadata + our ManualTool interface
|
||||
- AI extracts:
|
||||
- Export names (handles multiple exports per package)
|
||||
- Parameters from code examples
|
||||
- Environment variables
|
||||
- Categories and tags
|
||||
- Use cases and limitations
|
||||
|
||||
**Step 4: Append to File**
|
||||
- Generates properly formatted TypeScript code
|
||||
- Inserts before closing `];` of manualTools array
|
||||
- Preserves existing formatting
|
||||
|
||||
### 2. GitHub Action: `.github/workflows/sync-vercel-registry.yml`
|
||||
|
||||
**Triggers:**
|
||||
- **Schedule:** Every hour (`0 * * * *`)
|
||||
- **Manual:** Via workflow_dispatch
|
||||
- **Auto:** On push to main that modifies `sync-vercel-registry.ts`
|
||||
|
||||
**Steps:**
|
||||
1. Checkout repository with `GITHUB_TOKEN` for commits
|
||||
2. Setup Node.js 22 and pnpm 10.14.0
|
||||
3. Install dependencies
|
||||
4. Run sync script with `OPENAI_API_KEY`
|
||||
5. Capture output and extract statistics
|
||||
6. If changes detected:
|
||||
- Commit with detailed message
|
||||
- Push to main
|
||||
7. Send Discord notification with results
|
||||
|
||||
## Configuration
|
||||
|
||||
### Required GitHub Secrets
|
||||
|
||||
| Secret | Description | Where to Get |
|
||||
|--------|-------------|--------------|
|
||||
| `OPENAI_API_KEY` | OpenAI API key for GPT-4 | https://platform.openai.com/api-keys |
|
||||
| `GITHUB_TOKEN` | Auto-provided by GitHub | (automatic) |
|
||||
| `DISCORD_WEBHOOK` | Discord webhook URL | Discord Server Settings → Integrations → Webhooks |
|
||||
|
||||
### Environment Variables
|
||||
|
||||
The script uses:
|
||||
- `OPENAI_API_KEY` - Required for AI conversion
|
||||
- `GITHUB_TOKEN` - Required for committing changes
|
||||
|
||||
## Monitoring
|
||||
|
||||
### GitHub Actions
|
||||
|
||||
View workflow runs:
|
||||
```bash
|
||||
gh run list --workflow=sync-vercel-registry.yml
|
||||
gh run view <run-id> --log
|
||||
```
|
||||
|
||||
### Discord Notifications
|
||||
|
||||
Each run sends a Discord embed with:
|
||||
- ✅/⚠️/❌ Status indicator
|
||||
- Total tools in registry
|
||||
- Number processed (new)
|
||||
- Number skipped (existing)
|
||||
- Number of errors
|
||||
- Whether changes were committed
|
||||
- Link to GitHub Actions run
|
||||
- Link to commit (if changes made)
|
||||
|
||||
### Logs
|
||||
|
||||
The workflow includes extensive logging:
|
||||
- Timestamps and API key masking
|
||||
- Tool-by-tool processing
|
||||
- OpenAI conversion details
|
||||
- Git diff preview
|
||||
- Statistics summary
|
||||
|
||||
## Manual Testing
|
||||
|
||||
Test the script locally:
|
||||
|
||||
```bash
|
||||
# Set your OpenAI API key
|
||||
export OPENAI_API_KEY="sk-..."
|
||||
|
||||
# Run the sync script
|
||||
pnpm tsx sync-vercel-registry.ts
|
||||
```
|
||||
|
||||
This will:
|
||||
- Fetch the Vercel registry
|
||||
- Find new tools
|
||||
- Convert with AI
|
||||
- Show what would be added (but won't commit)
|
||||
|
||||
## Workflow Output Example
|
||||
|
||||
```
|
||||
════════════════════════════════════════
|
||||
🚀 Starting Vercel AI Registry Sync
|
||||
════════════════════════════════════════
|
||||
|
||||
📅 Time: 2025-12-04 15:00:00 UTC
|
||||
🔑 OpenAI API Key: sk-proj-...
|
||||
|
||||
📥 Fetching Vercel AI registry...
|
||||
URL: https://raw.githubusercontent.com/vercel/ai/refs/heads/main/content/tools-registry/registry.ts
|
||||
|
||||
✅ Fetched registry (15234 bytes)
|
||||
|
||||
🔍 Parsing TypeScript registry...
|
||||
Found tools array (12456 chars)
|
||||
|
||||
🔧 Converting to JSON...
|
||||
✅ Parsed 12 tools from registry
|
||||
|
||||
🔍 Checking for new tools...
|
||||
Existing manual tools: 25
|
||||
Vercel registry tools: 12
|
||||
|
||||
✨ Found 2 new tools:
|
||||
|
||||
1. Example Tool (@example/sdk)
|
||||
2. Another Tool (@another/tool)
|
||||
|
||||
🤖 Converting new tools with OpenAI...
|
||||
|
||||
🤖 Using OpenAI to convert: Example Tool
|
||||
Package: @example/sdk
|
||||
✅ Received OpenAI response (1234 chars)
|
||||
✨ Converted to 2 ManualTool(s):
|
||||
1. searchTool - Search the web for current information...
|
||||
2. extractTool - Extract structured content from web pages...
|
||||
|
||||
📊 Conversion Summary:
|
||||
New Vercel tools: 2
|
||||
Converted ManualTools: 3
|
||||
Errors: 0
|
||||
|
||||
📝 Adding 3 new tools to manual-tools.ts...
|
||||
|
||||
✅ Successfully updated manual-tools.ts
|
||||
|
||||
════════════════════════════════════════
|
||||
✅ Vercel AI Registry Sync Complete!
|
||||
|
||||
📊 Final Results:
|
||||
Processed: 3
|
||||
Skipped: 10
|
||||
Errors: 0
|
||||
Total: 12
|
||||
```
|
||||
|
||||
## Commit Message Format
|
||||
|
||||
When changes are committed, the message includes:
|
||||
|
||||
```
|
||||
chore: sync 3 new tools from Vercel AI registry
|
||||
|
||||
Added 3 tools from Vercel AI SDK registry:
|
||||
- Total tools in registry: 12
|
||||
- Already synced: 10
|
||||
- Newly added: 3
|
||||
- Errors: 0
|
||||
|
||||
🤖 Automated by GitHub Actions
|
||||
Run: https://github.com/org/repo/actions/runs/123456789
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### "Failed to parse JSON"
|
||||
|
||||
The TypeScript-to-JSON conversion may fail if Vercel changes their registry format.
|
||||
|
||||
**Fix:** Update the regex patterns in `fetchVercelRegistry()` function.
|
||||
|
||||
### "Empty response from OpenAI"
|
||||
|
||||
API key issue or rate limiting.
|
||||
|
||||
**Fix:**
|
||||
- Verify `OPENAI_API_KEY` is set correctly
|
||||
- Check OpenAI account balance
|
||||
- Check rate limits at https://platform.openai.com/usage
|
||||
|
||||
### "Could not find closing bracket"
|
||||
|
||||
The `manual-tools.ts` file structure changed.
|
||||
|
||||
**Fix:** Ensure the file ends with `];` on its own line.
|
||||
|
||||
### "Commit failed"
|
||||
|
||||
Git permissions issue.
|
||||
|
||||
**Fix:** Verify GitHub Actions has write permissions in repository settings:
|
||||
- Settings → Actions → General → Workflow permissions
|
||||
- Enable "Read and write permissions"
|
||||
|
||||
## Maintenance
|
||||
|
||||
### Updating the Conversion Prompt
|
||||
|
||||
The AI prompt is in `convertToolWithAI()` function. Key sections:
|
||||
|
||||
1. **Interface Definition** - Keep in sync with `ManualTool` interface
|
||||
2. **Examples** - Show the AI how to handle multi-export packages
|
||||
3. **Instructions** - Be explicit about response format
|
||||
|
||||
### Changing Sync Frequency
|
||||
|
||||
Edit `.github/workflows/sync-vercel-registry.yml`:
|
||||
|
||||
```yaml
|
||||
schedule:
|
||||
- cron: '0 */6 * * *' # Every 6 hours instead of hourly
|
||||
```
|
||||
|
||||
### Adding More Registries
|
||||
|
||||
To sync from additional registries:
|
||||
|
||||
1. Create new script: `sync-other-registry.ts`
|
||||
2. Copy workflow: `sync-other-registry.yml`
|
||||
3. Update fetch URL and parsing logic
|
||||
4. Add to documentation
|
||||
|
||||
## Statistics
|
||||
|
||||
As of December 2025:
|
||||
- Vercel registry: 12 tools
|
||||
- TPMJS manual tools: 25+ tools
|
||||
- Sync frequency: Every hour
|
||||
- Average execution time: ~30 seconds
|
||||
- OpenAI cost per run: ~$0.01
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [Manual Tools](../manual-tools.ts) - The target file
|
||||
- [Manual Sync](../sync-manual-tools.ts) - Syncs manual tools to database
|
||||
- [GitHub Actions Overview](../CLAUDE.md#github-actions) - All workflows
|
||||
|
|
@ -34,6 +34,7 @@
|
|||
"dependency-cruiser": "^17.3.1",
|
||||
"knip": "^5.70.2",
|
||||
"lefthook": "^1.10.1",
|
||||
"openai": "^6.9.1",
|
||||
"tsx": "^4.21.0",
|
||||
"turbo": "^2.6.1",
|
||||
"type-coverage": "^2.29.7",
|
||||
|
|
|
|||
6
pnpm-lock.yaml
generated
6
pnpm-lock.yaml
generated
|
|
@ -29,6 +29,9 @@ importers:
|
|||
lefthook:
|
||||
specifier: ^1.10.1
|
||||
version: 1.13.6
|
||||
openai:
|
||||
specifier: ^6.9.1
|
||||
version: 6.9.1(ws@8.18.3)(zod@4.1.13)
|
||||
tsx:
|
||||
specifier: ^4.21.0
|
||||
version: 4.21.0
|
||||
|
|
@ -50,6 +53,9 @@ importers:
|
|||
'@ai-sdk/react':
|
||||
specifier: 3.0.0-beta.131
|
||||
version: 3.0.0-beta.131(effect@3.18.4)(react@19.2.0)(zod@4.1.13)
|
||||
'@tpmjs/db':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/db
|
||||
'@tpmjs/env':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/env
|
||||
|
|
|
|||
452
sync-vercel-registry.ts
Normal file
452
sync-vercel-registry.ts
Normal file
|
|
@ -0,0 +1,452 @@
|
|||
/**
|
||||
* Sync Vercel AI Registry Tools
|
||||
*
|
||||
* Fetches tools from Vercel's AI SDK registry and adds new ones to manual-tools.ts
|
||||
* Uses OpenAI to intelligently convert Vercel's format to our ManualTool format.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import OpenAI from 'openai';
|
||||
import { manualTools } from './manual-tools.js';
|
||||
import type { ManualTool } from './manual-tools.js';
|
||||
|
||||
// Vercel registry structure (based on their TypeScript interface)
|
||||
interface VercelTool {
|
||||
slug: string;
|
||||
name: string;
|
||||
description: string;
|
||||
packageName: string;
|
||||
tags?: string[];
|
||||
apiKeyEnvName?: string;
|
||||
installCommand?: {
|
||||
pnpm?: string;
|
||||
npm?: string;
|
||||
yarn?: string;
|
||||
bun?: string;
|
||||
};
|
||||
codeExample?: string;
|
||||
docsUrl?: string;
|
||||
apiKeyUrl?: string;
|
||||
websiteUrl?: string;
|
||||
npmUrl?: string;
|
||||
}
|
||||
|
||||
const VERCEL_REGISTRY_URL =
|
||||
'https://raw.githubusercontent.com/vercel/ai/refs/heads/main/content/tools-registry/registry.ts';
|
||||
|
||||
const openai = new OpenAI({
|
||||
apiKey: process.env.OPENAI_API_KEY,
|
||||
});
|
||||
|
||||
async function fetchVercelRegistry(): Promise<VercelTool[]> {
|
||||
console.log('📥 Fetching Vercel AI registry...');
|
||||
console.log(` URL: ${VERCEL_REGISTRY_URL}\n`);
|
||||
|
||||
const response = await fetch(VERCEL_REGISTRY_URL);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch registry: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const content = await response.text();
|
||||
console.log(`✅ Fetched registry (${content.length} bytes)\n`);
|
||||
|
||||
// Parse TypeScript content to extract tools array
|
||||
// The registry.ts file exports: export const tools: Tool[] = [...]
|
||||
console.log('🔍 Parsing TypeScript registry...');
|
||||
|
||||
// Extract the tools array using regex
|
||||
const toolsMatch = content.match(/export const tools[^=]*=\s*(\[[\s\S]*?\n\]);/);
|
||||
|
||||
if (!toolsMatch) {
|
||||
throw new Error('Could not find tools array in registry');
|
||||
}
|
||||
|
||||
const toolsArrayString = toolsMatch[1];
|
||||
console.log(` Found tools array (${toolsArrayString.length} chars)\n`);
|
||||
|
||||
// Convert TypeScript to JSON by:
|
||||
// 1. Remove trailing commas
|
||||
// 2. Quote unquoted keys
|
||||
// 3. Remove template literals
|
||||
const jsonString = toolsArrayString
|
||||
// Remove single-line comments
|
||||
.replace(/\/\/.*$/gm, '')
|
||||
// Remove multi-line comments
|
||||
.replace(/\/\*[\s\S]*?\*\//g, '')
|
||||
// Handle template literals (convert to strings)
|
||||
.replace(/`([^`]*)`/g, '"$1"')
|
||||
// Quote unquoted object keys
|
||||
.replace(/(\w+):/g, '"$1":')
|
||||
// Remove trailing commas before closing braces/brackets
|
||||
.replace(/,(\s*[}\]])/g, '$1');
|
||||
|
||||
console.log('🔧 Converting to JSON...');
|
||||
|
||||
try {
|
||||
const tools = JSON.parse(jsonString) as VercelTool[];
|
||||
console.log(`✅ Parsed ${tools.length} tools from registry\n`);
|
||||
return tools;
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to parse JSON. Error:', error);
|
||||
console.error('Problematic JSON string preview:');
|
||||
console.error(jsonString.substring(0, 500));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function findNewTools(vercelTools: VercelTool[]): VercelTool[] {
|
||||
console.log('🔍 Checking for new tools...');
|
||||
console.log(` Existing manual tools: ${manualTools.length}`);
|
||||
console.log(` Vercel registry tools: ${vercelTools.length}\n`);
|
||||
|
||||
const existingPackages = new Set(manualTools.map((t) => t.npmPackageName));
|
||||
|
||||
const newTools = vercelTools.filter((tool) => !existingPackages.has(tool.packageName));
|
||||
|
||||
console.log(`✨ Found ${newTools.length} new tools:\n`);
|
||||
newTools.forEach((tool, idx) => {
|
||||
console.log(` ${idx + 1}. ${tool.name} (${tool.packageName})`);
|
||||
});
|
||||
console.log();
|
||||
|
||||
return newTools;
|
||||
}
|
||||
|
||||
async function convertToolWithAI(vercelTool: VercelTool): Promise<ManualTool[]> {
|
||||
console.log(`🤖 Using OpenAI to convert: ${vercelTool.name}`);
|
||||
console.log(` Package: ${vercelTool.packageName}`);
|
||||
|
||||
const prompt = `You are converting a tool from Vercel's AI registry to TPMJS manual-tools format.
|
||||
|
||||
VERCEL TOOL DATA:
|
||||
${JSON.stringify(vercelTool, null, 2)}
|
||||
|
||||
YOUR TASK:
|
||||
Convert this Vercel tool to TPMJS ManualTool format(s). Important:
|
||||
|
||||
1. A single npm package can have MULTIPLE exports (tools). Each export needs its own ManualTool entry.
|
||||
2. Analyze the codeExample to identify ALL exported tools/functions.
|
||||
3. For EACH tool/function, create a separate ManualTool object.
|
||||
|
||||
MANUALTOOLS INTERFACE:
|
||||
{
|
||||
npmPackageName: string; // Use vercelTool.packageName
|
||||
npmVersion?: string; // Omit - will fetch latest
|
||||
category: string; // Choose from: 'text-analysis', 'code-generation', 'data-processing', 'image-generation', 'audio-processing', 'search', 'integration', 'other'
|
||||
frameworks: Array<'vercel-ai' | 'langchain' | 'llamaindex' | 'other'>;
|
||||
exportName: string; // The actual export name (e.g., 'webSearch', 'executeCode')
|
||||
description: string; // Tool-specific description
|
||||
parameters?: Array<{ // Extract from codeExample if possible
|
||||
name: string;
|
||||
type: string;
|
||||
description: string;
|
||||
required: boolean;
|
||||
default?: string;
|
||||
}>;
|
||||
returns?: {
|
||||
type: string;
|
||||
description: string;
|
||||
};
|
||||
aiAgent?: {
|
||||
useCase: string; // When should AI agents use this?
|
||||
limitations?: string;
|
||||
examples?: string[];
|
||||
};
|
||||
env?: Array<{ // Convert apiKeyEnvName to this format
|
||||
name: string;
|
||||
description: string;
|
||||
required: boolean;
|
||||
}>;
|
||||
tags?: string[]; // Use vercelTool.tags
|
||||
docsUrl?: string;
|
||||
apiKeyUrl?: string;
|
||||
websiteUrl?: string;
|
||||
}
|
||||
|
||||
EXAMPLES:
|
||||
|
||||
If codeExample shows:
|
||||
\`\`\`ts
|
||||
import { webSearch, financeSearch } from '@valyu/ai-sdk';
|
||||
\`\`\`
|
||||
|
||||
Create TWO ManualTool entries:
|
||||
1. { exportName: 'webSearch', ... }
|
||||
2. { exportName: 'financeSearch', ... }
|
||||
|
||||
RESPONSE FORMAT:
|
||||
Return ONLY a valid JSON array of ManualTool objects. No markdown, no explanation, just the JSON array.
|
||||
|
||||
Example response:
|
||||
[
|
||||
{
|
||||
"npmPackageName": "@example/sdk",
|
||||
"category": "search",
|
||||
"frameworks": ["vercel-ai"],
|
||||
"exportName": "webSearch",
|
||||
"description": "...",
|
||||
"env": [{"name": "EXAMPLE_API_KEY", "description": "...", "required": true}],
|
||||
"tags": ["search"],
|
||||
"docsUrl": "...",
|
||||
"apiKeyUrl": "...",
|
||||
"websiteUrl": "..."
|
||||
}
|
||||
]`;
|
||||
|
||||
try {
|
||||
const completion = await openai.chat.completions.create({
|
||||
model: 'gpt-4o',
|
||||
messages: [
|
||||
{
|
||||
role: 'system',
|
||||
content:
|
||||
'You are a tool metadata converter. You convert tool definitions from one format to another. Always return valid JSON arrays.',
|
||||
},
|
||||
{ role: 'user', content: prompt },
|
||||
],
|
||||
temperature: 0.3,
|
||||
response_format: { type: 'json_object' },
|
||||
});
|
||||
|
||||
const responseText = completion.choices[0]?.message?.content;
|
||||
|
||||
if (!responseText) {
|
||||
throw new Error('Empty response from OpenAI');
|
||||
}
|
||||
|
||||
console.log(` ✅ Received OpenAI response (${responseText.length} chars)`);
|
||||
|
||||
// Parse the response
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(responseText);
|
||||
} catch (error) {
|
||||
console.error(' ❌ Failed to parse OpenAI response as JSON');
|
||||
console.error(' Response:', responseText);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Handle both array and object with array property
|
||||
let tools: ManualTool[];
|
||||
if (Array.isArray(parsed)) {
|
||||
tools = parsed;
|
||||
} else if (parsed.tools && Array.isArray(parsed.tools)) {
|
||||
tools = parsed.tools;
|
||||
} else if (parsed.manualTools && Array.isArray(parsed.manualTools)) {
|
||||
tools = parsed.manualTools;
|
||||
} else {
|
||||
// Assume single tool, wrap in array
|
||||
tools = [parsed];
|
||||
}
|
||||
|
||||
console.log(` ✨ Converted to ${tools.length} ManualTool(s):`);
|
||||
tools.forEach((tool, idx) => {
|
||||
console.log(` ${idx + 1}. ${tool.exportName} - ${tool.description.substring(0, 60)}...`);
|
||||
});
|
||||
console.log();
|
||||
|
||||
return tools;
|
||||
} catch (error) {
|
||||
console.error(' ❌ OpenAI conversion failed:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function appendToManualTools(newTools: ManualTool[]): Promise<void> {
|
||||
if (newTools.length === 0) {
|
||||
console.log('✅ No new tools to add\n');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`📝 Adding ${newTools.length} new tools to manual-tools.ts...\n`);
|
||||
|
||||
const filePath = path.join(process.cwd(), 'manual-tools.ts');
|
||||
|
||||
// Read the current file
|
||||
const currentContent = await fs.readFile(filePath, 'utf-8');
|
||||
|
||||
// Find the closing bracket of the manualTools array
|
||||
const lastBracketIndex = currentContent.lastIndexOf('];');
|
||||
|
||||
if (lastBracketIndex === -1) {
|
||||
throw new Error('Could not find closing bracket of manualTools array');
|
||||
}
|
||||
|
||||
// Generate TypeScript code for new tools
|
||||
const newToolsCode = newTools
|
||||
.map((tool) => {
|
||||
// Convert to properly formatted TypeScript
|
||||
const lines: string[] = [' {'];
|
||||
|
||||
// Required fields
|
||||
lines.push(` npmPackageName: '${tool.npmPackageName}',`);
|
||||
lines.push(` category: '${tool.category}',`);
|
||||
lines.push(` frameworks: [${tool.frameworks.map((f) => `'${f}'`).join(', ')}],`);
|
||||
lines.push(` exportName: '${tool.exportName}',`);
|
||||
lines.push(` description: '${tool.description.replace(/'/g, "\\'")}',`);
|
||||
|
||||
// Optional fields
|
||||
if (tool.tags && tool.tags.length > 0) {
|
||||
lines.push(` tags: [${tool.tags.map((t) => `'${t}'`).join(', ')}],`);
|
||||
}
|
||||
|
||||
if (tool.env && tool.env.length > 0) {
|
||||
lines.push(' env: [');
|
||||
tool.env.forEach((e, idx) => {
|
||||
lines.push(' {');
|
||||
lines.push(` name: '${e.name}',`);
|
||||
lines.push(` description: '${e.description.replace(/'/g, "\\'")}',`);
|
||||
lines.push(` required: ${e.required},`);
|
||||
lines.push(` }${idx < tool.env?.length - 1 ? ',' : ''}`);
|
||||
});
|
||||
lines.push(' ],');
|
||||
}
|
||||
|
||||
if (tool.parameters && tool.parameters.length > 0) {
|
||||
lines.push(' parameters: [');
|
||||
tool.parameters.forEach((p, idx) => {
|
||||
lines.push(' {');
|
||||
lines.push(` name: '${p.name}',`);
|
||||
lines.push(` type: '${p.type}',`);
|
||||
lines.push(` description: '${p.description.replace(/'/g, "\\'")}',`);
|
||||
lines.push(` required: ${p.required},`);
|
||||
if (p.default) {
|
||||
lines.push(` default: '${p.default}',`);
|
||||
}
|
||||
lines.push(` }${idx < tool.parameters?.length - 1 ? ',' : ''}`);
|
||||
});
|
||||
lines.push(' ],');
|
||||
}
|
||||
|
||||
if (tool.returns) {
|
||||
lines.push(' returns: {');
|
||||
lines.push(` type: '${tool.returns.type}',`);
|
||||
lines.push(` description: '${tool.returns.description.replace(/'/g, "\\'")}',`);
|
||||
lines.push(' },');
|
||||
}
|
||||
|
||||
if (tool.aiAgent) {
|
||||
lines.push(' aiAgent: {');
|
||||
lines.push(` useCase: '${tool.aiAgent.useCase.replace(/'/g, "\\'")}',`);
|
||||
if (tool.aiAgent.limitations) {
|
||||
lines.push(` limitations: '${tool.aiAgent.limitations.replace(/'/g, "\\'")}',`);
|
||||
}
|
||||
if (tool.aiAgent.examples && tool.aiAgent.examples.length > 0) {
|
||||
lines.push(
|
||||
` examples: [${tool.aiAgent.examples.map((e) => `'${e.replace(/'/g, "\\'")}'`).join(', ')}],`
|
||||
);
|
||||
}
|
||||
lines.push(' },');
|
||||
}
|
||||
|
||||
if (tool.docsUrl) {
|
||||
lines.push(` docsUrl: '${tool.docsUrl}',`);
|
||||
}
|
||||
|
||||
if (tool.apiKeyUrl) {
|
||||
lines.push(` apiKeyUrl: '${tool.apiKeyUrl}',`);
|
||||
}
|
||||
|
||||
if (tool.websiteUrl) {
|
||||
lines.push(` websiteUrl: '${tool.websiteUrl}',`);
|
||||
}
|
||||
|
||||
lines.push(' },');
|
||||
|
||||
return lines.join('\n');
|
||||
})
|
||||
.join('\n');
|
||||
|
||||
// Insert new tools before the closing bracket
|
||||
const updatedContent = `${
|
||||
currentContent.slice(0, lastBracketIndex) + newToolsCode
|
||||
}\n${currentContent.slice(lastBracketIndex)}`;
|
||||
|
||||
// Write back to file
|
||||
await fs.writeFile(filePath, updatedContent, 'utf-8');
|
||||
|
||||
console.log('✅ Successfully updated manual-tools.ts\n');
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('\n🚀 Starting Vercel AI Registry Sync\n');
|
||||
console.log('═══════════════════════════════════════\n');
|
||||
|
||||
try {
|
||||
// Step 1: Fetch Vercel registry
|
||||
const vercelTools = await fetchVercelRegistry();
|
||||
|
||||
// Step 2: Find new tools
|
||||
const newVercelTools = findNewTools(vercelTools);
|
||||
|
||||
if (newVercelTools.length === 0) {
|
||||
console.log('✅ No new tools found. Manual tools are up to date!\n');
|
||||
return {
|
||||
processed: 0,
|
||||
skipped: vercelTools.length,
|
||||
errors: 0,
|
||||
total: vercelTools.length,
|
||||
};
|
||||
}
|
||||
|
||||
// Step 3: Convert new tools using AI
|
||||
console.log('🤖 Converting new tools with OpenAI...\n');
|
||||
|
||||
const convertedTools: ManualTool[] = [];
|
||||
let errors = 0;
|
||||
|
||||
for (const vercelTool of newVercelTools) {
|
||||
try {
|
||||
const tools = await convertToolWithAI(vercelTool);
|
||||
convertedTools.push(...tools);
|
||||
} catch (error) {
|
||||
console.error(`❌ Failed to convert ${vercelTool.name}:`, error);
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n📊 Conversion Summary:');
|
||||
console.log(` New Vercel tools: ${newVercelTools.length}`);
|
||||
console.log(` Converted ManualTools: ${convertedTools.length}`);
|
||||
console.log(` Errors: ${errors}\n`);
|
||||
|
||||
// Step 4: Append to manual-tools.ts
|
||||
if (convertedTools.length > 0) {
|
||||
await appendToManualTools(convertedTools);
|
||||
}
|
||||
|
||||
console.log('═══════════════════════════════════════\n');
|
||||
console.log('✅ Vercel AI Registry Sync Complete!\n');
|
||||
|
||||
return {
|
||||
processed: convertedTools.length,
|
||||
skipped: vercelTools.length - newVercelTools.length,
|
||||
errors,
|
||||
total: vercelTools.length,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('\n❌ Sync failed:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Run if called directly
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main()
|
||||
.then((result) => {
|
||||
console.log('\n📊 Final Results:');
|
||||
console.log(` Processed: ${result.processed}`);
|
||||
console.log(` Skipped: ${result.skipped}`);
|
||||
console.log(` Errors: ${result.errors}`);
|
||||
console.log(` Total: ${result.total}\n`);
|
||||
process.exit(0);
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error('Fatal error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
export { main as syncVercelRegistry };
|
||||
Loading…
Add table
Add a link
Reference in a new issue