tpmjs/sync-manual-tools.ts
Ajax Davis 141a64d888 feat: implement multi-tool package architecture with manual tool registry
BREAKING CHANGE: Complete refactoring from single-tool to multi-tool package support

Database Schema:
- Split Tool model into Package (1) and Tool (many) with one-to-many relationship
- Package stores npm metadata and package-level tpmjs fields (category, env, frameworks, tier)
- Tool stores individual tool exports with tool-level metadata (exportName, description, parameters, returns, aiAgent)
- Unique constraint on (packageId, exportName) to prevent duplicate tools
- Cascade deletes when packages are removed

Type System:
- Updated tpmjs field schema to support tools array
- Each tool has exportName, description, parameters, returns, aiAgent
- Package-level fields: category, env, frameworks shared across all tools
- Backward compatible with legacy single-tool format (auto-migrates to exportName: "default")

API Updates:
- Updated all /api/tools routes to query Tool model with Package relations
- Updated /api/tools/[slug] to accept package/export path segments
- Updated tool-executor-agent to use actual exportName instead of hardcoded "default"
- Updated metrics sync to calculate quality scores per Tool

Frontend Updates:
- Updated tool search page to display exportName as primary heading
- Updated tool detail pages to show package name as secondary info
- Removed tag-based filtering (tags moved to package level)

Manual Tool Registry:
- Added manual-tools.ts with 23 curated tools from major providers
- Created sync-manual-tools.ts script to sync manual tools to database
- Added MANUAL_TOOLS.md documentation for manual tool system
- Added GitHub workflow for automated daily sync
- Includes tools from: Vercel, Exa, Firecrawl, AWS Bedrock, Perplexity, Tavily, Superagent, Valyu

Playground Updates:
- Updated tool loader to load multiple tools per package
- Added sanitizeToolName for OpenAI API compatibility

Sync System Updates:
- Updated changes feed sync to handle multi-tool packages
- Updated keyword sync to upsert multiple tools per package
- Added orphaned tool deletion when tools removed from package.json

Migration Strategy:
- Database uses same Neon instance for dev and prod
- Schema updated via prisma db push (no migration files yet)
- All data repopulates from npm via sync system

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2025-12-04 06:39:58 +10:00

138 lines
4.9 KiB
TypeScript

import { prisma } from './packages/db/src/index.js';
import { manualTools } from './manual-tools.js';
import { fetchLatestPackageWithMetadata } from './packages/npm-client/src/package.js';
async function syncManualTools() {
console.log('\n🔧 Starting manual tools sync...\n');
let processed = 0;
let skipped = 0;
let errors = 0;
for (const manualTool of manualTools) {
try {
console.log(`Processing: ${manualTool.npmPackageName} (${manualTool.exportName})`);
// Fetch package metadata from npm
const npmData = await fetchLatestPackageWithMetadata(manualTool.npmPackageName);
if (!npmData) {
console.error(` ❌ Package not found on npm: ${manualTool.npmPackageName}`);
errors++;
continue;
}
// Use manual version if specified, otherwise use latest from npm
const version = manualTool.npmVersion || npmData.version;
// Get published date
const publishedAt = npmData.time?.[version] || npmData.time?.modified || new Date().toISOString();
// Upsert Package record
const packageRecord = await prisma.package.upsert({
where: { npmPackageName: manualTool.npmPackageName },
create: {
npmPackageName: manualTool.npmPackageName,
npmVersion: version,
npmPublishedAt: new Date(publishedAt),
npmDescription: npmData.description || null,
npmRepository: npmData.repository || null,
npmHomepage: npmData.homepage || manualTool.websiteUrl || null,
npmLicense: npmData.license || null,
npmKeywords: npmData.keywords || [],
npmReadme: npmData.readme || null,
npmAuthor: npmData.author || null,
npmMaintainers: npmData.maintainers || null,
category: manualTool.category,
env: manualTool.env ? (manualTool.env as any) : null,
frameworks: manualTool.frameworks,
tier: calculateTier(manualTool),
discoveryMethod: 'manual',
isOfficial: manualTool.npmPackageName.startsWith('@tpmjs/'),
npmDownloadsLastMonth: 0,
},
update: {
npmVersion: version,
npmPublishedAt: new Date(publishedAt),
npmDescription: npmData.description || null,
npmRepository: npmData.repository || null,
npmHomepage: npmData.homepage || manualTool.websiteUrl || null,
npmLicense: npmData.license || null,
npmKeywords: npmData.keywords || [],
npmReadme: npmData.readme || null,
npmAuthor: npmData.author || null,
npmMaintainers: npmData.maintainers || null,
category: manualTool.category,
env: manualTool.env ? (manualTool.env as any) : null,
frameworks: manualTool.frameworks,
tier: calculateTier(manualTool),
isOfficial: manualTool.npmPackageName.startsWith('@tpmjs/'),
},
});
console.log(` ✅ Package upserted: ${packageRecord.id}`);
// Get existing tools for this package
const existingTools = await prisma.tool.findMany({
where: { packageId: packageRecord.id },
});
// Upsert the tool
const tool = await prisma.tool.upsert({
where: {
packageId_exportName: {
packageId: packageRecord.id,
exportName: manualTool.exportName,
},
},
create: {
packageId: packageRecord.id,
exportName: manualTool.exportName,
description: manualTool.description,
parameters: manualTool.parameters ? (manualTool.parameters as any) : null,
returns: manualTool.returns ? (manualTool.returns as any) : null,
aiAgent: manualTool.aiAgent ? (manualTool.aiAgent as any) : null,
},
update: {
description: manualTool.description,
parameters: manualTool.parameters ? (manualTool.parameters as any) : null,
returns: manualTool.returns ? (manualTool.returns as any) : null,
aiAgent: manualTool.aiAgent ? (manualTool.aiAgent as any) : null,
},
});
console.log(` ✅ Tool upserted: ${tool.exportName} (${tool.id})`);
processed++;
} catch (error) {
console.error(
` ❌ Error processing ${manualTool.npmPackageName} (${manualTool.exportName}):`,
error
);
errors++;
}
}
console.log('\n📊 Manual sync complete!');
console.log(` Processed: ${processed}`);
console.log(` Skipped: ${skipped}`);
console.log(` Errors: ${errors}`);
console.log(` Total manual tools: ${manualTools.length}\n`);
}
function calculateTier(tool: typeof manualTools[0]): 'minimal' | 'rich' {
// Tier is 'rich' if tool has parameters OR returns OR aiAgent
if (tool.parameters || tool.returns || tool.aiAgent) {
return 'rich';
}
return 'minimal';
}
syncManualTools()
.then(() => process.exit(0))
.catch((error) => {
console.error('❌ Manual sync failed:', error);
process.exit(1);
})
.finally(() => {
prisma.$disconnect();
});