feat: implement multi-tool package architecture with manual tool registry
BREAKING CHANGE: Complete refactoring from single-tool to multi-tool package support Database Schema: - Split Tool model into Package (1) and Tool (many) with one-to-many relationship - Package stores npm metadata and package-level tpmjs fields (category, env, frameworks, tier) - Tool stores individual tool exports with tool-level metadata (exportName, description, parameters, returns, aiAgent) - Unique constraint on (packageId, exportName) to prevent duplicate tools - Cascade deletes when packages are removed Type System: - Updated tpmjs field schema to support tools array - Each tool has exportName, description, parameters, returns, aiAgent - Package-level fields: category, env, frameworks shared across all tools - Backward compatible with legacy single-tool format (auto-migrates to exportName: "default") API Updates: - Updated all /api/tools routes to query Tool model with Package relations - Updated /api/tools/[slug] to accept package/export path segments - Updated tool-executor-agent to use actual exportName instead of hardcoded "default" - Updated metrics sync to calculate quality scores per Tool Frontend Updates: - Updated tool search page to display exportName as primary heading - Updated tool detail pages to show package name as secondary info - Removed tag-based filtering (tags moved to package level) Manual Tool Registry: - Added manual-tools.ts with 23 curated tools from major providers - Created sync-manual-tools.ts script to sync manual tools to database - Added MANUAL_TOOLS.md documentation for manual tool system - Added GitHub workflow for automated daily sync - Includes tools from: Vercel, Exa, Firecrawl, AWS Bedrock, Perplexity, Tavily, Superagent, Valyu Playground Updates: - Updated tool loader to load multiple tools per package - Added sanitizeToolName for OpenAI API compatibility Sync System Updates: - Updated changes feed sync to handle multi-tool packages - Updated keyword sync to upsert multiple tools per package - Added orphaned tool deletion when tools removed from package.json Migration Strategy: - Database uses same Neon instance for dev and prod - Schema updated via prisma db push (no migration files yet) - All data repopulates from npm via sync system 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
fdd1b2c304
commit
0612eac5e2
31 changed files with 3783 additions and 786 deletions
138
sync-manual-tools.ts
Normal file
138
sync-manual-tools.ts
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
import { prisma } from './packages/db/src/index.js';
|
||||
import { manualTools } from './manual-tools.js';
|
||||
import { fetchLatestPackageWithMetadata } from './packages/npm-client/src/package.js';
|
||||
|
||||
async function syncManualTools() {
|
||||
console.log('\n🔧 Starting manual tools sync...\n');
|
||||
|
||||
let processed = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
|
||||
for (const manualTool of manualTools) {
|
||||
try {
|
||||
console.log(`Processing: ${manualTool.npmPackageName} (${manualTool.exportName})`);
|
||||
|
||||
// Fetch package metadata from npm
|
||||
const npmData = await fetchLatestPackageWithMetadata(manualTool.npmPackageName);
|
||||
|
||||
if (!npmData) {
|
||||
console.error(` ❌ Package not found on npm: ${manualTool.npmPackageName}`);
|
||||
errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Use manual version if specified, otherwise use latest from npm
|
||||
const version = manualTool.npmVersion || npmData.version;
|
||||
|
||||
// Get published date
|
||||
const publishedAt = npmData.time?.[version] || npmData.time?.modified || new Date().toISOString();
|
||||
|
||||
// Upsert Package record
|
||||
const packageRecord = await prisma.package.upsert({
|
||||
where: { npmPackageName: manualTool.npmPackageName },
|
||||
create: {
|
||||
npmPackageName: manualTool.npmPackageName,
|
||||
npmVersion: version,
|
||||
npmPublishedAt: new Date(publishedAt),
|
||||
npmDescription: npmData.description || null,
|
||||
npmRepository: npmData.repository || null,
|
||||
npmHomepage: npmData.homepage || manualTool.websiteUrl || null,
|
||||
npmLicense: npmData.license || null,
|
||||
npmKeywords: npmData.keywords || [],
|
||||
npmReadme: npmData.readme || null,
|
||||
npmAuthor: npmData.author || null,
|
||||
npmMaintainers: npmData.maintainers || null,
|
||||
category: manualTool.category,
|
||||
env: manualTool.env ? (manualTool.env as any) : null,
|
||||
frameworks: manualTool.frameworks,
|
||||
tier: calculateTier(manualTool),
|
||||
discoveryMethod: 'manual',
|
||||
isOfficial: manualTool.npmPackageName.startsWith('@tpmjs/'),
|
||||
npmDownloadsLastMonth: 0,
|
||||
},
|
||||
update: {
|
||||
npmVersion: version,
|
||||
npmPublishedAt: new Date(publishedAt),
|
||||
npmDescription: npmData.description || null,
|
||||
npmRepository: npmData.repository || null,
|
||||
npmHomepage: npmData.homepage || manualTool.websiteUrl || null,
|
||||
npmLicense: npmData.license || null,
|
||||
npmKeywords: npmData.keywords || [],
|
||||
npmReadme: npmData.readme || null,
|
||||
npmAuthor: npmData.author || null,
|
||||
npmMaintainers: npmData.maintainers || null,
|
||||
category: manualTool.category,
|
||||
env: manualTool.env ? (manualTool.env as any) : null,
|
||||
frameworks: manualTool.frameworks,
|
||||
tier: calculateTier(manualTool),
|
||||
isOfficial: manualTool.npmPackageName.startsWith('@tpmjs/'),
|
||||
},
|
||||
});
|
||||
|
||||
console.log(` ✅ Package upserted: ${packageRecord.id}`);
|
||||
|
||||
// Get existing tools for this package
|
||||
const existingTools = await prisma.tool.findMany({
|
||||
where: { packageId: packageRecord.id },
|
||||
});
|
||||
|
||||
// Upsert the tool
|
||||
const tool = await prisma.tool.upsert({
|
||||
where: {
|
||||
packageId_exportName: {
|
||||
packageId: packageRecord.id,
|
||||
exportName: manualTool.exportName,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
packageId: packageRecord.id,
|
||||
exportName: manualTool.exportName,
|
||||
description: manualTool.description,
|
||||
parameters: manualTool.parameters ? (manualTool.parameters as any) : null,
|
||||
returns: manualTool.returns ? (manualTool.returns as any) : null,
|
||||
aiAgent: manualTool.aiAgent ? (manualTool.aiAgent as any) : null,
|
||||
},
|
||||
update: {
|
||||
description: manualTool.description,
|
||||
parameters: manualTool.parameters ? (manualTool.parameters as any) : null,
|
||||
returns: manualTool.returns ? (manualTool.returns as any) : null,
|
||||
aiAgent: manualTool.aiAgent ? (manualTool.aiAgent as any) : null,
|
||||
},
|
||||
});
|
||||
|
||||
console.log(` ✅ Tool upserted: ${tool.exportName} (${tool.id})`);
|
||||
processed++;
|
||||
} catch (error) {
|
||||
console.error(
|
||||
` ❌ Error processing ${manualTool.npmPackageName} (${manualTool.exportName}):`,
|
||||
error
|
||||
);
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n📊 Manual sync complete!');
|
||||
console.log(` Processed: ${processed}`);
|
||||
console.log(` Skipped: ${skipped}`);
|
||||
console.log(` Errors: ${errors}`);
|
||||
console.log(` Total manual tools: ${manualTools.length}\n`);
|
||||
}
|
||||
|
||||
function calculateTier(tool: typeof manualTools[0]): 'minimal' | 'rich' {
|
||||
// Tier is 'rich' if tool has parameters OR returns OR aiAgent
|
||||
if (tool.parameters || tool.returns || tool.aiAgent) {
|
||||
return 'rich';
|
||||
}
|
||||
return 'minimal';
|
||||
}
|
||||
|
||||
syncManualTools()
|
||||
.then(() => process.exit(0))
|
||||
.catch((error) => {
|
||||
console.error('❌ Manual sync failed:', error);
|
||||
process.exit(1);
|
||||
})
|
||||
.finally(() => {
|
||||
prisma.$disconnect();
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue