feat: add chunked skills.md generation for large collections
Implements batched generation to avoid Vercel 120s timeout for collections with 60+ tools. Uses per-tool caching and recursive serverless invocations. - Add ToolSkillsCache and SkillsGenerationJob models to schema - Create tool-skills-generator.ts for per-tool markdown generation - Create skills-summary-generator.ts for final pass summary/intro - Update route handler with chunked generation logic - Small collections (<20 tools) use original monolithic approach - Large collections use 10-tool batches with progress tracking
This commit is contained in:
Ajax Davis
parent
514ea3c0db
commit
5d00f2a711
39 changed files with 6068 additions and 286 deletions
|
|
@ -75,6 +75,86 @@
|
|||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "playground.js"]
|
||||
},
|
||||
"run": {
|
||||
"aliases": [],
|
||||
"args": {},
|
||||
"description": "Execute a tool from a collection via MCP",
|
||||
"examples": [
|
||||
"<%= config.bin %> run -c ajax/unsandbox -t execute --args '{\"code\":\"print(1)\",\"language\":\"python\"}'",
|
||||
"<%= config.bin %> run --collection ajax/ajax-collection --tool base64Encode --args '{\"data\":\"hello\"}'",
|
||||
"OPENAI_API_KEY=xxx <%= config.bin %> run -c ajax/my-collection -t myTool",
|
||||
"<%= config.bin %> run -c ajax/tools -t search --args '{\"query\":\"test\"}' --json",
|
||||
"<%= config.bin %> run -c ajax/tools -t search --env API_KEY=xxx --env DEBUG=true"
|
||||
],
|
||||
"flags": {
|
||||
"collection": {
|
||||
"char": "c",
|
||||
"description": "Collection identifier (username/slug)",
|
||||
"name": "collection",
|
||||
"required": true,
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"tool": {
|
||||
"char": "t",
|
||||
"description": "Tool name (MCP format: package--toolName)",
|
||||
"name": "tool",
|
||||
"required": true,
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"args": {
|
||||
"char": "a",
|
||||
"description": "Tool arguments as JSON string",
|
||||
"name": "args",
|
||||
"default": "{}",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"env": {
|
||||
"char": "e",
|
||||
"description": "Environment variables (key=value)",
|
||||
"name": "env",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": true,
|
||||
"type": "option"
|
||||
},
|
||||
"json": {
|
||||
"description": "Output result as JSON",
|
||||
"name": "json",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"verbose": {
|
||||
"char": "v",
|
||||
"description": "Show verbose output",
|
||||
"name": "verbose",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"timeout": {
|
||||
"description": "Request timeout in seconds",
|
||||
"name": "timeout",
|
||||
"default": 60,
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
}
|
||||
},
|
||||
"hasDynamicHelp": false,
|
||||
"hiddenAliases": [],
|
||||
"id": "run",
|
||||
"pluginAlias": "@tpmjs/cli",
|
||||
"pluginName": "@tpmjs/cli",
|
||||
"pluginType": "core",
|
||||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "run.js"]
|
||||
},
|
||||
"update": {
|
||||
"aliases": [],
|
||||
"args": {},
|
||||
|
|
@ -614,6 +694,127 @@
|
|||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "auth", "whoami.js"]
|
||||
},
|
||||
"mcp:config": {
|
||||
"aliases": [],
|
||||
"args": {
|
||||
"collection": {
|
||||
"description": "Collection path (username/slug)",
|
||||
"name": "collection",
|
||||
"required": true
|
||||
}
|
||||
},
|
||||
"description": "Generate MCP configuration for AI clients",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %> ajax/ajax-collection",
|
||||
"<%= config.bin %> <%= command.id %> ajax/ajax-collection --client cursor",
|
||||
"<%= config.bin %> <%= command.id %> ajax/ajax-collection --output ~/Library/Application\\ Support/Claude/claude_desktop_config.json"
|
||||
],
|
||||
"flags": {
|
||||
"client": {
|
||||
"char": "c",
|
||||
"description": "Target client (claude, cursor, windsurf, generic)",
|
||||
"name": "client",
|
||||
"default": "claude",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"options": ["claude", "cursor", "windsurf", "generic"],
|
||||
"type": "option"
|
||||
},
|
||||
"output": {
|
||||
"char": "o",
|
||||
"description": "Output file path (will merge with existing config)",
|
||||
"name": "output",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
"name": "json",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"api-key": {
|
||||
"char": "k",
|
||||
"description": "API key to include in config (optional)",
|
||||
"name": "api-key",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
}
|
||||
},
|
||||
"hasDynamicHelp": false,
|
||||
"hiddenAliases": [],
|
||||
"id": "mcp:config",
|
||||
"pluginAlias": "@tpmjs/cli",
|
||||
"pluginName": "@tpmjs/cli",
|
||||
"pluginType": "core",
|
||||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "mcp", "config.js"]
|
||||
},
|
||||
"mcp:serve": {
|
||||
"aliases": [],
|
||||
"args": {},
|
||||
"description": "Run as a local MCP server",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %>",
|
||||
"<%= config.bin %> <%= command.id %> --port 8080",
|
||||
"<%= config.bin %> <%= command.id %> --stdio",
|
||||
"<%= config.bin %> <%= command.id %> --collection my-collection"
|
||||
],
|
||||
"flags": {
|
||||
"port": {
|
||||
"char": "p",
|
||||
"description": "Port to run the server on (HTTP mode)",
|
||||
"name": "port",
|
||||
"default": 3333,
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"stdio": {
|
||||
"description": "Use stdio transport instead of HTTP",
|
||||
"name": "stdio",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"collection": {
|
||||
"char": "c",
|
||||
"description": "Serve tools from a specific collection",
|
||||
"name": "collection",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"tool": {
|
||||
"char": "t",
|
||||
"description": "Serve specific tools (comma-separated)",
|
||||
"name": "tool",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": true,
|
||||
"type": "option"
|
||||
},
|
||||
"verbose": {
|
||||
"char": "v",
|
||||
"description": "Show verbose output",
|
||||
"name": "verbose",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"hasDynamicHelp": false,
|
||||
"hiddenAliases": [],
|
||||
"id": "mcp:serve",
|
||||
"pluginAlias": "@tpmjs/cli",
|
||||
"pluginName": "@tpmjs/cli",
|
||||
"pluginType": "core",
|
||||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "mcp", "serve.js"]
|
||||
},
|
||||
"collection:add": {
|
||||
"aliases": [],
|
||||
"args": {
|
||||
|
|
@ -963,127 +1164,6 @@
|
|||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "collection", "update.js"]
|
||||
},
|
||||
"mcp:config": {
|
||||
"aliases": [],
|
||||
"args": {
|
||||
"collection": {
|
||||
"description": "Collection path (username/slug)",
|
||||
"name": "collection",
|
||||
"required": true
|
||||
}
|
||||
},
|
||||
"description": "Generate MCP configuration for AI clients",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %> ajax/ajax-collection",
|
||||
"<%= config.bin %> <%= command.id %> ajax/ajax-collection --client cursor",
|
||||
"<%= config.bin %> <%= command.id %> ajax/ajax-collection --output ~/Library/Application\\ Support/Claude/claude_desktop_config.json"
|
||||
],
|
||||
"flags": {
|
||||
"client": {
|
||||
"char": "c",
|
||||
"description": "Target client (claude, cursor, windsurf, generic)",
|
||||
"name": "client",
|
||||
"default": "claude",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"options": ["claude", "cursor", "windsurf", "generic"],
|
||||
"type": "option"
|
||||
},
|
||||
"output": {
|
||||
"char": "o",
|
||||
"description": "Output file path (will merge with existing config)",
|
||||
"name": "output",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"json": {
|
||||
"description": "Output in JSON format",
|
||||
"name": "json",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"api-key": {
|
||||
"char": "k",
|
||||
"description": "API key to include in config (optional)",
|
||||
"name": "api-key",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
}
|
||||
},
|
||||
"hasDynamicHelp": false,
|
||||
"hiddenAliases": [],
|
||||
"id": "mcp:config",
|
||||
"pluginAlias": "@tpmjs/cli",
|
||||
"pluginName": "@tpmjs/cli",
|
||||
"pluginType": "core",
|
||||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "mcp", "config.js"]
|
||||
},
|
||||
"mcp:serve": {
|
||||
"aliases": [],
|
||||
"args": {},
|
||||
"description": "Run as a local MCP server",
|
||||
"examples": [
|
||||
"<%= config.bin %> <%= command.id %>",
|
||||
"<%= config.bin %> <%= command.id %> --port 8080",
|
||||
"<%= config.bin %> <%= command.id %> --stdio",
|
||||
"<%= config.bin %> <%= command.id %> --collection my-collection"
|
||||
],
|
||||
"flags": {
|
||||
"port": {
|
||||
"char": "p",
|
||||
"description": "Port to run the server on (HTTP mode)",
|
||||
"name": "port",
|
||||
"default": 3333,
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"stdio": {
|
||||
"description": "Use stdio transport instead of HTTP",
|
||||
"name": "stdio",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"collection": {
|
||||
"char": "c",
|
||||
"description": "Serve tools from a specific collection",
|
||||
"name": "collection",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": false,
|
||||
"type": "option"
|
||||
},
|
||||
"tool": {
|
||||
"char": "t",
|
||||
"description": "Serve specific tools (comma-separated)",
|
||||
"name": "tool",
|
||||
"hasDynamicHelp": false,
|
||||
"multiple": true,
|
||||
"type": "option"
|
||||
},
|
||||
"verbose": {
|
||||
"char": "v",
|
||||
"description": "Show verbose output",
|
||||
"name": "verbose",
|
||||
"allowNo": false,
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"hasDynamicHelp": false,
|
||||
"hiddenAliases": [],
|
||||
"id": "mcp:serve",
|
||||
"pluginAlias": "@tpmjs/cli",
|
||||
"pluginName": "@tpmjs/cli",
|
||||
"pluginType": "core",
|
||||
"strict": true,
|
||||
"enableJsonFlag": false,
|
||||
"isESM": true,
|
||||
"relativePath": ["dist", "commands", "mcp", "serve.js"]
|
||||
},
|
||||
"publish:check": {
|
||||
"aliases": [],
|
||||
"args": {
|
||||
|
|
|
|||
|
|
@ -108,6 +108,7 @@ model Tool {
|
|||
likes ToolLike[]
|
||||
ratings ToolRating[]
|
||||
reviews ToolReview[]
|
||||
skillsCache ToolSkillsCache?
|
||||
|
||||
@@unique([packageId, name])
|
||||
@@index([qualityScore])
|
||||
|
|
@ -467,11 +468,12 @@ model Collection {
|
|||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
|
||||
// Relations
|
||||
tools CollectionTool[]
|
||||
agents AgentCollection[]
|
||||
likes CollectionLike[]
|
||||
bridgeTools CollectionBridgeTool[]
|
||||
scenarios Scenario[]
|
||||
tools CollectionTool[]
|
||||
agents AgentCollection[]
|
||||
likes CollectionLike[]
|
||||
bridgeTools CollectionBridgeTool[]
|
||||
scenarios Scenario[]
|
||||
skillsGenerationJobs SkillsGenerationJob[]
|
||||
|
||||
// Unique constraint: user can't have duplicate collection slugs
|
||||
@@unique([userId, slug])
|
||||
|
|
@ -1418,3 +1420,53 @@ model SocialProof {
|
|||
@@index([totalRuns])
|
||||
@@map("social_proofs")
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Skills Generation Models (Chunked Generation for Large Collections)
|
||||
// ============================================================================
|
||||
|
||||
/// ToolSkillsCache - per-tool cached skills markdown section
|
||||
model ToolSkillsCache {
|
||||
id String @id @default(cuid())
|
||||
|
||||
// Tool relationship
|
||||
toolId String @unique @map("tool_id")
|
||||
tool Tool @relation(fields: [toolId], references: [id], onDelete: Cascade)
|
||||
|
||||
// Cached markdown section for this tool
|
||||
skillsMarkdown String @map("skills_markdown") @db.Text
|
||||
|
||||
// Timestamps
|
||||
generatedAt DateTime @default(now()) @map("generated_at")
|
||||
|
||||
@@index([toolId])
|
||||
@@map("tool_skills_cache")
|
||||
}
|
||||
|
||||
/// SkillsGenerationJob - tracks chunked skills.md generation progress
|
||||
model SkillsGenerationJob {
|
||||
id String @id @default(cuid())
|
||||
|
||||
// Collection relationship
|
||||
collectionId String @map("collection_id")
|
||||
collection Collection @relation(fields: [collectionId], references: [id], onDelete: Cascade)
|
||||
|
||||
// Job status
|
||||
status String @default("pending") @db.VarChar(20) // 'pending' | 'processing' | 'completed' | 'failed'
|
||||
|
||||
// Progress tracking
|
||||
currentBatch Int @default(0) @map("current_batch")
|
||||
totalBatches Int @map("total_batches")
|
||||
completedToolIds String[] @default([]) @map("completed_tool_ids")
|
||||
|
||||
// Error tracking
|
||||
error String? @db.Text
|
||||
|
||||
// Timestamps
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
|
||||
@@index([collectionId])
|
||||
@@index([status])
|
||||
@@map("skills_generation_jobs")
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue