feat: add chunked skills.md generation for large collections

Implements batched generation to avoid Vercel 120s timeout for collections
with 60+ tools. Uses per-tool caching and recursive serverless invocations.

- Add ToolSkillsCache and SkillsGenerationJob models to schema
- Create tool-skills-generator.ts for per-tool markdown generation
- Create skills-summary-generator.ts for final pass summary/intro
- Update route handler with chunked generation logic
- Small collections (<20 tools) use original monolithic approach
- Large collections use 10-tool batches with progress tracking
This commit is contained in:
Ajax Davis 2026-01-21 16:26:06 +10:00
parent 514ea3c0db
commit 5d00f2a711
39 changed files with 6068 additions and 286 deletions

View file

@ -108,6 +108,7 @@ model Tool {
likes ToolLike[]
ratings ToolRating[]
reviews ToolReview[]
skillsCache ToolSkillsCache?
@@unique([packageId, name])
@@index([qualityScore])
@ -467,11 +468,12 @@ model Collection {
updatedAt DateTime @updatedAt @map("updated_at")
// Relations
tools CollectionTool[]
agents AgentCollection[]
likes CollectionLike[]
bridgeTools CollectionBridgeTool[]
scenarios Scenario[]
tools CollectionTool[]
agents AgentCollection[]
likes CollectionLike[]
bridgeTools CollectionBridgeTool[]
scenarios Scenario[]
skillsGenerationJobs SkillsGenerationJob[]
// Unique constraint: user can't have duplicate collection slugs
@@unique([userId, slug])
@ -1418,3 +1420,53 @@ model SocialProof {
@@index([totalRuns])
@@map("social_proofs")
}
// ============================================================================
// Skills Generation Models (Chunked Generation for Large Collections)
// ============================================================================
/// ToolSkillsCache - per-tool cached skills markdown section
model ToolSkillsCache {
id String @id @default(cuid())
// Tool relationship
toolId String @unique @map("tool_id")
tool Tool @relation(fields: [toolId], references: [id], onDelete: Cascade)
// Cached markdown section for this tool
skillsMarkdown String @map("skills_markdown") @db.Text
// Timestamps
generatedAt DateTime @default(now()) @map("generated_at")
@@index([toolId])
@@map("tool_skills_cache")
}
/// SkillsGenerationJob - tracks chunked skills.md generation progress
model SkillsGenerationJob {
id String @id @default(cuid())
// Collection relationship
collectionId String @map("collection_id")
collection Collection @relation(fields: [collectionId], references: [id], onDelete: Cascade)
// Job status
status String @default("pending") @db.VarChar(20) // 'pending' | 'processing' | 'completed' | 'failed'
// Progress tracking
currentBatch Int @default(0) @map("current_batch")
totalBatches Int @map("total_batches")
completedToolIds String[] @default([]) @map("completed_tool_ids")
// Error tracking
error String? @db.Text
// Timestamps
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
@@index([collectionId])
@@index([status])
@@map("skills_generation_jobs")
}