feat: add chunked skills.md generation for large collections
Implements batched generation to avoid Vercel 120s timeout for collections with 60+ tools. Uses per-tool caching and recursive serverless invocations. - Add ToolSkillsCache and SkillsGenerationJob models to schema - Create tool-skills-generator.ts for per-tool markdown generation - Create skills-summary-generator.ts for final pass summary/intro - Update route handler with chunked generation logic - Small collections (<20 tools) use original monolithic approach - Large collections use 10-tool batches with progress tracking
This commit is contained in:
Ajax Davis
parent
514ea3c0db
commit
5d00f2a711
39 changed files with 6068 additions and 286 deletions
|
|
@ -1,26 +1,144 @@
|
|||
/**
|
||||
* GET /:username/collections/:slug/skills.md
|
||||
* Generate AI-powered skills documentation for a collection
|
||||
* Clean URL endpoint for machine-readable skills documentation
|
||||
*
|
||||
* Supports chunked generation for large collections (60+ tools) to avoid
|
||||
* Vercel's 120s timeout limit. Uses per-tool caching and recursive batch
|
||||
* processing.
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import type { NextRequest } from 'next/server';
|
||||
|
||||
import { fetchMultiplePackageSources } from '~/lib/ai/package-source-fetcher';
|
||||
import {
|
||||
fetchMultiplePackageSources,
|
||||
fetchPackageSource,
|
||||
type PackageSource,
|
||||
} from '~/lib/ai/package-source-fetcher';
|
||||
import { generateSkillsMarkdown } from '~/lib/ai/skills-generator';
|
||||
import { assembleSkillsDocument, generateSkillsSummary } from '~/lib/ai/skills-summary-generator';
|
||||
import { generateToolSkillsBatch } from '~/lib/ai/tool-skills-generator';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 120; // Extended for source fetching + AI generation
|
||||
export const maxDuration = 300; // Extended for chunked generation (5 min)
|
||||
|
||||
// Cache for 1 week
|
||||
const CACHE_DURATION_SECONDS = 7 * 24 * 60 * 60; // 604800 seconds
|
||||
|
||||
// Batch size for chunked generation
|
||||
const BATCH_SIZE = 10;
|
||||
|
||||
// Threshold for using chunked generation (tools)
|
||||
const CHUNKED_THRESHOLD = 20;
|
||||
|
||||
type RouteContext = {
|
||||
params: Promise<{ username: string; slug: string }>;
|
||||
};
|
||||
|
||||
interface CollectionWithTools {
|
||||
id: string;
|
||||
name: string;
|
||||
slug: string | null;
|
||||
description: string | null;
|
||||
isPublic: boolean;
|
||||
skillsMarkdown: string | null;
|
||||
skillsGeneratedAt: Date | null;
|
||||
tools: Array<{
|
||||
tool: {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
inputSchema: unknown;
|
||||
package: {
|
||||
npmPackageName: string;
|
||||
npmVersion: string;
|
||||
};
|
||||
};
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load collection with tools from database
|
||||
*/
|
||||
async function loadCollection(
|
||||
username: string,
|
||||
slug: string
|
||||
): Promise<{ user: { username: string }; collection: CollectionWithTools } | Response> {
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { username },
|
||||
select: { id: true, username: true },
|
||||
});
|
||||
|
||||
if (!user || !user.username) {
|
||||
return new Response('User not found', {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'text/plain' },
|
||||
});
|
||||
}
|
||||
|
||||
const collection = await prisma.collection.findFirst({
|
||||
where: { slug, userId: user.id },
|
||||
include: {
|
||||
tools: {
|
||||
include: {
|
||||
tool: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
description: true,
|
||||
inputSchema: true,
|
||||
package: { select: { npmPackageName: true, npmVersion: true } },
|
||||
},
|
||||
},
|
||||
},
|
||||
orderBy: { position: 'asc' },
|
||||
take: 100,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!collection) {
|
||||
return new Response('Collection not found', {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'text/plain' },
|
||||
});
|
||||
}
|
||||
|
||||
if (!collection.isPublic) {
|
||||
return new Response('This collection is not public', {
|
||||
status: 403,
|
||||
headers: { 'Content-Type': 'text/plain' },
|
||||
});
|
||||
}
|
||||
|
||||
return { user: { username: user.username }, collection };
|
||||
}
|
||||
|
||||
/**
|
||||
* Check cache and return cached response if valid
|
||||
*/
|
||||
function checkCache(collection: CollectionWithTools, cachebust: boolean): Response | null {
|
||||
if (cachebust || !collection.skillsMarkdown || !collection.skillsGeneratedAt) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const cacheAge = Date.now() - collection.skillsGeneratedAt.getTime();
|
||||
if (cacheAge >= CACHE_DURATION_SECONDS * 1000) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return new Response(collection.skillsMarkdown, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/markdown; charset=utf-8',
|
||||
'Cache-Control': `public, s-maxage=${CACHE_DURATION_SECONDS}, stale-while-revalidate=86400`,
|
||||
'X-Cache': 'HIT',
|
||||
'X-Cache-Age': Math.floor(cacheAge / 1000).toString(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /:username/collections/:slug/skills.md
|
||||
* Generate skills.md markdown for a collection
|
||||
|
|
@ -30,116 +148,31 @@ export async function GET(request: NextRequest, context: RouteContext) {
|
|||
const { username: rawUsername, slug } = await context.params;
|
||||
const username = rawUsername.startsWith('@') ? rawUsername.slice(1) : rawUsername;
|
||||
|
||||
// Check for cachebust query param
|
||||
const cachebust = request.nextUrl.searchParams.has('cachebust');
|
||||
const batchIndex = request.nextUrl.searchParams.get('_batch');
|
||||
const jobId = request.nextUrl.searchParams.get('_jobId');
|
||||
|
||||
// Find the user first
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { username },
|
||||
select: { id: true, username: true },
|
||||
});
|
||||
|
||||
if (!user || !user.username) {
|
||||
return new Response('User not found', {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'text/plain' },
|
||||
});
|
||||
// Handle batch continuation requests
|
||||
if (batchIndex !== null && jobId) {
|
||||
return handleBatchContinuation(jobId, parseInt(batchIndex, 10), username, slug);
|
||||
}
|
||||
|
||||
// Find the collection by slug belonging to this user
|
||||
const collection = await prisma.collection.findFirst({
|
||||
where: {
|
||||
slug,
|
||||
userId: user.id,
|
||||
},
|
||||
include: {
|
||||
tools: {
|
||||
include: {
|
||||
tool: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
description: true,
|
||||
inputSchema: true,
|
||||
package: {
|
||||
select: {
|
||||
npmPackageName: true,
|
||||
npmVersion: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
orderBy: { position: 'asc' },
|
||||
take: 50, // Limit tools for performance
|
||||
},
|
||||
},
|
||||
});
|
||||
// Load collection from database
|
||||
const result = await loadCollection(username, slug);
|
||||
if (result instanceof Response) return result;
|
||||
const { user, collection } = result;
|
||||
|
||||
if (!collection) {
|
||||
return new Response('Collection not found', {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'text/plain' },
|
||||
});
|
||||
}
|
||||
// Check cache
|
||||
const cachedResponse = checkCache(collection, cachebust);
|
||||
if (cachedResponse) return cachedResponse;
|
||||
|
||||
// Only return if public
|
||||
if (!collection.isPublic) {
|
||||
return new Response('This collection is not public', {
|
||||
status: 403,
|
||||
headers: { 'Content-Type': 'text/plain' },
|
||||
});
|
||||
}
|
||||
|
||||
// Check cache if not busting
|
||||
if (!cachebust && collection.skillsMarkdown && collection.skillsGeneratedAt) {
|
||||
// Check if cache is still valid (within 1 week)
|
||||
const cacheAge = Date.now() - collection.skillsGeneratedAt.getTime();
|
||||
if (cacheAge < CACHE_DURATION_SECONDS * 1000) {
|
||||
return new Response(collection.skillsMarkdown, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/markdown; charset=utf-8',
|
||||
'Cache-Control': `public, s-maxage=${CACHE_DURATION_SECONDS}, stale-while-revalidate=86400`,
|
||||
'X-Cache': 'HIT',
|
||||
'X-Cache-Age': Math.floor(cacheAge / 1000).toString(),
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Extract tools data
|
||||
const tools = collection.tools.map((ct) => ({
|
||||
id: ct.tool.id,
|
||||
name: ct.tool.name,
|
||||
description: ct.tool.description,
|
||||
packageName: ct.tool.package.npmPackageName,
|
||||
inputSchema: ct.tool.inputSchema,
|
||||
}));
|
||||
|
||||
if (tools.length === 0) {
|
||||
if (collection.tools.length === 0) {
|
||||
return new Response('Collection has no tools', {
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'text/plain' },
|
||||
});
|
||||
}
|
||||
|
||||
// Get unique packages to fetch
|
||||
const uniquePackages = [
|
||||
...new Map(
|
||||
collection.tools.map((ct) => [
|
||||
ct.tool.package.npmPackageName,
|
||||
{
|
||||
name: ct.tool.package.npmPackageName,
|
||||
version: ct.tool.package.npmVersion,
|
||||
},
|
||||
])
|
||||
).values(),
|
||||
];
|
||||
|
||||
// Fetch package sources (limit to 5 packages for performance)
|
||||
const packageSources = await fetchMultiplePackageSources(uniquePackages.slice(0, 5));
|
||||
|
||||
// Build MCP URLs
|
||||
const baseUrl = process.env.NEXT_PUBLIC_APP_URL || 'https://tpmjs.com';
|
||||
const mcpUrls = {
|
||||
|
|
@ -147,46 +180,480 @@ export async function GET(request: NextRequest, context: RouteContext) {
|
|||
sse: `${baseUrl}/api/mcp/${username}/${slug}/sse`,
|
||||
};
|
||||
|
||||
// Generate skills markdown
|
||||
const skillsMarkdown = await generateSkillsMarkdown(
|
||||
{
|
||||
id: collection.id,
|
||||
name: collection.name,
|
||||
slug: collection.slug || slug,
|
||||
description: collection.description,
|
||||
username: user.username,
|
||||
},
|
||||
tools,
|
||||
packageSources,
|
||||
mcpUrls
|
||||
);
|
||||
|
||||
// Save to database for caching
|
||||
await prisma.collection.update({
|
||||
where: { id: collection.id },
|
||||
data: {
|
||||
skillsMarkdown,
|
||||
skillsGeneratedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
return new Response(skillsMarkdown, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/markdown; charset=utf-8',
|
||||
'Cache-Control': `public, s-maxage=${CACHE_DURATION_SECONDS}, stale-while-revalidate=86400`,
|
||||
'X-Cache': 'MISS',
|
||||
},
|
||||
});
|
||||
// Route to appropriate handler based on collection size
|
||||
if (collection.tools.length < CHUNKED_THRESHOLD) {
|
||||
return handleSmallCollection(collection, user.username, mcpUrls);
|
||||
}
|
||||
return handleLargeCollection(collection, user.username, mcpUrls, cachebust);
|
||||
} catch (error) {
|
||||
console.error('[Skills.md Error] GET /:username/collections/:slug/skills.md:', error);
|
||||
|
||||
console.error('[Skills.md Error]:', error);
|
||||
const message =
|
||||
error instanceof Error ? error.message : 'Failed to generate skills documentation';
|
||||
|
||||
return new Response(`Error: ${message}`, {
|
||||
status: 500,
|
||||
headers: { 'Content-Type': 'text/plain' },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle small collections with the original monolithic approach
|
||||
*/
|
||||
async function handleSmallCollection(
|
||||
collection: CollectionWithTools,
|
||||
username: string,
|
||||
mcpUrls: { http: string; sse: string }
|
||||
): Promise<Response> {
|
||||
const tools = collection.tools.map((ct) => ({
|
||||
id: ct.tool.id,
|
||||
name: ct.tool.name,
|
||||
description: ct.tool.description,
|
||||
packageName: ct.tool.package.npmPackageName,
|
||||
inputSchema: ct.tool.inputSchema,
|
||||
}));
|
||||
|
||||
// Get unique packages to fetch
|
||||
const uniquePackages = [
|
||||
...new Map(
|
||||
collection.tools.map((ct) => [
|
||||
ct.tool.package.npmPackageName,
|
||||
{
|
||||
name: ct.tool.package.npmPackageName,
|
||||
version: ct.tool.package.npmVersion,
|
||||
},
|
||||
])
|
||||
).values(),
|
||||
];
|
||||
|
||||
// Fetch package sources (limit to 5 packages for performance)
|
||||
const packageSources = await fetchMultiplePackageSources(uniquePackages.slice(0, 5));
|
||||
|
||||
// Generate skills markdown using original function
|
||||
const skillsMarkdown = await generateSkillsMarkdown(
|
||||
{
|
||||
id: collection.id,
|
||||
name: collection.name,
|
||||
slug: collection.slug || '',
|
||||
description: collection.description,
|
||||
username,
|
||||
},
|
||||
tools,
|
||||
packageSources,
|
||||
mcpUrls
|
||||
);
|
||||
|
||||
// Save to database for caching
|
||||
await prisma.collection.update({
|
||||
where: { id: collection.id },
|
||||
data: {
|
||||
skillsMarkdown,
|
||||
skillsGeneratedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
return new Response(skillsMarkdown, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/markdown; charset=utf-8',
|
||||
'Cache-Control': `public, s-maxage=${CACHE_DURATION_SECONDS}, stale-while-revalidate=86400`,
|
||||
'X-Cache': 'MISS',
|
||||
'X-Generation-Mode': 'monolithic',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle large collections with chunked generation
|
||||
*/
|
||||
async function handleLargeCollection(
|
||||
collection: CollectionWithTools,
|
||||
username: string,
|
||||
mcpUrls: { http: string; sse: string },
|
||||
cachebust: boolean
|
||||
): Promise<Response> {
|
||||
const toolIds = collection.tools.map((ct) => ct.tool.id);
|
||||
|
||||
// Check for existing in-progress job
|
||||
const existingJob = await prisma.skillsGenerationJob.findFirst({
|
||||
where: {
|
||||
collectionId: collection.id,
|
||||
status: { in: ['pending', 'processing'] },
|
||||
// Only consider jobs from the last 10 minutes (prevent stuck jobs)
|
||||
createdAt: { gte: new Date(Date.now() - 10 * 60 * 1000) },
|
||||
},
|
||||
});
|
||||
|
||||
if (existingJob) {
|
||||
// Job in progress - return 202 Accepted with retry hint
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
status: 'processing',
|
||||
message: 'Skills generation in progress',
|
||||
progress: {
|
||||
currentBatch: existingJob.currentBatch,
|
||||
totalBatches: existingJob.totalBatches,
|
||||
completedTools: existingJob.completedToolIds.length,
|
||||
totalTools: toolIds.length,
|
||||
},
|
||||
}),
|
||||
{
|
||||
status: 202,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Retry-After': '10',
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
// Check which tools already have cached skills (unless cachebust)
|
||||
let toolsWithCache: string[] = [];
|
||||
if (!cachebust) {
|
||||
const cachedTools = await prisma.toolSkillsCache.findMany({
|
||||
where: { toolId: { in: toolIds } },
|
||||
select: { toolId: true },
|
||||
});
|
||||
toolsWithCache = cachedTools.map((c) => c.toolId);
|
||||
}
|
||||
|
||||
const toolsNeedingGeneration = toolIds.filter((id) => !toolsWithCache.includes(id));
|
||||
|
||||
// If all tools are cached, assemble final document
|
||||
if (toolsNeedingGeneration.length === 0) {
|
||||
return assembleFinalDocument(collection, username, mcpUrls);
|
||||
}
|
||||
|
||||
// Create a new generation job
|
||||
const totalBatches = Math.ceil(toolsNeedingGeneration.length / BATCH_SIZE);
|
||||
const job = await prisma.skillsGenerationJob.create({
|
||||
data: {
|
||||
collectionId: collection.id,
|
||||
status: 'processing',
|
||||
currentBatch: 0,
|
||||
totalBatches,
|
||||
completedToolIds: toolsWithCache, // Include already-cached tools
|
||||
},
|
||||
});
|
||||
|
||||
// Process the first batch synchronously
|
||||
try {
|
||||
await processBatch(job.id, 0, collection, toolsNeedingGeneration);
|
||||
} catch (error) {
|
||||
console.error('[Skills.md] Batch 0 failed:', error);
|
||||
await prisma.skillsGenerationJob.update({
|
||||
where: { id: job.id },
|
||||
data: {
|
||||
status: 'failed',
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
},
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Check if there are more batches to process
|
||||
if (totalBatches > 1) {
|
||||
// Trigger the next batch via recursive fetch
|
||||
const baseUrl = process.env.NEXT_PUBLIC_APP_URL || 'https://tpmjs.com';
|
||||
const nextBatchUrl = `${baseUrl}/${username}/collections/${collection.slug}/skills.md?_batch=1&_jobId=${job.id}`;
|
||||
|
||||
// Fire and forget - don't await
|
||||
fetch(nextBatchUrl, {
|
||||
method: 'GET',
|
||||
headers: { 'x-internal': 'true' },
|
||||
}).catch((err) => {
|
||||
console.error('[Skills.md] Failed to trigger batch 1:', err);
|
||||
});
|
||||
|
||||
// Return 202 to indicate processing
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
status: 'processing',
|
||||
message: 'Skills generation started',
|
||||
progress: {
|
||||
currentBatch: 1,
|
||||
totalBatches,
|
||||
completedTools: BATCH_SIZE + toolsWithCache.length,
|
||||
totalTools: toolIds.length,
|
||||
},
|
||||
}),
|
||||
{
|
||||
status: 202,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Retry-After': '15',
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
// Single batch - assemble and return final document
|
||||
return assembleFinalDocument(collection, username, mcpUrls);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle batch continuation (recursive call)
|
||||
*/
|
||||
async function handleBatchContinuation(
|
||||
jobId: string,
|
||||
batchIndex: number,
|
||||
username: string,
|
||||
slug: string
|
||||
): Promise<Response> {
|
||||
// Fetch the job
|
||||
const job = await prisma.skillsGenerationJob.findUnique({
|
||||
where: { id: jobId },
|
||||
include: {
|
||||
collection: {
|
||||
include: {
|
||||
tools: {
|
||||
include: {
|
||||
tool: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
description: true,
|
||||
inputSchema: true,
|
||||
package: {
|
||||
select: {
|
||||
npmPackageName: true,
|
||||
npmVersion: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
orderBy: { position: 'asc' },
|
||||
take: 100,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!job || job.status === 'failed' || job.status === 'completed') {
|
||||
return new Response('Job not found or already completed', {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'text/plain' },
|
||||
});
|
||||
}
|
||||
|
||||
const collection = job.collection;
|
||||
const baseUrl = process.env.NEXT_PUBLIC_APP_URL || 'https://tpmjs.com';
|
||||
const mcpUrls = {
|
||||
http: `${baseUrl}/api/mcp/${username}/${slug}/http`,
|
||||
sse: `${baseUrl}/api/mcp/${username}/${slug}/sse`,
|
||||
};
|
||||
|
||||
// Get tools needing generation
|
||||
const allToolIds = collection.tools.map((ct) => ct.tool.id);
|
||||
const toolsNeedingGeneration = allToolIds.filter((id) => !job.completedToolIds.includes(id));
|
||||
|
||||
try {
|
||||
await processBatch(job.id, batchIndex, collection, toolsNeedingGeneration);
|
||||
} catch (error) {
|
||||
console.error(`[Skills.md] Batch ${batchIndex} failed:`, error);
|
||||
await prisma.skillsGenerationJob.update({
|
||||
where: { id: job.id },
|
||||
data: {
|
||||
status: 'failed',
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
},
|
||||
});
|
||||
return new Response('Batch processing failed', {
|
||||
status: 500,
|
||||
headers: { 'Content-Type': 'text/plain' },
|
||||
});
|
||||
}
|
||||
|
||||
// Check if there are more batches
|
||||
if (batchIndex + 1 < job.totalBatches) {
|
||||
// Trigger next batch
|
||||
const nextBatchUrl = `${baseUrl}/${username}/collections/${slug}/skills.md?_batch=${batchIndex + 1}&_jobId=${job.id}`;
|
||||
|
||||
fetch(nextBatchUrl, {
|
||||
method: 'GET',
|
||||
headers: { 'x-internal': 'true' },
|
||||
}).catch((err) => {
|
||||
console.error(`[Skills.md] Failed to trigger batch ${batchIndex + 1}:`, err);
|
||||
});
|
||||
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
status: 'processing',
|
||||
batchCompleted: batchIndex,
|
||||
nextBatch: batchIndex + 1,
|
||||
}),
|
||||
{
|
||||
status: 202,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
// All batches done - run final pass
|
||||
const result = await assembleFinalDocument(collection, username, mcpUrls);
|
||||
|
||||
// Mark job as completed
|
||||
await prisma.skillsGenerationJob.update({
|
||||
where: { id: job.id },
|
||||
data: { status: 'completed' },
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single batch of tools
|
||||
*/
|
||||
async function processBatch(
|
||||
jobId: string,
|
||||
batchIndex: number,
|
||||
collection: CollectionWithTools,
|
||||
toolsNeedingGeneration: string[]
|
||||
): Promise<void> {
|
||||
const startIdx = batchIndex * BATCH_SIZE;
|
||||
const endIdx = Math.min(startIdx + BATCH_SIZE, toolsNeedingGeneration.length);
|
||||
const batchToolIds = toolsNeedingGeneration.slice(startIdx, endIdx);
|
||||
|
||||
if (batchToolIds.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Get tool data for this batch
|
||||
const batchTools = collection.tools
|
||||
.filter((ct) => batchToolIds.includes(ct.tool.id))
|
||||
.map((ct) => ({
|
||||
id: ct.tool.id,
|
||||
name: ct.tool.name,
|
||||
description: ct.tool.description,
|
||||
packageName: ct.tool.package.npmPackageName,
|
||||
packageVersion: ct.tool.package.npmVersion,
|
||||
inputSchema: ct.tool.inputSchema,
|
||||
}));
|
||||
|
||||
// Get unique packages for this batch
|
||||
const uniquePackages = [...new Set(batchTools.map((t) => t.packageName))];
|
||||
|
||||
// Fetch package sources for this batch (limit to 5 per batch)
|
||||
const packageSourceResults = await Promise.allSettled(
|
||||
uniquePackages
|
||||
.slice(0, 5)
|
||||
.map((name) =>
|
||||
fetchPackageSource(name, batchTools.find((t) => t.packageName === name)?.packageVersion)
|
||||
)
|
||||
);
|
||||
|
||||
const packageSourcesMap = new Map<string, PackageSource>();
|
||||
for (const result of packageSourceResults) {
|
||||
if (result.status === 'fulfilled') {
|
||||
packageSourcesMap.set(result.value.packageName, result.value);
|
||||
}
|
||||
}
|
||||
|
||||
// Generate skills for batch
|
||||
const results = await generateToolSkillsBatch(batchTools, packageSourcesMap);
|
||||
|
||||
// Save to per-tool cache
|
||||
const cacheUpserts = Array.from(results.entries()).map(([toolId, markdown]) =>
|
||||
prisma.toolSkillsCache.upsert({
|
||||
where: { toolId },
|
||||
create: {
|
||||
toolId,
|
||||
skillsMarkdown: markdown,
|
||||
},
|
||||
update: {
|
||||
skillsMarkdown: markdown,
|
||||
generatedAt: new Date(),
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
await Promise.all(cacheUpserts);
|
||||
|
||||
// Update job progress
|
||||
const completedToolIds = [...batchToolIds];
|
||||
|
||||
await prisma.skillsGenerationJob.update({
|
||||
where: { id: jobId },
|
||||
data: {
|
||||
currentBatch: batchIndex + 1,
|
||||
completedToolIds: { push: completedToolIds },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Assemble the final document from cached tool sections
|
||||
*/
|
||||
async function assembleFinalDocument(
|
||||
collection: CollectionWithTools,
|
||||
username: string,
|
||||
mcpUrls: { http: string; sse: string }
|
||||
): Promise<Response> {
|
||||
const toolIds = collection.tools.map((ct) => ct.tool.id);
|
||||
|
||||
// Fetch all cached tool sections
|
||||
const cachedTools = await prisma.toolSkillsCache.findMany({
|
||||
where: { toolId: { in: toolIds } },
|
||||
});
|
||||
|
||||
// Create a map for ordering
|
||||
const cacheMap = new Map(cachedTools.map((c) => [c.toolId, c.skillsMarkdown]));
|
||||
|
||||
// Order sections by collection tool order
|
||||
const toolSections = collection.tools
|
||||
.map((ct) => cacheMap.get(ct.tool.id))
|
||||
.filter((section): section is string => !!section);
|
||||
|
||||
// Get package names for summary
|
||||
const packageNames = collection.tools.map((ct) => ct.tool.package.npmPackageName);
|
||||
|
||||
// Generate summary (second AI pass)
|
||||
const summary = await generateSkillsSummary(
|
||||
{
|
||||
id: collection.id,
|
||||
name: collection.name,
|
||||
slug: collection.slug || '',
|
||||
description: collection.description,
|
||||
username,
|
||||
},
|
||||
toolSections,
|
||||
mcpUrls,
|
||||
packageNames
|
||||
);
|
||||
|
||||
// Assemble final document
|
||||
const skillsMarkdown = assembleSkillsDocument(
|
||||
{
|
||||
id: collection.id,
|
||||
name: collection.name,
|
||||
slug: collection.slug || '',
|
||||
description: collection.description,
|
||||
username,
|
||||
},
|
||||
toolSections,
|
||||
summary,
|
||||
mcpUrls,
|
||||
packageNames
|
||||
);
|
||||
|
||||
// Save to database for caching
|
||||
await prisma.collection.update({
|
||||
where: { id: collection.id },
|
||||
data: {
|
||||
skillsMarkdown,
|
||||
skillsGeneratedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
return new Response(skillsMarkdown, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/markdown; charset=utf-8',
|
||||
'Cache-Control': `public, s-maxage=${CACHE_DURATION_SECONDS}, stale-while-revalidate=86400`,
|
||||
'X-Cache': 'MISS',
|
||||
'X-Generation-Mode': 'chunked',
|
||||
'X-Tool-Count': toolSections.length.toString(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,11 @@
|
|||
/**
|
||||
* AI-powered skills documentation generator for tool collections
|
||||
* Uses Vercel AI SDK to analyze package source code and generate comprehensive skills.md
|
||||
*
|
||||
* This module provides the original monolithic generation function for backward compatibility.
|
||||
* For chunked generation of large collections, use the new modules:
|
||||
* - tool-skills-generator.ts: Per-tool skills generation
|
||||
* - skills-summary-generator.ts: Summary/intro generation
|
||||
*/
|
||||
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
|
|
|
|||
271
apps/web/src/lib/ai/skills-summary-generator.ts
Normal file
271
apps/web/src/lib/ai/skills-summary-generator.ts
Normal file
|
|
@ -0,0 +1,271 @@
|
|||
/**
|
||||
* Skills Summary Generator (Second Pass)
|
||||
* Analyzes all tool sections and generates cohesive intro, workflows, and summary
|
||||
*/
|
||||
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import { generateText } from 'ai';
|
||||
|
||||
export interface CollectionData {
|
||||
id: string;
|
||||
name: string;
|
||||
slug: string;
|
||||
description: string | null;
|
||||
username: string;
|
||||
}
|
||||
|
||||
export interface McpUrls {
|
||||
http: string;
|
||||
sse: string;
|
||||
}
|
||||
|
||||
export interface SkillsSummary {
|
||||
intro: string;
|
||||
workflows: string;
|
||||
summary: string;
|
||||
}
|
||||
|
||||
const SUMMARY_SYSTEM_PROMPT = `You are an expert technical writer creating cohesive documentation from individual tool sections.
|
||||
|
||||
Your task is to analyze all tool documentation sections and generate:
|
||||
1. An introduction that describes the collection's overall capabilities
|
||||
2. Multi-tool workflow examples showing how tools work together
|
||||
3. A summary with constraints and safety considerations
|
||||
|
||||
Guidelines:
|
||||
- Be concise and focused on practical usage
|
||||
- Identify synergies between tools
|
||||
- Generate realistic workflow examples
|
||||
- Document limitations based on tool analysis
|
||||
|
||||
Output clean, well-organized Markdown sections.`;
|
||||
|
||||
function buildSummaryPrompt(
|
||||
collection: CollectionData,
|
||||
toolSections: string[],
|
||||
mcpUrls: McpUrls,
|
||||
packageNames: string[]
|
||||
): string {
|
||||
const baseUrl = mcpUrls.http.replace(/\/api\/mcp\/.*$/, '');
|
||||
const collectionUrl = `${baseUrl}/${collection.username}/collections/${collection.slug}`;
|
||||
|
||||
return `Analyze these tool documentation sections and generate summary content for the skills.md document.
|
||||
|
||||
## Collection Info
|
||||
- **Name:** ${collection.name}
|
||||
- **Owner:** @${collection.username}
|
||||
- **Slug:** ${collection.slug}
|
||||
- **Description:** ${collection.description || 'No description'}
|
||||
- **Collection URL:** ${collectionUrl}
|
||||
- **Tool Count:** ${toolSections.length} tools from ${new Set(packageNames).size} packages
|
||||
|
||||
## MCP Endpoints
|
||||
- **HTTP:** ${mcpUrls.http}
|
||||
- **SSE:** ${mcpUrls.sse}
|
||||
|
||||
## Individual Tool Sections
|
||||
${toolSections.join('\n\n---\n\n')}
|
||||
|
||||
---
|
||||
|
||||
Generate three sections. Output each section with its exact header:
|
||||
|
||||
## INTRO_START
|
||||
[Generate an introduction paragraph describing the collection's overall capabilities and primary use cases. Keep it to 2-3 paragraphs max.]
|
||||
## INTRO_END
|
||||
|
||||
## WORKFLOWS_START
|
||||
[Generate 2-3 multi-tool workflow examples showing how tools can work together. Include TypeScript/bash code examples for each workflow.]
|
||||
## WORKFLOWS_END
|
||||
|
||||
## SUMMARY_START
|
||||
[Generate a summary section with:
|
||||
- What the collection does NOT support (limitations)
|
||||
- Rate limits info
|
||||
- Authentication requirements
|
||||
- Versioning info with current timestamp]
|
||||
## SUMMARY_END`;
|
||||
}
|
||||
|
||||
function parseSummaryResponse(text: string): SkillsSummary {
|
||||
// Extract sections using markers
|
||||
const introMatch = text.match(/## INTRO_START\n([\s\S]*?)## INTRO_END/);
|
||||
const workflowsMatch = text.match(/## WORKFLOWS_START\n([\s\S]*?)## WORKFLOWS_END/);
|
||||
const summaryMatch = text.match(/## SUMMARY_START\n([\s\S]*?)## SUMMARY_END/);
|
||||
|
||||
return {
|
||||
intro: introMatch?.[1]?.trim() || 'Skills documentation for this collection.',
|
||||
workflows: workflowsMatch?.[1]?.trim() || '',
|
||||
summary: summaryMatch?.[1]?.trim() || '',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate skills summary by analyzing all tool sections
|
||||
*/
|
||||
export async function generateSkillsSummary(
|
||||
collection: CollectionData,
|
||||
toolSections: string[],
|
||||
mcpUrls: McpUrls,
|
||||
packageNames: string[]
|
||||
): Promise<SkillsSummary> {
|
||||
if (toolSections.length === 0) {
|
||||
return {
|
||||
intro: 'This collection has no tools.',
|
||||
workflows: '',
|
||||
summary: '',
|
||||
};
|
||||
}
|
||||
|
||||
const prompt = buildSummaryPrompt(collection, toolSections, mcpUrls, packageNames);
|
||||
|
||||
const { text } = await generateText({
|
||||
model: openai('gpt-4.1-mini'),
|
||||
system: SUMMARY_SYSTEM_PROMPT,
|
||||
prompt,
|
||||
temperature: 0.3,
|
||||
});
|
||||
|
||||
return parseSummaryResponse(text);
|
||||
}
|
||||
|
||||
/**
|
||||
* Assemble the final skills.md document from individual sections
|
||||
*/
|
||||
export function assembleSkillsDocument(
|
||||
collection: CollectionData,
|
||||
toolSections: string[],
|
||||
summary: SkillsSummary,
|
||||
mcpUrls: McpUrls,
|
||||
packageNames: string[]
|
||||
): string {
|
||||
const baseUrl = mcpUrls.http.replace(/\/api\/mcp\/.*$/, '');
|
||||
const collectionUrl = `${baseUrl}/${collection.username}/collections/${collection.slug}`;
|
||||
const uniquePackages = new Set(packageNames).size;
|
||||
const timestamp = new Date().toISOString();
|
||||
|
||||
return `# Agent Skills Declaration: ${collection.name}
|
||||
|
||||
> Machine-consumable capability contract for AI agents (Claude Code, MCP clients, tool routers)
|
||||
|
||||
## 1. Agent Identity
|
||||
|
||||
**Collection:** ${collection.name}
|
||||
**Owner:** @${collection.username}
|
||||
**Description:** ${collection.description || 'No description'}
|
||||
**Tool Count:** ${toolSections.length} tools from ${uniquePackages} packages
|
||||
|
||||
${summary.intro}
|
||||
|
||||
---
|
||||
|
||||
## 2. Core Skills
|
||||
|
||||
${toolSections.join('\n\n---\n\n')}
|
||||
|
||||
---
|
||||
|
||||
## 3. Multi-Tool Workflows
|
||||
|
||||
${summary.workflows || '*No workflow examples generated.*'}
|
||||
|
||||
---
|
||||
|
||||
## 4. Agent Integration Methods
|
||||
|
||||
### 4.1 CLI (\`tpm run\`)
|
||||
|
||||
Install CLI:
|
||||
\`\`\`bash
|
||||
npm install -g @tpmjs/cli
|
||||
\`\`\`
|
||||
|
||||
Run a tool:
|
||||
\`\`\`bash
|
||||
tpm run --collection ${collection.username}/${collection.slug} --tool [tool-name] --args '{"key": "value"}'
|
||||
\`\`\`
|
||||
|
||||
### 4.2 REST API (JSON-RPC 2.0)
|
||||
|
||||
**Endpoint:** \`POST ${mcpUrls.http}\`
|
||||
|
||||
\`\`\`bash
|
||||
curl -X POST ${mcpUrls.http} \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-H "Authorization: Bearer YOUR_API_KEY" \\
|
||||
-d '{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "tools/call",
|
||||
"params": {
|
||||
"name": "[tool-name]",
|
||||
"arguments": { ... }
|
||||
}
|
||||
}'
|
||||
\`\`\`
|
||||
|
||||
**List available tools:**
|
||||
\`\`\`bash
|
||||
curl -X POST ${mcpUrls.http} \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-H "Authorization: Bearer YOUR_API_KEY" \\
|
||||
-d '{"jsonrpc": "2.0", "id": 1, "method": "tools/list"}'
|
||||
\`\`\`
|
||||
|
||||
### 4.3 MCP Server
|
||||
|
||||
**URLs:**
|
||||
- HTTP: \`${mcpUrls.http}\`
|
||||
- SSE: \`${mcpUrls.sse}\`
|
||||
|
||||
**Claude Desktop Config (\`claude_desktop_config.json\`):**
|
||||
\`\`\`json
|
||||
{
|
||||
"mcpServers": {
|
||||
"${collection.slug}": {
|
||||
"url": "${mcpUrls.sse}",
|
||||
"headers": {
|
||||
"Authorization": "Bearer YOUR_API_KEY"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
**Local MCP Server (via CLI):**
|
||||
\`\`\`bash
|
||||
tpm mcp serve ${collection.username}/${collection.slug}
|
||||
\`\`\`
|
||||
|
||||
---
|
||||
|
||||
## 5. Constraints & Safety
|
||||
|
||||
${
|
||||
summary.summary ||
|
||||
`**Rate Limits:**
|
||||
- Free tier: 100 requests/hour
|
||||
- Authenticated: 1000 requests/hour
|
||||
|
||||
**Authentication:**
|
||||
- Public collections: No auth required for read
|
||||
- Tool execution: API key required`
|
||||
}
|
||||
|
||||
---
|
||||
|
||||
## 6. Versioning
|
||||
|
||||
**Skills Version:** 1.0.0
|
||||
**Generated:** ${timestamp}
|
||||
**Collection Page:** ${collectionUrl}
|
||||
|
||||
---
|
||||
|
||||
## 7. Canonical References
|
||||
|
||||
- TPMJS: https://tpmjs.com
|
||||
- MCP Protocol: https://modelcontextprotocol.io
|
||||
- Collection Page: ${collectionUrl}
|
||||
`;
|
||||
}
|
||||
140
apps/web/src/lib/ai/tool-skills-generator.ts
Normal file
140
apps/web/src/lib/ai/tool-skills-generator.ts
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
/**
|
||||
* Per-tool skills documentation generator
|
||||
* Generates markdown section for a single tool, designed for batched processing
|
||||
*/
|
||||
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import { generateText } from 'ai';
|
||||
|
||||
import type { PackageSource } from './package-source-fetcher';
|
||||
|
||||
export interface ToolData {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
packageName: string;
|
||||
packageVersion: string;
|
||||
inputSchema: unknown | null;
|
||||
}
|
||||
|
||||
const TOOL_SKILLS_SYSTEM_PROMPT = `You are an expert technical writer generating skills documentation for a single tool.
|
||||
|
||||
Your task is to analyze the source code and generate a comprehensive skills section for this tool that can be used by AI agents.
|
||||
|
||||
Guidelines:
|
||||
1. **Accuracy First**: Only document capabilities you can verify from the source code
|
||||
2. **Real Examples**: Generate code examples based on actual function signatures
|
||||
3. **Input/Output Schemas**: Extract exact TypeScript types when available
|
||||
4. **Source Analysis**: Provide insights about implementation details
|
||||
5. **Concise**: Focus on what's useful for tool invocation
|
||||
|
||||
Output clean, well-organized Markdown.`;
|
||||
|
||||
function buildToolPrompt(tool: ToolData, packageSource: PackageSource | null): string {
|
||||
const sourceContext = packageSource
|
||||
? packageSource.files
|
||||
.map((f) => `### ${f.path}\n\`\`\`typescript\n${f.content.slice(0, 4000)}\n\`\`\``)
|
||||
.join('\n\n')
|
||||
: 'Source code not available';
|
||||
|
||||
return `Generate a skills section for this tool.
|
||||
|
||||
## Tool Info
|
||||
- **Name:** ${tool.name}
|
||||
- **Package:** ${tool.packageName}@${tool.packageVersion}
|
||||
- **Description:** ${tool.description}
|
||||
|
||||
## Input Schema
|
||||
${tool.inputSchema ? JSON.stringify(tool.inputSchema, null, 2) : 'Not available'}
|
||||
|
||||
## Package Source Code
|
||||
${sourceContext}
|
||||
|
||||
---
|
||||
|
||||
Generate a markdown section following this structure:
|
||||
|
||||
### Skill: ${tool.name}
|
||||
**Package:** \`${tool.packageName}\`
|
||||
**Description:** ${tool.description}
|
||||
|
||||
**Input Schema:**
|
||||
\`\`\`typescript
|
||||
[Extract from source code or input schema]
|
||||
\`\`\`
|
||||
|
||||
**Output Format:** [Analyze from source]
|
||||
|
||||
**Source Analysis:**
|
||||
[AI-generated insights from reading actual code - what does this tool actually do internally?]
|
||||
|
||||
**Example Usage:**
|
||||
\`\`\`typescript
|
||||
[Real example based on source code patterns]
|
||||
\`\`\`
|
||||
|
||||
**Constraints:**
|
||||
- [What this tool can and cannot do]
|
||||
`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate skills markdown section for a single tool
|
||||
*/
|
||||
export async function generateToolSkills(
|
||||
tool: ToolData,
|
||||
packageSource: PackageSource | null
|
||||
): Promise<string> {
|
||||
const prompt = buildToolPrompt(tool, packageSource);
|
||||
|
||||
const { text } = await generateText({
|
||||
model: openai('gpt-4.1-mini'),
|
||||
system: TOOL_SKILLS_SYSTEM_PROMPT,
|
||||
prompt,
|
||||
temperature: 0.3,
|
||||
});
|
||||
|
||||
return text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate skills markdown for a batch of tools in parallel
|
||||
* Returns a map of toolId -> markdown
|
||||
*/
|
||||
export async function generateToolSkillsBatch(
|
||||
tools: ToolData[],
|
||||
packageSources: Map<string, PackageSource>
|
||||
): Promise<Map<string, string>> {
|
||||
const results = new Map<string, string>();
|
||||
|
||||
// Process tools in parallel (10 at a time is reasonable for API limits)
|
||||
const promises = tools.map(async (tool) => {
|
||||
try {
|
||||
const packageSource = packageSources.get(tool.packageName) || null;
|
||||
const markdown = await generateToolSkills(tool, packageSource);
|
||||
return { toolId: tool.id, markdown };
|
||||
} catch (error) {
|
||||
console.error(`[ToolSkillsGenerator] Failed to generate skills for ${tool.name}:`, error);
|
||||
// Return a fallback section on error
|
||||
return {
|
||||
toolId: tool.id,
|
||||
markdown: `### Skill: ${tool.name}
|
||||
**Package:** \`${tool.packageName}\`
|
||||
**Description:** ${tool.description}
|
||||
|
||||
*Skills documentation generation failed. Please retry.*
|
||||
`,
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
const settledResults = await Promise.allSettled(promises);
|
||||
|
||||
for (const result of settledResults) {
|
||||
if (result.status === 'fulfilled') {
|
||||
results.set(result.value.toolId, result.value.markdown);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue