feat(skills): add RealSkills living endpoint for agent Q&A
Implements a skills endpoint that evolves through agent conversations: - GET /:username/collections/:slug/skills - Returns markdown skill summary - POST /:username/collections/:slug/skills - Ask questions, get RAG+LLM responses Features: - OpenAI text-embedding-3-large (3072 dims) for semantic similarity - GPT-4.1-mini for response generation with RAG context - Lazy seeding of synthetic questions on first access - Cache hits for >95% similar questions - Real-time skill graph updates (emergent skill taxonomy) - Session support for multi-turn conversations - Activity feed and stats APIs for UI Database models: SkillQuestion, Skill, SkillSession, SkillQuestionSkill, SkillQuestionTool
This commit is contained in:
Ajax Davis
parent
0489646e55
commit
1118463e6b
13 changed files with 3001 additions and 8 deletions
|
|
@ -12,6 +12,7 @@ import { ForkedFromBadge } from '~/components/ForkedFromBadge';
|
|||
import { LikeButton } from '~/components/LikeButton';
|
||||
import { ScenariosSection } from '~/components/ScenariosSection';
|
||||
import { ShareButton } from '~/components/ShareButton';
|
||||
import { SkillsSection } from '~/components/skills/SkillsSection';
|
||||
import { UseCasesSection } from '~/components/UseCasesSection';
|
||||
import { useSession } from '~/lib/auth-client';
|
||||
|
||||
|
|
@ -412,6 +413,15 @@ export function CollectionDetailClient({
|
|||
slug={collection.slug}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Skills Section */}
|
||||
{collection.tools.length > 0 && (
|
||||
<SkillsSection
|
||||
collectionId={collection.id}
|
||||
username={username}
|
||||
slug={collection.slug}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,537 @@
|
|||
/**
|
||||
* RealSkills API Endpoint
|
||||
*
|
||||
* A living skills endpoint that evolves through agent conversations.
|
||||
* Skills emerge organically from question patterns.
|
||||
*
|
||||
* GET - Return skill summary markdown (triggers lazy seeding)
|
||||
* POST - Ask a question (RAG + LLM response)
|
||||
*/
|
||||
|
||||
import { createHash } from 'crypto';
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { NextResponse, type NextRequest } from 'next/server';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { checkQuestionSimilarity } from '~/lib/ai/skills-embedding';
|
||||
import { updateSkillGraph, getCollectionSkillsSummary } from '~/lib/ai/skills-graph-updater';
|
||||
import {
|
||||
generateSkillResponse,
|
||||
generateFollowupSuggestions,
|
||||
calculateConfidence,
|
||||
type CollectionContext,
|
||||
} from '~/lib/ai/skills-response-generator';
|
||||
import {
|
||||
seedCollectionSkills,
|
||||
getSeedingStatus,
|
||||
type CollectionWithTools,
|
||||
} from '~/lib/ai/skills-seeder';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const maxDuration = 300;
|
||||
|
||||
type RouteContext = {
|
||||
params: Promise<{ username: string; slug: string }>;
|
||||
};
|
||||
|
||||
// Request validation schemas
|
||||
const PostRequestSchema = z.object({
|
||||
question: z.string().min(5).max(2000),
|
||||
sessionId: z.string().optional(),
|
||||
agentName: z.string().max(100).optional(),
|
||||
context: z.string().max(2000).optional(),
|
||||
tags: z.array(z.string().max(50)).max(10).optional(),
|
||||
});
|
||||
|
||||
// Session expiry time (24 hours)
|
||||
const SESSION_EXPIRY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Hash agent identity for anonymization
|
||||
*/
|
||||
function hashAgentIdentity(ip: string, userAgent: string): string {
|
||||
return createHash('sha256')
|
||||
.update(`${ip}:${userAgent}`)
|
||||
.digest('hex')
|
||||
.slice(0, 16);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load collection with tools
|
||||
*/
|
||||
async function loadCollection(
|
||||
username: string,
|
||||
slug: string
|
||||
): Promise<CollectionWithTools | NextResponse> {
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { username },
|
||||
select: { id: true, username: true },
|
||||
});
|
||||
|
||||
if (!user || !user.username) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: 'User not found' },
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
const collection = await prisma.collection.findFirst({
|
||||
where: { slug, userId: user.id },
|
||||
include: {
|
||||
tools: {
|
||||
include: {
|
||||
tool: {
|
||||
include: {
|
||||
package: {
|
||||
select: { npmPackageName: true, npmVersion: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
orderBy: { position: 'asc' },
|
||||
take: 100,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!collection) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: 'Collection not found' },
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
if (!collection.isPublic) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: 'Collection is not public' },
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
|
||||
// Flatten the tools structure
|
||||
const collectionWithTools = {
|
||||
...collection,
|
||||
tools: collection.tools.map((ct) => ({
|
||||
...ct.tool,
|
||||
package: ct.tool.package,
|
||||
})),
|
||||
};
|
||||
|
||||
return collectionWithTools as CollectionWithTools;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure collection is seeded (lazy seeding)
|
||||
*/
|
||||
async function ensureSeeded(collection: CollectionWithTools): Promise<{
|
||||
isSeeding: boolean;
|
||||
wasSeeded: boolean;
|
||||
}> {
|
||||
const status = await getSeedingStatus(collection.id);
|
||||
|
||||
if (status.isSeeded) {
|
||||
return { isSeeding: false, wasSeeded: false };
|
||||
}
|
||||
|
||||
if (status.isSeeding) {
|
||||
return { isSeeding: true, wasSeeded: false };
|
||||
}
|
||||
|
||||
// Trigger seeding (non-blocking for GET, blocking for POST)
|
||||
try {
|
||||
const result = await seedCollectionSkills(collection);
|
||||
return { isSeeding: false, wasSeeded: result.seeded };
|
||||
} catch (error) {
|
||||
console.error('[Skills] Seeding failed:', error);
|
||||
// Continue without seeding - endpoint still works
|
||||
return { isSeeding: false, wasSeeded: false };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /:username/collections/:slug/skills
|
||||
*
|
||||
* Returns skill summary as markdown.
|
||||
* Triggers lazy seeding on first access.
|
||||
*/
|
||||
export async function GET(_request: NextRequest, context: RouteContext) {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const { username: rawUsername, slug } = await context.params;
|
||||
const username = rawUsername.startsWith('@')
|
||||
? rawUsername.slice(1)
|
||||
: rawUsername;
|
||||
|
||||
// Load collection
|
||||
const result = await loadCollection(username, slug);
|
||||
if (result instanceof NextResponse) return result;
|
||||
const collection = result;
|
||||
|
||||
// Check/trigger seeding
|
||||
const seedStatus = await ensureSeeded(collection);
|
||||
|
||||
if (seedStatus.isSeeding) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: true,
|
||||
data: {
|
||||
status: 'seeding',
|
||||
message: 'Skills are being generated. Please retry in a few seconds.',
|
||||
},
|
||||
},
|
||||
{
|
||||
status: 202,
|
||||
headers: { 'Retry-After': '10' },
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
// Get skill summary
|
||||
const summary = await getCollectionSkillsSummary(collection.id);
|
||||
|
||||
// Build markdown response
|
||||
const markdown = buildSkillsSummaryMarkdown(collection, summary, username);
|
||||
|
||||
return new Response(markdown, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/markdown; charset=utf-8',
|
||||
'X-Skills-Total-Questions': summary.totalQuestions.toString(),
|
||||
'X-Skills-Total-Skills': summary.totalSkills.toString(),
|
||||
'X-Processing-Time-Ms': (Date.now() - startTime).toString(),
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[Skills GET Error]:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Internal server error',
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /:username/collections/:slug/skills
|
||||
*
|
||||
* Submit a question and get a skill-based response.
|
||||
*/
|
||||
export async function POST(request: NextRequest, context: RouteContext) {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const { username: rawUsername, slug } = await context.params;
|
||||
const username = rawUsername.startsWith('@')
|
||||
? rawUsername.slice(1)
|
||||
: rawUsername;
|
||||
|
||||
// Parse and validate request body
|
||||
let body: unknown;
|
||||
try {
|
||||
body = await request.json();
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: 'Invalid JSON body' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const parseResult = PostRequestSchema.safeParse(body);
|
||||
if (!parseResult.success) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: 'Validation error',
|
||||
details: parseResult.error.format(),
|
||||
},
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const { question, sessionId, agentName, context: questionContext, tags } =
|
||||
parseResult.data;
|
||||
|
||||
// Load collection
|
||||
const result = await loadCollection(username, slug);
|
||||
if (result instanceof NextResponse) return result;
|
||||
const collection = result;
|
||||
|
||||
// Ensure seeded
|
||||
await ensureSeeded(collection);
|
||||
|
||||
// Get agent identity
|
||||
const ip = request.headers.get('x-forwarded-for') || 'unknown';
|
||||
const userAgent = request.headers.get('user-agent') || 'unknown';
|
||||
const agentHash = hashAgentIdentity(ip, userAgent);
|
||||
|
||||
// Check for similarity / cache hit
|
||||
const similarityResult = await checkQuestionSimilarity(
|
||||
question,
|
||||
collection.id
|
||||
);
|
||||
|
||||
// If very similar question exists (>95%), return cached answer
|
||||
if (similarityResult.isCacheHit && similarityResult.cachedAnswer) {
|
||||
const cachedQuestion = similarityResult.similarQuestions[0];
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
answer: similarityResult.cachedAnswer,
|
||||
confidence: cachedQuestion?.similarity || 0.95,
|
||||
basedOn: 1,
|
||||
skillsIdentified: [],
|
||||
cached: true,
|
||||
},
|
||||
meta: {
|
||||
cached: true,
|
||||
questionId: cachedQuestion?.id || null,
|
||||
processingMs: Date.now() - startTime,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Build collection context
|
||||
const collectionContext: CollectionContext = {
|
||||
collection,
|
||||
tools: collection.tools,
|
||||
skillsMarkdown: collection.skillsMarkdown,
|
||||
};
|
||||
|
||||
// Get session history if session exists
|
||||
let sessionHistory: Array<{ role: 'user' | 'assistant'; content: string }> =
|
||||
[];
|
||||
let activeSessionId = sessionId;
|
||||
|
||||
if (sessionId) {
|
||||
const session = await prisma.skillSession.findUnique({
|
||||
where: { id: sessionId },
|
||||
});
|
||||
if (session && session.collectionId === collection.id) {
|
||||
sessionHistory = session.context as Array<{
|
||||
role: 'user' | 'assistant';
|
||||
content: string;
|
||||
}>;
|
||||
}
|
||||
}
|
||||
|
||||
// Generate response
|
||||
const fullQuestion = questionContext
|
||||
? `${question}\n\nContext: ${questionContext}`
|
||||
: question;
|
||||
|
||||
const { answer, tokensUsed } = await generateSkillResponse({
|
||||
question: fullQuestion,
|
||||
collectionContext,
|
||||
similarQuestions: similarityResult.similarQuestions,
|
||||
sessionHistory,
|
||||
tags,
|
||||
});
|
||||
|
||||
// Calculate confidence
|
||||
const confidence = calculateConfidence(
|
||||
similarityResult.similarQuestions,
|
||||
!!collection.skillsMarkdown
|
||||
);
|
||||
|
||||
// Store the question
|
||||
const storedQuestion = await prisma.skillQuestion.create({
|
||||
data: {
|
||||
collectionId: collection.id,
|
||||
question,
|
||||
embedding: similarityResult.embedding as unknown as object,
|
||||
answer,
|
||||
answerTokens: tokensUsed,
|
||||
agentHash,
|
||||
agentName: agentName || null,
|
||||
sessionId: activeSessionId,
|
||||
confidence,
|
||||
tags: tags || [],
|
||||
},
|
||||
});
|
||||
|
||||
// Update skill graph (best-effort, don't fail request)
|
||||
let skillLinks: Array<{ skillId: string; skillName: string }> = [];
|
||||
try {
|
||||
const graphResult = await updateSkillGraph({
|
||||
questionId: storedQuestion.id,
|
||||
collectionId: collection.id,
|
||||
question,
|
||||
answer,
|
||||
tools: collection.tools,
|
||||
});
|
||||
skillLinks = graphResult.skillLinks;
|
||||
} catch (error) {
|
||||
console.error('[Skills] Graph update failed:', error);
|
||||
}
|
||||
|
||||
// Update or create session
|
||||
if (sessionId || sessionHistory.length > 0) {
|
||||
const newHistory = [
|
||||
...sessionHistory,
|
||||
{ role: 'user' as const, content: question },
|
||||
{ role: 'assistant' as const, content: answer },
|
||||
].slice(-20); // Keep last 20 messages
|
||||
|
||||
if (sessionId) {
|
||||
await prisma.skillSession.update({
|
||||
where: { id: sessionId },
|
||||
data: {
|
||||
context: newHistory,
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
});
|
||||
} else {
|
||||
const newSession = await prisma.skillSession.create({
|
||||
data: {
|
||||
collectionId: collection.id,
|
||||
context: newHistory,
|
||||
agentHash,
|
||||
agentName,
|
||||
expiresAt: new Date(Date.now() + SESSION_EXPIRY_MS),
|
||||
},
|
||||
});
|
||||
activeSessionId = newSession.id;
|
||||
}
|
||||
}
|
||||
|
||||
// Generate follow-up suggestions (optional, don't block)
|
||||
let suggestedFollowups: string[] = [];
|
||||
try {
|
||||
suggestedFollowups = await generateFollowupSuggestions(
|
||||
question,
|
||||
answer,
|
||||
collection.name
|
||||
);
|
||||
} catch {
|
||||
// Ignore errors for followups
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
answer,
|
||||
confidence,
|
||||
basedOn: similarityResult.similarQuestions.length,
|
||||
skillsIdentified: skillLinks.map((s) => s.skillName),
|
||||
sessionId: activeSessionId,
|
||||
suggestedFollowups,
|
||||
},
|
||||
meta: {
|
||||
cached: false,
|
||||
questionId: storedQuestion.id,
|
||||
processingMs: Date.now() - startTime,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[Skills POST Error]:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Internal server error',
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build markdown summary of skills
|
||||
*/
|
||||
function buildSkillsSummaryMarkdown(
|
||||
collection: CollectionWithTools,
|
||||
summary: {
|
||||
totalQuestions: number;
|
||||
totalSkills: number;
|
||||
topSkills: Array<{
|
||||
name: string;
|
||||
questionCount: number;
|
||||
confidence: number;
|
||||
}>;
|
||||
},
|
||||
username: string
|
||||
): string {
|
||||
const baseUrl = process.env.NEXT_PUBLIC_APP_URL || 'https://tpmjs.com';
|
||||
const skillsUrl = `${baseUrl}/${username}/collections/${collection.slug}/skills`;
|
||||
|
||||
let markdown = `# Skills: ${collection.name}
|
||||
|
||||
> Skills, proven in the wild — not declared on paper.
|
||||
|
||||
This collection has evolved through **${summary.totalQuestions} questions** from agents, identifying **${summary.totalSkills} distinct skills**.
|
||||
|
||||
## API Usage
|
||||
|
||||
\`\`\`bash
|
||||
# Ask a question
|
||||
curl -X POST ${skillsUrl} \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-d '{"question": "How do I handle errors with these tools?"}'
|
||||
\`\`\`
|
||||
|
||||
## Top Skills
|
||||
|
||||
`;
|
||||
|
||||
if (summary.topSkills.length > 0) {
|
||||
for (const skill of summary.topSkills) {
|
||||
const confidenceBar = '█'.repeat(Math.floor(skill.confidence * 10));
|
||||
const confidenceEmpty = '░'.repeat(10 - Math.floor(skill.confidence * 10));
|
||||
markdown += `- **${skill.name}** (${skill.questionCount} questions) ${confidenceBar}${confidenceEmpty}\n`;
|
||||
}
|
||||
} else {
|
||||
markdown += `*No skills identified yet. Ask questions to start building the skill graph.*\n`;
|
||||
}
|
||||
|
||||
markdown += `
|
||||
## How It Works
|
||||
|
||||
1. **You ask a question** via POST
|
||||
2. We find similar past questions (RAG)
|
||||
3. We generate a tailored response
|
||||
4. Your question helps evolve the skill graph
|
||||
5. Future questions get better answers
|
||||
|
||||
## Request Schema
|
||||
|
||||
\`\`\`typescript
|
||||
interface SkillsRequest {
|
||||
question: string; // Required (5-2000 chars)
|
||||
sessionId?: string; // For multi-turn conversations
|
||||
agentName?: string; // Self-reported agent identity
|
||||
context?: string; // Additional context (max 2000 chars)
|
||||
tags?: string[]; // Hint tags (max 10)
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
## Response Schema
|
||||
|
||||
\`\`\`typescript
|
||||
interface SkillsResponse {
|
||||
success: boolean;
|
||||
data: {
|
||||
answer: string; // Markdown response
|
||||
confidence: number; // 0-1 confidence score
|
||||
basedOn: number; // Similar questions used
|
||||
skillsIdentified: string[];
|
||||
sessionId?: string;
|
||||
suggestedFollowups?: string[];
|
||||
};
|
||||
meta: {
|
||||
cached: boolean;
|
||||
questionId: string;
|
||||
processingMs: number;
|
||||
};
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
---
|
||||
|
||||
*Last updated: ${new Date().toISOString()}*
|
||||
`;
|
||||
|
||||
return markdown;
|
||||
}
|
||||
81
apps/web/src/app/api/skills/activity/route.ts
Normal file
81
apps/web/src/app/api/skills/activity/route.ts
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
/**
|
||||
* GET /api/skills/activity
|
||||
*
|
||||
* Fetch recent skill questions for a collection (anonymized for activity feed)
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { NextResponse, type NextRequest } from 'next/server';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
const collectionId = searchParams.get('collectionId');
|
||||
const limitParam = searchParams.get('limit');
|
||||
const limit = Math.min(50, Math.max(1, parseInt(limitParam || '10', 10)));
|
||||
|
||||
if (!collectionId) {
|
||||
return NextResponse.json(
|
||||
{ error: 'collectionId is required' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
// Verify collection exists and is public
|
||||
const collection = await prisma.collection.findUnique({
|
||||
where: { id: collectionId },
|
||||
select: { id: true, isPublic: true },
|
||||
});
|
||||
|
||||
if (!collection) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Collection not found' },
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
if (!collection.isPublic) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Collection is not public' },
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
|
||||
// Fetch recent questions with skill links (anonymized - no agent info)
|
||||
const questions = await prisma.skillQuestion.findMany({
|
||||
where: { collectionId },
|
||||
orderBy: { createdAt: 'desc' },
|
||||
take: limit,
|
||||
select: {
|
||||
id: true,
|
||||
question: true,
|
||||
answer: true,
|
||||
confidence: true,
|
||||
similarCount: true,
|
||||
tags: true,
|
||||
createdAt: true,
|
||||
// Include skills but not agent info for privacy
|
||||
skillNodes: {
|
||||
select: {
|
||||
skill: {
|
||||
select: {
|
||||
name: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return NextResponse.json({ questions });
|
||||
} catch (error) {
|
||||
console.error('[Skills Activity Error]:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch activity' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
73
apps/web/src/app/api/skills/stats/route.ts
Normal file
73
apps/web/src/app/api/skills/stats/route.ts
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
/**
|
||||
* GET /api/skills/stats
|
||||
*
|
||||
* Get skill statistics for a collection
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { NextResponse, type NextRequest } from 'next/server';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
const collectionId = searchParams.get('collectionId');
|
||||
|
||||
if (!collectionId) {
|
||||
return NextResponse.json(
|
||||
{ error: 'collectionId is required' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
// Verify collection exists and is public
|
||||
const collection = await prisma.collection.findUnique({
|
||||
where: { id: collectionId },
|
||||
select: { id: true, isPublic: true },
|
||||
});
|
||||
|
||||
if (!collection) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Collection not found' },
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
if (!collection.isPublic) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Collection is not public' },
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
|
||||
// Fetch stats in parallel
|
||||
const [totalQuestions, totalSkills, topSkills] = await Promise.all([
|
||||
prisma.skillQuestion.count({ where: { collectionId } }),
|
||||
prisma.skill.count({ where: { collectionId } }),
|
||||
prisma.skill.findMany({
|
||||
where: { collectionId },
|
||||
orderBy: { questionCount: 'desc' },
|
||||
take: 10,
|
||||
select: {
|
||||
name: true,
|
||||
questionCount: true,
|
||||
confidence: true,
|
||||
},
|
||||
}),
|
||||
]);
|
||||
|
||||
return NextResponse.json({
|
||||
totalQuestions,
|
||||
totalSkills,
|
||||
topSkills,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[Skills Stats Error]:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch stats' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
521
apps/web/src/app/docs/skills/page.tsx
Normal file
521
apps/web/src/app/docs/skills/page.tsx
Normal file
|
|
@ -0,0 +1,521 @@
|
|||
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@tpmjs/ui/Card/Card';
|
||||
import { CodeBlock } from '@tpmjs/ui/CodeBlock/CodeBlock';
|
||||
import type { Metadata } from 'next';
|
||||
import Link from 'next/link';
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: 'RealSkills API | TPMJS Docs',
|
||||
description:
|
||||
'A living skills endpoint that evolves through agent conversations. Skills emerge organically from question patterns.',
|
||||
};
|
||||
|
||||
const exampleResponseJson = `{
|
||||
"success": true,
|
||||
"data": {
|
||||
"answer": "To handle errors with these tools...",
|
||||
"confidence": 0.85,
|
||||
"basedOn": 3,
|
||||
"skillsIdentified": ["error-handling", "try-catch-patterns"],
|
||||
"sessionId": "sess_abc123",
|
||||
"suggestedFollowups": [
|
||||
"What are the retry patterns?",
|
||||
"How do I log errors?"
|
||||
]
|
||||
},
|
||||
"meta": {
|
||||
"cached": false,
|
||||
"questionId": "clx123abc456",
|
||||
"processingMs": 1234
|
||||
}
|
||||
}`;
|
||||
|
||||
export default function SkillsPage(): React.ReactElement {
|
||||
return (
|
||||
<div className="space-y-8">
|
||||
<div>
|
||||
<h1 className="text-3xl font-bold mb-4">RealSkills API</h1>
|
||||
<p className="text-foreground-secondary text-lg">
|
||||
Skills, proven in the wild — not declared on paper.
|
||||
</p>
|
||||
<p className="text-foreground-secondary mt-2">
|
||||
A living skills endpoint that evolves through agent conversations. Unlike static
|
||||
documentation, skills emerge organically from question patterns and improve over time.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Philosophy */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Philosophy</CardTitle>
|
||||
<CardDescription>Why living skills beats static skills.md</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<p className="text-foreground-secondary">
|
||||
Traditional documentation is written once and becomes outdated. RealSkills takes a
|
||||
different approach:
|
||||
</p>
|
||||
<ul className="list-disc list-inside space-y-2 text-foreground-secondary">
|
||||
<li>
|
||||
<strong>Questions drive discovery</strong> — Every agent question reveals what users
|
||||
actually need
|
||||
</li>
|
||||
<li>
|
||||
<strong>Answers compound</strong> — Similar questions get better answers based on
|
||||
previous responses
|
||||
</li>
|
||||
<li>
|
||||
<strong>Skills emerge</strong> — Patterns in questions automatically create skill
|
||||
categories
|
||||
</li>
|
||||
<li>
|
||||
<strong>Quality improves</strong> — More questions = more context = better responses
|
||||
</li>
|
||||
</ul>
|
||||
<p className="text-foreground-secondary">
|
||||
Think of it as a knowledge base that learns from every interaction.
|
||||
</p>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* How It Works */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>How It Works</CardTitle>
|
||||
<CardDescription>The question → skill inference loop</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<div className="bg-surface p-4 rounded-lg border border-border font-mono text-sm">
|
||||
<pre className="whitespace-pre-wrap">{`Agent POSTs question
|
||||
↓
|
||||
┌─────────────────────────────┐
|
||||
│ /skills endpoint │
|
||||
│ - Embed question │
|
||||
│ - Check similarity cache │
|
||||
│ - RAG from stored Q&A │
|
||||
│ - Generate response (LLM) │
|
||||
│ - Store question + answer │
|
||||
│ - Update skill graph │
|
||||
└─────────────────────────────┘
|
||||
↓
|
||||
Return skill guidance (markdown)
|
||||
↓
|
||||
Skill graph evolves in real-time`}</pre>
|
||||
</div>
|
||||
|
||||
<ol className="list-decimal list-inside space-y-3 text-foreground-secondary mt-4">
|
||||
<li>
|
||||
<strong>Question Received</strong> — Agent submits a question via POST
|
||||
</li>
|
||||
<li>
|
||||
<strong>Embedding Generated</strong> — Question is converted to a 3072-dimensional
|
||||
vector
|
||||
</li>
|
||||
<li>
|
||||
<strong>Similarity Check</strong> — If >95% similar to existing question, return
|
||||
cached answer
|
||||
</li>
|
||||
<li>
|
||||
<strong>RAG Context</strong> — Find similar past questions/answers for context
|
||||
</li>
|
||||
<li>
|
||||
<strong>Response Generation</strong> — GPT-4.1-mini generates a tailored response
|
||||
</li>
|
||||
<li>
|
||||
<strong>Storage & Graph Update</strong> — Question stored, skills inferred and linked
|
||||
</li>
|
||||
</ol>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* API Reference */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>API Reference</CardTitle>
|
||||
<CardDescription>GET and POST endpoints</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-6">
|
||||
<div>
|
||||
<h4 className="font-semibold mb-2">GET /:username/collections/:slug/skills</h4>
|
||||
<p className="text-foreground-secondary text-sm mb-3">
|
||||
Returns the skill summary as markdown. Triggers lazy seeding on first access.
|
||||
</p>
|
||||
<CodeBlock
|
||||
code={`curl https://tpmjs.com/ajaxdavis/collections/my-tools/skills`}
|
||||
language="bash"
|
||||
showCopy={true}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<h4 className="font-semibold mb-2">POST /:username/collections/:slug/skills</h4>
|
||||
<p className="text-foreground-secondary text-sm mb-3">
|
||||
Submit a question and receive an AI-generated response based on the collection's
|
||||
tools and previous Q&A.
|
||||
</p>
|
||||
<CodeBlock
|
||||
code={`curl -X POST https://tpmjs.com/ajaxdavis/collections/my-tools/skills \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-d '{
|
||||
"question": "How do I handle errors with these tools?",
|
||||
"agentName": "my-agent",
|
||||
"tags": ["error-handling"]
|
||||
}'`}
|
||||
language="bash"
|
||||
showCopy={true}
|
||||
/>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Request Schema */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Request Schema</CardTitle>
|
||||
<CardDescription>POST request body format</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<CodeBlock
|
||||
code={`interface SkillsRequest {
|
||||
// Required: The question to ask (5-2000 characters)
|
||||
question: string;
|
||||
|
||||
// Optional: Session ID for multi-turn conversations
|
||||
sessionId?: string;
|
||||
|
||||
// Optional: Self-reported agent identity
|
||||
agentName?: string;
|
||||
|
||||
// Optional: Additional context (max 2000 chars)
|
||||
context?: string;
|
||||
|
||||
// Optional: Hint tags to guide response (max 10)
|
||||
tags?: string[];
|
||||
}`}
|
||||
language="typescript"
|
||||
showCopy={true}
|
||||
/>
|
||||
|
||||
<div className="bg-background-secondary p-4 rounded-lg mt-4">
|
||||
<h4 className="font-semibold text-sm mb-2">Multi-Turn Conversations</h4>
|
||||
<p className="text-foreground-secondary text-sm">
|
||||
To continue a conversation, include the <code>sessionId</code> from a previous
|
||||
response. Sessions maintain context for up to 24 hours and include the last 20
|
||||
messages.
|
||||
</p>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Response Schema */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Response Schema</CardTitle>
|
||||
<CardDescription>Successful response format</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<CodeBlock
|
||||
code={`interface SkillsResponse {
|
||||
success: boolean;
|
||||
data: {
|
||||
// Markdown-formatted response
|
||||
answer: string;
|
||||
|
||||
// Confidence score (0-1)
|
||||
confidence: number;
|
||||
|
||||
// Number of similar questions used for RAG
|
||||
basedOn: number;
|
||||
|
||||
// Skills this question relates to
|
||||
skillsIdentified: string[];
|
||||
|
||||
// Session ID for continuing conversation
|
||||
sessionId?: string;
|
||||
|
||||
// Suggested follow-up questions
|
||||
suggestedFollowups?: string[];
|
||||
};
|
||||
meta: {
|
||||
// Whether response was from cache
|
||||
cached: boolean;
|
||||
|
||||
// ID of stored question
|
||||
questionId: string;
|
||||
|
||||
// Processing time in milliseconds
|
||||
processingMs: number;
|
||||
};
|
||||
}`}
|
||||
language="typescript"
|
||||
showCopy={true}
|
||||
/>
|
||||
|
||||
<h4 className="font-semibold mt-4">Example Response</h4>
|
||||
<CodeBlock
|
||||
code={exampleResponseJson}
|
||||
language="json"
|
||||
showCopy={true}
|
||||
/>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Integration Guide */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Integration Guide</CardTitle>
|
||||
<CardDescription>How agents should use the Skills API</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<h4 className="font-semibold">1. Initial Discovery</h4>
|
||||
<p className="text-foreground-secondary text-sm mb-3">
|
||||
When an agent first encounters a collection, fetch the skills summary:
|
||||
</p>
|
||||
<CodeBlock
|
||||
code={`const response = await fetch(\`\${baseUrl}/\${username}/collections/\${slug}/skills\`);
|
||||
const skillsMarkdown = await response.text();
|
||||
// Parse or display the markdown summary`}
|
||||
language="typescript"
|
||||
showCopy={true}
|
||||
/>
|
||||
|
||||
<h4 className="font-semibold mt-6">2. Asking Questions</h4>
|
||||
<p className="text-foreground-secondary text-sm mb-3">
|
||||
When the agent needs guidance on using the tools:
|
||||
</p>
|
||||
<CodeBlock
|
||||
code={`const response = await fetch(\`\${baseUrl}/\${username}/collections/\${slug}/skills\`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
question: "How do I parse JSON responses from the API?",
|
||||
agentName: "my-automation-agent",
|
||||
tags: ["json", "parsing"]
|
||||
})
|
||||
});
|
||||
|
||||
const { data } = await response.json();
|
||||
console.log(data.answer); // Use this guidance`}
|
||||
language="typescript"
|
||||
showCopy={true}
|
||||
/>
|
||||
|
||||
<h4 className="font-semibold mt-6">3. Multi-Turn Conversations</h4>
|
||||
<p className="text-foreground-secondary text-sm mb-3">
|
||||
For follow-up questions, use the session ID:
|
||||
</p>
|
||||
<CodeBlock
|
||||
code={`// First question
|
||||
const first = await askSkills("How do I handle pagination?");
|
||||
const sessionId = first.data.sessionId;
|
||||
|
||||
// Follow-up (maintains context)
|
||||
const followUp = await askSkills(
|
||||
"Can you show me an example with async iteration?",
|
||||
{ sessionId }
|
||||
);`}
|
||||
language="typescript"
|
||||
showCopy={true}
|
||||
/>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Best Practices */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Best Practices</CardTitle>
|
||||
<CardDescription>Effective questioning patterns</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div className="bg-green-500/10 p-4 rounded-lg">
|
||||
<h5 className="font-semibold text-green-600 dark:text-green-400 text-sm mb-2">
|
||||
Good Questions
|
||||
</h5>
|
||||
<ul className="text-foreground-secondary text-sm space-y-2">
|
||||
<li>✓ "How do I handle rate limiting with the API tool?"</li>
|
||||
<li>✓ "What's the best way to batch multiple requests?"</li>
|
||||
<li>✓ "Can I use these tools with streaming responses?"</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div className="bg-red-500/10 p-4 rounded-lg">
|
||||
<h5 className="font-semibold text-red-600 dark:text-red-400 text-sm mb-2">
|
||||
Avoid These
|
||||
</h5>
|
||||
<ul className="text-foreground-secondary text-sm space-y-2">
|
||||
<li>✗ "Tell me everything about this collection"</li>
|
||||
<li>✗ Single-word questions like "Help"</li>
|
||||
<li>✗ Questions unrelated to the collection's tools</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="bg-background-secondary p-4 rounded-lg mt-4">
|
||||
<h4 className="font-semibold text-sm mb-2">Tips for Better Responses</h4>
|
||||
<ul className="text-foreground-secondary text-sm space-y-1">
|
||||
<li>• Be specific about what you're trying to accomplish</li>
|
||||
<li>• Include relevant context in the <code>context</code> field</li>
|
||||
<li>• Use tags to hint at the problem domain</li>
|
||||
<li>• Use sessions for related follow-up questions</li>
|
||||
</ul>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Confidence Scores */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Confidence Scores</CardTitle>
|
||||
<CardDescription>How confidence is calculated</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<p className="text-foreground-secondary">
|
||||
Each response includes a confidence score (0-1) based on:
|
||||
</p>
|
||||
|
||||
<ul className="list-disc list-inside space-y-2 text-foreground-secondary">
|
||||
<li>
|
||||
<strong>Base confidence (30%)</strong> — Minimum for any generated response
|
||||
</li>
|
||||
<li>
|
||||
<strong>Similar questions (up to 40%)</strong> — More similar past Q&A = higher
|
||||
confidence
|
||||
</li>
|
||||
<li>
|
||||
<strong>Skills documentation (20%)</strong> — Collection has generated skills.md
|
||||
</li>
|
||||
<li>
|
||||
<strong>Question volume (10%)</strong> — 3+ similar questions adds bonus
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
<div className="bg-background-secondary p-4 rounded-lg mt-4">
|
||||
<h4 className="font-semibold text-sm mb-2">Interpreting Scores</h4>
|
||||
<ul className="text-foreground-secondary text-sm space-y-1">
|
||||
<li>
|
||||
<code>>0.8</code> — High confidence, well-supported by prior Q&A
|
||||
</li>
|
||||
<li>
|
||||
<code>0.5-0.8</code> — Moderate confidence, some relevant context
|
||||
</li>
|
||||
<li>
|
||||
<code><0.5</code> — Lower confidence, limited prior knowledge
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Lazy Seeding */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Lazy Seeding</CardTitle>
|
||||
<CardDescription>Automatic bootstrapping on first access</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<p className="text-foreground-secondary">
|
||||
When a collection's skills endpoint is accessed for the first time, it automatically
|
||||
seeds with synthetic questions generated from:
|
||||
</p>
|
||||
|
||||
<ul className="list-disc list-inside space-y-2 text-foreground-secondary">
|
||||
<li>Existing skills.md documentation (if available)</li>
|
||||
<li>Tool descriptions and capabilities</li>
|
||||
<li>Common use case patterns for the tool category</li>
|
||||
</ul>
|
||||
|
||||
<p className="text-foreground-secondary mt-4">
|
||||
This ensures the endpoint is useful immediately, even before any real agent interactions.
|
||||
Seeding typically adds 10-15 synthetic Q&A pairs.
|
||||
</p>
|
||||
|
||||
<div className="bg-background-secondary p-4 rounded-lg mt-4">
|
||||
<h4 className="font-semibold text-sm mb-2">Seeding Status Response</h4>
|
||||
<p className="text-foreground-secondary text-sm">
|
||||
If seeding is in progress when you make a request, you'll receive a 202 response:
|
||||
</p>
|
||||
<CodeBlock
|
||||
code={`{
|
||||
"success": true,
|
||||
"data": {
|
||||
"status": "seeding",
|
||||
"message": "Skills are being generated. Please retry in a few seconds."
|
||||
}
|
||||
}`}
|
||||
language="json"
|
||||
showCopy={false}
|
||||
/>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Caching */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Caching Behavior</CardTitle>
|
||||
<CardDescription>How similar questions are cached</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<p className="text-foreground-secondary">
|
||||
Questions with >95% similarity to existing questions return cached answers instantly.
|
||||
This provides:
|
||||
</p>
|
||||
|
||||
<ul className="list-disc list-inside space-y-2 text-foreground-secondary">
|
||||
<li>Faster response times (~50ms vs ~1-2s)</li>
|
||||
<li>Reduced API costs</li>
|
||||
<li>Consistent answers for equivalent questions</li>
|
||||
</ul>
|
||||
|
||||
<p className="text-foreground-secondary mt-4">
|
||||
The <code>meta.cached</code> field indicates whether a cached response was used. Cached
|
||||
responses increment a <code>similarCount</code> counter for analytics.
|
||||
</p>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Next Steps */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Next Steps</CardTitle>
|
||||
<CardDescription>Continue exploring TPMJS</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<ul className="space-y-3">
|
||||
<li>
|
||||
<Link href="/docs/scenarios" className="text-primary hover:underline font-medium">
|
||||
Scenarios Guide →
|
||||
</Link>
|
||||
<p className="text-foreground-secondary text-sm">
|
||||
Automated testing for tool collections
|
||||
</p>
|
||||
</li>
|
||||
<li>
|
||||
<Link href="/docs/api/collections" className="text-primary hover:underline font-medium">
|
||||
Collections API →
|
||||
</Link>
|
||||
<p className="text-foreground-secondary text-sm">
|
||||
Create and manage tool collections
|
||||
</p>
|
||||
</li>
|
||||
<li>
|
||||
<Link href="/docs/agents" className="text-primary hover:underline font-medium">
|
||||
Agents Documentation →
|
||||
</Link>
|
||||
<p className="text-foreground-secondary text-sm">
|
||||
Build AI agents with your collections
|
||||
</p>
|
||||
</li>
|
||||
<li>
|
||||
<Link href="/" className="text-primary hover:underline font-medium">
|
||||
Browse Tool Registry →
|
||||
</Link>
|
||||
<p className="text-foreground-secondary text-sm">
|
||||
Discover tools to add to your collections
|
||||
</p>
|
||||
</li>
|
||||
</ul>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
174
apps/web/src/components/skills/SkillsActivityFeed.tsx
Normal file
174
apps/web/src/components/skills/SkillsActivityFeed.tsx
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
'use client';
|
||||
|
||||
import { Badge } from '@tpmjs/ui/Badge/Badge';
|
||||
import {
|
||||
Card,
|
||||
CardContent,
|
||||
CardDescription,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from '@tpmjs/ui/Card/Card';
|
||||
import { Icon } from '@tpmjs/ui/Icon/Icon';
|
||||
import { Skeleton } from '@tpmjs/ui/Skeleton/Skeleton';
|
||||
// Simple relative time formatter
|
||||
function formatRelativeTime(date: Date): string {
|
||||
const now = Date.now();
|
||||
const diff = now - date.getTime();
|
||||
const seconds = Math.floor(diff / 1000);
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const hours = Math.floor(minutes / 60);
|
||||
const days = Math.floor(hours / 24);
|
||||
|
||||
if (days > 0) return `${days}d ago`;
|
||||
if (hours > 0) return `${hours}h ago`;
|
||||
if (minutes > 0) return `${minutes}m ago`;
|
||||
return 'just now';
|
||||
}
|
||||
import { useEffect, useState } from 'react';
|
||||
|
||||
interface SkillQuestion {
|
||||
id: string;
|
||||
question: string;
|
||||
answer: string;
|
||||
confidence: number;
|
||||
similarCount: number;
|
||||
tags: string[];
|
||||
createdAt: string;
|
||||
skillNodes: Array<{
|
||||
skill: {
|
||||
name: string;
|
||||
};
|
||||
}>;
|
||||
}
|
||||
|
||||
interface SkillsActivityFeedProps {
|
||||
collectionId: string;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export function SkillsActivityFeed({
|
||||
collectionId,
|
||||
limit = 10,
|
||||
}: SkillsActivityFeedProps): React.ReactElement {
|
||||
const [questions, setQuestions] = useState<SkillQuestion[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
async function fetchQuestions() {
|
||||
try {
|
||||
const response = await fetch(
|
||||
`/api/skills/activity?collectionId=${collectionId}&limit=${limit}`
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to fetch activity');
|
||||
}
|
||||
const data = await response.json();
|
||||
setQuestions(data.questions || []);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Failed to load');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
fetchQuestions();
|
||||
}, [collectionId, limit]);
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div className="space-y-3">
|
||||
{[1, 2, 3].map((i) => (
|
||||
<Card key={i} variant="default">
|
||||
<CardContent padding="md">
|
||||
<Skeleton className="h-4 w-3/4 mb-2" />
|
||||
<Skeleton className="h-3 w-1/2" />
|
||||
</CardContent>
|
||||
</Card>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<Card variant="default" className="border-red-200 bg-red-50">
|
||||
<CardContent padding="md">
|
||||
<p className="text-sm text-red-600">{error}</p>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
if (questions.length === 0) {
|
||||
return (
|
||||
<Card variant="default" className="border-dashed">
|
||||
<CardContent padding="lg" className="text-center">
|
||||
<Icon
|
||||
icon="message"
|
||||
size="lg"
|
||||
className="mx-auto text-foreground-tertiary mb-2"
|
||||
/>
|
||||
<p className="text-foreground-secondary text-sm">
|
||||
No questions yet. Be the first to ask!
|
||||
</p>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-3">
|
||||
{questions.map((q) => (
|
||||
<Card key={q.id} variant="default" className="hover:border-foreground/20 transition-colors">
|
||||
<CardHeader padding="sm" className="pb-2">
|
||||
<div className="flex items-start justify-between gap-2">
|
||||
<CardTitle as="h4" className="text-sm font-medium line-clamp-2">
|
||||
{q.question}
|
||||
</CardTitle>
|
||||
<div className="flex items-center gap-1 flex-shrink-0">
|
||||
<Badge
|
||||
variant={q.confidence >= 0.7 ? 'success' : 'secondary'}
|
||||
size="sm"
|
||||
>
|
||||
{Math.round(q.confidence * 100)}%
|
||||
</Badge>
|
||||
</div>
|
||||
</div>
|
||||
</CardHeader>
|
||||
|
||||
<CardContent padding="sm" className="pt-0">
|
||||
<CardDescription className="line-clamp-2 text-xs mb-2">
|
||||
{q.answer.slice(0, 150)}
|
||||
{q.answer.length > 150 ? '...' : ''}
|
||||
</CardDescription>
|
||||
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex gap-1 flex-wrap">
|
||||
{q.skillNodes.slice(0, 2).map((sn, i) => (
|
||||
<Badge key={i} variant="outline" size="sm">
|
||||
{sn.skill.name}
|
||||
</Badge>
|
||||
))}
|
||||
{q.skillNodes.length > 2 && (
|
||||
<Badge variant="outline" size="sm">
|
||||
+{q.skillNodes.length - 2}
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex items-center gap-2 text-xs text-foreground-tertiary">
|
||||
{q.similarCount > 0 && (
|
||||
<span className="flex items-center gap-1">
|
||||
<Icon icon="user" size="sm" />
|
||||
{q.similarCount} similar
|
||||
</span>
|
||||
)}
|
||||
<span>{formatRelativeTime(new Date(q.createdAt))}</span>
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
158
apps/web/src/components/skills/SkillsSection.tsx
Normal file
158
apps/web/src/components/skills/SkillsSection.tsx
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
'use client';
|
||||
|
||||
import { Button } from '@tpmjs/ui/Button/Button';
|
||||
import { CodeBlock } from '@tpmjs/ui/CodeBlock/CodeBlock';
|
||||
import { Icon } from '@tpmjs/ui/Icon/Icon';
|
||||
import Link from 'next/link';
|
||||
import { useState } from 'react';
|
||||
import { SkillsActivityFeed } from './SkillsActivityFeed';
|
||||
import { SkillsStats } from './SkillsStats';
|
||||
|
||||
interface SkillsSectionProps {
|
||||
collectionId: string;
|
||||
username: string;
|
||||
slug: string;
|
||||
}
|
||||
|
||||
export function SkillsSection({
|
||||
collectionId,
|
||||
username,
|
||||
slug,
|
||||
}: SkillsSectionProps): React.ReactElement {
|
||||
const [showApiDocs, setShowApiDocs] = useState(false);
|
||||
|
||||
const baseUrl = typeof window !== 'undefined' ? window.location.origin : 'https://tpmjs.com';
|
||||
const skillsUrl = `${baseUrl}/${username}/collections/${slug}/skills`;
|
||||
|
||||
const apiExample = `# Ask a question about this collection's tools
|
||||
curl -X POST "${skillsUrl}" \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-d '{
|
||||
"question": "How do I handle errors with these tools?",
|
||||
"agentName": "my-agent"
|
||||
}'`;
|
||||
|
||||
const responseExample = `{
|
||||
"success": true,
|
||||
"data": {
|
||||
"answer": "To handle errors with these tools...",
|
||||
"confidence": 0.85,
|
||||
"basedOn": 3,
|
||||
"skillsIdentified": ["error-handling", "debugging"],
|
||||
"suggestedFollowups": [
|
||||
"What are the retry patterns?",
|
||||
"How do I log errors?"
|
||||
]
|
||||
},
|
||||
"meta": {
|
||||
"cached": false,
|
||||
"questionId": "clx...",
|
||||
"processingMs": 1234
|
||||
}
|
||||
}`;
|
||||
|
||||
return (
|
||||
<section className="space-y-4">
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="p-1.5 bg-purple-100 rounded-lg">
|
||||
<Icon icon="star" className="w-4 h-4 text-purple-600" />
|
||||
</div>
|
||||
<div>
|
||||
<h2 className="text-lg font-semibold text-foreground">Skills</h2>
|
||||
<p className="text-xs text-foreground-tertiary">
|
||||
Proven in the wild — not declared on paper
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => setShowApiDocs(!showApiDocs)}
|
||||
>
|
||||
<Icon icon="terminal" className="w-4 h-4 mr-1" />
|
||||
API
|
||||
</Button>
|
||||
<Link href={`/${username}/collections/${slug}/skills`}>
|
||||
<Button variant="secondary" size="sm">
|
||||
<Icon icon="externalLink" className="w-4 h-4 mr-1" />
|
||||
Full Docs
|
||||
</Button>
|
||||
</Link>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* API Documentation Toggle */}
|
||||
{showApiDocs && (
|
||||
<div className="p-4 bg-surface border border-border rounded-lg space-y-4">
|
||||
<div>
|
||||
<h4 className="text-sm font-medium mb-2">API Endpoint</h4>
|
||||
<div className="px-3 py-2 bg-background border border-border rounded-lg font-mono text-sm text-foreground-secondary overflow-x-auto">
|
||||
POST {skillsUrl}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<h4 className="text-sm font-medium mb-2">Example Request</h4>
|
||||
<CodeBlock language="bash" code={apiExample} />
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<h4 className="text-sm font-medium mb-2">Example Response</h4>
|
||||
<CodeBlock language="json" code={responseExample} />
|
||||
</div>
|
||||
|
||||
<div className="text-xs text-foreground-tertiary">
|
||||
<Link href="/docs/skills" className="text-primary hover:underline">
|
||||
View full API documentation
|
||||
</Link>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Main Content */}
|
||||
<div className="grid md:grid-cols-3 gap-4">
|
||||
{/* Stats Column */}
|
||||
<div className="md:col-span-1">
|
||||
<SkillsStats collectionId={collectionId} />
|
||||
</div>
|
||||
|
||||
{/* Activity Feed Column */}
|
||||
<div className="md:col-span-2">
|
||||
<div className="flex items-center justify-between mb-3">
|
||||
<h3 className="text-sm font-medium text-foreground-secondary">
|
||||
Recent Questions
|
||||
</h3>
|
||||
</div>
|
||||
<SkillsActivityFeed collectionId={collectionId} limit={5} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* CTA for empty state */}
|
||||
<div className="p-4 bg-gradient-to-r from-purple-50 to-blue-50 border border-purple-100 rounded-lg">
|
||||
<div className="flex items-center gap-3">
|
||||
<div className="p-2 bg-white rounded-lg shadow-sm">
|
||||
<Icon icon="message" className="w-5 h-5 text-purple-600" />
|
||||
</div>
|
||||
<div className="flex-1">
|
||||
<p className="text-sm font-medium text-foreground">
|
||||
Ask questions to build the skill graph
|
||||
</p>
|
||||
<p className="text-xs text-foreground-secondary">
|
||||
Every question helps improve future responses for all agents.
|
||||
</p>
|
||||
</div>
|
||||
<Button
|
||||
variant="default"
|
||||
size="sm"
|
||||
onClick={() => setShowApiDocs(!showApiDocs)}
|
||||
>
|
||||
Get Started
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
155
apps/web/src/components/skills/SkillsStats.tsx
Normal file
155
apps/web/src/components/skills/SkillsStats.tsx
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
'use client';
|
||||
|
||||
import { Badge } from '@tpmjs/ui/Badge/Badge';
|
||||
import {
|
||||
Card,
|
||||
CardContent,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from '@tpmjs/ui/Card/Card';
|
||||
import { Icon } from '@tpmjs/ui/Icon/Icon';
|
||||
import { Skeleton } from '@tpmjs/ui/Skeleton/Skeleton';
|
||||
import { useEffect, useState } from 'react';
|
||||
|
||||
interface SkillStats {
|
||||
totalQuestions: number;
|
||||
totalSkills: number;
|
||||
topSkills: Array<{
|
||||
name: string;
|
||||
questionCount: number;
|
||||
confidence: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface SkillsStatsProps {
|
||||
collectionId: string;
|
||||
}
|
||||
|
||||
export function SkillsStats({
|
||||
collectionId,
|
||||
}: SkillsStatsProps): React.ReactElement | null {
|
||||
const [stats, setStats] = useState<SkillStats | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
async function fetchStats() {
|
||||
try {
|
||||
const response = await fetch(
|
||||
`/api/skills/stats?collectionId=${collectionId}`
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to fetch stats');
|
||||
}
|
||||
const data = await response.json();
|
||||
setStats(data);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Failed to load');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
fetchStats();
|
||||
}, [collectionId]);
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
{[1, 2].map((i) => (
|
||||
<Card key={i} variant="default">
|
||||
<CardContent padding="md">
|
||||
<Skeleton className="h-8 w-16 mb-1" />
|
||||
<Skeleton className="h-3 w-20" />
|
||||
</CardContent>
|
||||
</Card>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<Card variant="default" className="border-red-200 bg-red-50">
|
||||
<CardContent padding="md">
|
||||
<p className="text-sm text-red-600">{error}</p>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
if (!stats) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
{/* Summary Stats */}
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<Card variant="default">
|
||||
<CardContent padding="md">
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="p-2 rounded-lg bg-blue-50">
|
||||
<Icon icon="message" size="md" className="text-blue-600" />
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-2xl font-semibold">{stats.totalQuestions}</p>
|
||||
<p className="text-xs text-foreground-secondary">Questions</p>
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Card variant="default">
|
||||
<CardContent padding="md">
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="p-2 rounded-lg bg-purple-50">
|
||||
<Icon icon="star" size="md" className="text-purple-600" />
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-2xl font-semibold">{stats.totalSkills}</p>
|
||||
<p className="text-xs text-foreground-secondary">Skills</p>
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
|
||||
{/* Top Skills */}
|
||||
{stats.topSkills.length > 0 && (
|
||||
<Card variant="default">
|
||||
<CardHeader padding="sm">
|
||||
<CardTitle as="h4" className="text-sm font-medium">
|
||||
Top Skills
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent padding="sm" className="pt-0">
|
||||
<div className="space-y-2">
|
||||
{stats.topSkills.slice(0, 5).map((skill, i) => (
|
||||
<div key={i} className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-2 min-w-0">
|
||||
<Badge variant="outline" size="sm" className="truncate max-w-[180px]">
|
||||
{skill.name}
|
||||
</Badge>
|
||||
</div>
|
||||
<div className="flex items-center gap-2 text-xs text-foreground-secondary flex-shrink-0">
|
||||
<span>{skill.questionCount} Q</span>
|
||||
<div
|
||||
className="w-12 h-1.5 bg-gray-200 rounded-full overflow-hidden"
|
||||
title={`${Math.round(skill.confidence * 100)}% confidence`}
|
||||
>
|
||||
<div
|
||||
className="h-full bg-green-500 rounded-full"
|
||||
style={{ width: `${skill.confidence * 100}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
206
apps/web/src/lib/ai/skills-embedding.ts
Normal file
206
apps/web/src/lib/ai/skills-embedding.ts
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
/**
|
||||
* Skills Embedding Service
|
||||
*
|
||||
* Uses OpenAI text-embedding-3-large (3072 dims) for high-quality
|
||||
* semantic similarity detection in the RealSkills endpoint.
|
||||
*/
|
||||
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { embed } from 'ai';
|
||||
|
||||
export const SKILLS_EMBEDDING_MODEL = 'text-embedding-3-large';
|
||||
export const SKILLS_EMBEDDING_DIMS = 3072;
|
||||
export const DEFAULT_SIMILARITY_THRESHOLD = 0.8;
|
||||
export const CACHE_HIT_THRESHOLD = 0.95;
|
||||
|
||||
/**
|
||||
* Compute embedding for a question or skill description
|
||||
*/
|
||||
export async function embedQuestion(text: string): Promise<number[]> {
|
||||
const { embedding } = await embed({
|
||||
model: openai.embedding(SKILLS_EMBEDDING_MODEL),
|
||||
value: text,
|
||||
});
|
||||
return embedding;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute cosine similarity between two vectors
|
||||
*/
|
||||
export function cosineSimilarity(a: number[], b: number[]): number {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error(
|
||||
`Vector dimension mismatch: ${a.length} vs ${b.length}`
|
||||
);
|
||||
}
|
||||
|
||||
let dotProduct = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
const aVal = a[i] ?? 0;
|
||||
const bVal = b[i] ?? 0;
|
||||
dotProduct += aVal * bVal;
|
||||
normA += aVal * aVal;
|
||||
normB += bVal * bVal;
|
||||
}
|
||||
|
||||
const denominator = Math.sqrt(normA) * Math.sqrt(normB);
|
||||
if (denominator === 0) return 0;
|
||||
|
||||
return dotProduct / denominator;
|
||||
}
|
||||
|
||||
export interface SimilarQuestion {
|
||||
id: string;
|
||||
question: string;
|
||||
answer: string;
|
||||
similarity: number;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find questions similar to a given query embedding
|
||||
*/
|
||||
export async function findSimilarQuestions(
|
||||
queryEmbedding: number[],
|
||||
collectionId: string,
|
||||
options: {
|
||||
threshold?: number;
|
||||
limit?: number;
|
||||
excludeId?: string;
|
||||
} = {}
|
||||
): Promise<SimilarQuestion[]> {
|
||||
const {
|
||||
threshold = DEFAULT_SIMILARITY_THRESHOLD,
|
||||
limit = 5,
|
||||
excludeId,
|
||||
} = options;
|
||||
|
||||
// Fetch all questions for this collection
|
||||
const questions = await prisma.skillQuestion.findMany({
|
||||
where: {
|
||||
collectionId,
|
||||
...(excludeId && { id: { not: excludeId } }),
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
question: true,
|
||||
answer: true,
|
||||
embedding: true,
|
||||
createdAt: true,
|
||||
},
|
||||
});
|
||||
|
||||
// Calculate similarity scores
|
||||
const similar: SimilarQuestion[] = [];
|
||||
|
||||
for (const q of questions) {
|
||||
const existingEmbedding = q.embedding as number[];
|
||||
if (!existingEmbedding || existingEmbedding.length === 0) continue;
|
||||
|
||||
const similarity = cosineSimilarity(queryEmbedding, existingEmbedding);
|
||||
|
||||
if (similarity >= threshold) {
|
||||
similar.push({
|
||||
id: q.id,
|
||||
question: q.question,
|
||||
answer: q.answer,
|
||||
similarity,
|
||||
createdAt: q.createdAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by similarity descending and limit results
|
||||
return similar
|
||||
.sort((a, b) => b.similarity - a.similarity)
|
||||
.slice(0, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check for a cache hit (very similar question already answered)
|
||||
* Returns the cached answer if similarity > 95%
|
||||
*/
|
||||
export async function checkCacheHit(
|
||||
queryEmbedding: number[],
|
||||
collectionId: string
|
||||
): Promise<SimilarQuestion | null> {
|
||||
const similar = await findSimilarQuestions(queryEmbedding, collectionId, {
|
||||
threshold: CACHE_HIT_THRESHOLD,
|
||||
limit: 1,
|
||||
});
|
||||
|
||||
if (similar.length > 0 && similar[0]) {
|
||||
// Increment the similar count for analytics
|
||||
await prisma.skillQuestion.update({
|
||||
where: { id: similar[0].id },
|
||||
data: { similarCount: { increment: 1 } },
|
||||
});
|
||||
return similar[0];
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find questions that match a text query (convenience wrapper)
|
||||
*/
|
||||
export async function searchQuestions(
|
||||
query: string,
|
||||
collectionId: string,
|
||||
options: {
|
||||
threshold?: number;
|
||||
limit?: number;
|
||||
} = {}
|
||||
): Promise<SimilarQuestion[]> {
|
||||
const embedding = await embedQuestion(query);
|
||||
return findSimilarQuestions(embedding, collectionId, options);
|
||||
}
|
||||
|
||||
export interface SimilarityResult {
|
||||
isCacheHit: boolean;
|
||||
cachedAnswer: string | null;
|
||||
similarQuestions: SimilarQuestion[];
|
||||
embedding: number[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Full similarity check for a new question
|
||||
* - Checks for cache hit (>95% similar)
|
||||
* - Returns similar questions for RAG context
|
||||
*/
|
||||
export async function checkQuestionSimilarity(
|
||||
question: string,
|
||||
collectionId: string
|
||||
): Promise<SimilarityResult> {
|
||||
// Generate embedding for the question
|
||||
const embedding = await embedQuestion(question);
|
||||
|
||||
// Check for cache hit first
|
||||
const cacheHit = await checkCacheHit(embedding, collectionId);
|
||||
if (cacheHit) {
|
||||
return {
|
||||
isCacheHit: true,
|
||||
cachedAnswer: cacheHit.answer,
|
||||
similarQuestions: [cacheHit],
|
||||
embedding,
|
||||
};
|
||||
}
|
||||
|
||||
// Find similar questions for RAG context
|
||||
const similarQuestions = await findSimilarQuestions(
|
||||
embedding,
|
||||
collectionId,
|
||||
{ threshold: DEFAULT_SIMILARITY_THRESHOLD, limit: 5 }
|
||||
);
|
||||
|
||||
return {
|
||||
isCacheHit: false,
|
||||
cachedAnswer: null,
|
||||
similarQuestions,
|
||||
embedding,
|
||||
};
|
||||
}
|
||||
389
apps/web/src/lib/ai/skills-graph-updater.ts
Normal file
389
apps/web/src/lib/ai/skills-graph-updater.ts
Normal file
|
|
@ -0,0 +1,389 @@
|
|||
/**
|
||||
* Skills Graph Updater
|
||||
*
|
||||
* Handles real-time skill graph updates:
|
||||
* - Infers skills from questions using embeddings
|
||||
* - Creates/matches skill nodes
|
||||
* - Links questions to skills and tools
|
||||
* - Updates confidence scores
|
||||
*/
|
||||
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import type { Skill, Tool } from '@prisma/client';
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { generateObject } from 'ai';
|
||||
import { z } from 'zod';
|
||||
import {
|
||||
cosineSimilarity,
|
||||
embedQuestion,
|
||||
} from './skills-embedding';
|
||||
|
||||
const SKILL_MATCH_THRESHOLD = 0.75;
|
||||
|
||||
/**
|
||||
* Generate a URL-safe slug from a skill name
|
||||
*/
|
||||
function slugify(text: string): string {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-|-$/g, '')
|
||||
.slice(0, 200);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract potential skills from a question using LLM
|
||||
*/
|
||||
export async function extractSkillsFromQuestion(
|
||||
question: string,
|
||||
tools: Tool[]
|
||||
): Promise<
|
||||
Array<{
|
||||
name: string;
|
||||
description: string;
|
||||
}>
|
||||
> {
|
||||
const toolNames = tools.map((t) => t.name).join(', ');
|
||||
|
||||
const { object } = await generateObject({
|
||||
model: openai('gpt-4.1-mini'),
|
||||
schema: z.object({
|
||||
skills: z.array(
|
||||
z.object({
|
||||
name: z
|
||||
.string()
|
||||
.describe('Short skill name (2-5 words), e.g., "API error handling"'),
|
||||
description: z
|
||||
.string()
|
||||
.describe('One sentence describing what this skill enables'),
|
||||
})
|
||||
),
|
||||
}),
|
||||
system: `You extract skills/capabilities from questions about tool collections.
|
||||
Tools in this collection: ${toolNames}
|
||||
|
||||
A skill represents a specific capability or use case that the tools enable.
|
||||
Examples: "API error handling", "Data transformation", "File parsing", "React state management"
|
||||
|
||||
Return 1-3 skills that this question relates to.`,
|
||||
prompt: question,
|
||||
temperature: 0.3,
|
||||
});
|
||||
|
||||
return object.skills;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find existing skills that match by embedding similarity
|
||||
*/
|
||||
async function findMatchingSkills(
|
||||
skillEmbedding: number[],
|
||||
collectionId: string,
|
||||
threshold: number = SKILL_MATCH_THRESHOLD
|
||||
): Promise<Array<{ skill: Skill; similarity: number }>> {
|
||||
const existingSkills = await prisma.skill.findMany({
|
||||
where: { collectionId },
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
slug: true,
|
||||
description: true,
|
||||
embedding: true,
|
||||
questionCount: true,
|
||||
confidence: true,
|
||||
collectionId: true,
|
||||
parentSkillId: true,
|
||||
createdAt: true,
|
||||
updatedAt: true,
|
||||
},
|
||||
});
|
||||
|
||||
const matches: Array<{ skill: Skill; similarity: number }> = [];
|
||||
|
||||
for (const skill of existingSkills) {
|
||||
const existingEmbedding = skill.embedding as number[];
|
||||
if (!existingEmbedding || existingEmbedding.length === 0) continue;
|
||||
|
||||
const similarity = cosineSimilarity(skillEmbedding, existingEmbedding);
|
||||
if (similarity >= threshold) {
|
||||
matches.push({
|
||||
skill: skill as unknown as Skill,
|
||||
similarity,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return matches.sort((a, b) => b.similarity - a.similarity);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create a skill node
|
||||
*/
|
||||
async function getOrCreateSkill(
|
||||
collectionId: string,
|
||||
name: string,
|
||||
description: string
|
||||
): Promise<Skill> {
|
||||
const slug = slugify(name);
|
||||
|
||||
// Try to find existing skill by slug
|
||||
const existing = await prisma.skill.findUnique({
|
||||
where: {
|
||||
collectionId_slug: {
|
||||
collectionId,
|
||||
slug,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
// Check by embedding similarity
|
||||
const embedding = await embedQuestion(`${name}: ${description}`);
|
||||
const matches = await findMatchingSkills(embedding, collectionId);
|
||||
|
||||
if (matches.length > 0 && matches[0]) {
|
||||
// Use existing skill if close match
|
||||
return matches[0].skill;
|
||||
}
|
||||
|
||||
// Create new skill
|
||||
return prisma.skill.create({
|
||||
data: {
|
||||
collectionId,
|
||||
name,
|
||||
slug,
|
||||
description,
|
||||
embedding: embedding as unknown as object,
|
||||
questionCount: 0,
|
||||
confidence: 0,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export interface SkillLink {
|
||||
skillId: string;
|
||||
skillName: string;
|
||||
relevance: number;
|
||||
}
|
||||
|
||||
export interface ToolLink {
|
||||
toolId: string;
|
||||
toolName: string;
|
||||
relevance: number;
|
||||
}
|
||||
|
||||
export interface GraphUpdateResult {
|
||||
skillLinks: SkillLink[];
|
||||
toolLinks: ToolLink[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract tool mentions from question/answer text
|
||||
*/
|
||||
function extractToolMentions(
|
||||
text: string,
|
||||
tools: Tool[]
|
||||
): Array<{ tool: Tool; relevance: number }> {
|
||||
const mentions: Array<{ tool: Tool; relevance: number }> = [];
|
||||
const lowerText = text.toLowerCase();
|
||||
|
||||
for (const tool of tools) {
|
||||
// Check if tool name is mentioned
|
||||
if (lowerText.includes(tool.name.toLowerCase())) {
|
||||
mentions.push({ tool, relevance: 1.0 });
|
||||
}
|
||||
}
|
||||
|
||||
return mentions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the skill graph after storing a new question
|
||||
*
|
||||
* This:
|
||||
* 1. Extracts skills from the question
|
||||
* 2. Matches or creates skill nodes
|
||||
* 3. Links the question to skills
|
||||
* 4. Links the question to mentioned tools
|
||||
* 5. Updates skill confidence scores
|
||||
*/
|
||||
export async function updateSkillGraph(params: {
|
||||
questionId: string;
|
||||
collectionId: string;
|
||||
question: string;
|
||||
answer: string;
|
||||
tools: Tool[];
|
||||
}): Promise<GraphUpdateResult> {
|
||||
const { questionId, collectionId, question, answer, tools } = params;
|
||||
|
||||
const skillLinks: SkillLink[] = [];
|
||||
const toolLinks: ToolLink[] = [];
|
||||
|
||||
// 1. Extract skills from the question
|
||||
const extractedSkills = await extractSkillsFromQuestion(question, tools);
|
||||
|
||||
// 2. Get or create skill nodes and link to question
|
||||
for (const extracted of extractedSkills) {
|
||||
try {
|
||||
const skill = await getOrCreateSkill(
|
||||
collectionId,
|
||||
extracted.name,
|
||||
extracted.description
|
||||
);
|
||||
|
||||
// Link question to skill
|
||||
await prisma.skillQuestionSkill.upsert({
|
||||
where: {
|
||||
questionId_skillId: {
|
||||
questionId,
|
||||
skillId: skill.id,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
questionId,
|
||||
skillId: skill.id,
|
||||
relevance: 1.0,
|
||||
},
|
||||
update: {
|
||||
relevance: 1.0,
|
||||
},
|
||||
});
|
||||
|
||||
// Update skill question count
|
||||
await prisma.skill.update({
|
||||
where: { id: skill.id },
|
||||
data: {
|
||||
questionCount: { increment: 1 },
|
||||
// Increase confidence with more questions
|
||||
confidence: {
|
||||
increment: 0.05,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
skillLinks.push({
|
||||
skillId: skill.id,
|
||||
skillName: skill.name,
|
||||
relevance: 1.0,
|
||||
});
|
||||
} catch (error) {
|
||||
// Log but don't fail - graph updates are best-effort
|
||||
console.error(`Failed to link skill "${extracted.name}":`, error);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Extract and link tool mentions
|
||||
const combinedText = `${question} ${answer}`;
|
||||
const toolMentions = extractToolMentions(combinedText, tools);
|
||||
|
||||
for (const { tool, relevance } of toolMentions) {
|
||||
try {
|
||||
await prisma.skillQuestionTool.upsert({
|
||||
where: {
|
||||
questionId_toolId: {
|
||||
questionId,
|
||||
toolId: tool.id,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
questionId,
|
||||
toolId: tool.id,
|
||||
relevance,
|
||||
},
|
||||
update: {
|
||||
relevance,
|
||||
},
|
||||
});
|
||||
|
||||
toolLinks.push({
|
||||
toolId: tool.id,
|
||||
toolName: tool.name,
|
||||
relevance,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(`Failed to link tool "${tool.name}":`, error);
|
||||
}
|
||||
}
|
||||
|
||||
return { skillLinks, toolLinks };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get skill summary for a collection
|
||||
*/
|
||||
export async function getCollectionSkillsSummary(
|
||||
collectionId: string
|
||||
): Promise<{
|
||||
totalQuestions: number;
|
||||
totalSkills: number;
|
||||
topSkills: Array<{
|
||||
name: string;
|
||||
questionCount: number;
|
||||
confidence: number;
|
||||
}>;
|
||||
}> {
|
||||
const [totalQuestions, totalSkills, topSkills] = await Promise.all([
|
||||
prisma.skillQuestion.count({ where: { collectionId } }),
|
||||
prisma.skill.count({ where: { collectionId } }),
|
||||
prisma.skill.findMany({
|
||||
where: { collectionId },
|
||||
orderBy: { questionCount: 'desc' },
|
||||
take: 10,
|
||||
select: {
|
||||
name: true,
|
||||
questionCount: true,
|
||||
confidence: true,
|
||||
},
|
||||
}),
|
||||
]);
|
||||
|
||||
return {
|
||||
totalQuestions,
|
||||
totalSkills,
|
||||
topSkills,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Recalculate confidence scores for all skills in a collection
|
||||
* (Useful for batch updates or maintenance)
|
||||
*/
|
||||
export async function recalculateSkillConfidence(
|
||||
collectionId: string
|
||||
): Promise<void> {
|
||||
const skills = await prisma.skill.findMany({
|
||||
where: { collectionId },
|
||||
include: {
|
||||
questions: {
|
||||
include: {
|
||||
question: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
for (const skill of skills) {
|
||||
// Base confidence on question count (logarithmic scale)
|
||||
const questionCount = skill.questions.length;
|
||||
let confidence = Math.min(1.0, Math.log10(questionCount + 1) / 2);
|
||||
|
||||
// Boost for recent questions
|
||||
const recentQuestions = skill.questions.filter((q) => {
|
||||
const daysSinceQuestion =
|
||||
(Date.now() - q.question.createdAt.getTime()) / (1000 * 60 * 60 * 24);
|
||||
return daysSinceQuestion < 30;
|
||||
});
|
||||
|
||||
if (recentQuestions.length > 0) {
|
||||
confidence += 0.1;
|
||||
}
|
||||
|
||||
await prisma.skill.update({
|
||||
where: { id: skill.id },
|
||||
data: { confidence: Math.min(1.0, confidence) },
|
||||
});
|
||||
}
|
||||
}
|
||||
221
apps/web/src/lib/ai/skills-response-generator.ts
Normal file
221
apps/web/src/lib/ai/skills-response-generator.ts
Normal file
|
|
@ -0,0 +1,221 @@
|
|||
/**
|
||||
* Skills Response Generator
|
||||
*
|
||||
* Uses GPT-4.1-mini to generate skill responses using RAG
|
||||
* from stored questions and collection context.
|
||||
*/
|
||||
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import type { Collection, Tool } from '@prisma/client';
|
||||
import { generateText, streamText } from 'ai';
|
||||
import type { SimilarQuestion } from './skills-embedding';
|
||||
|
||||
const RESPONSE_MODEL = 'gpt-4.1-mini';
|
||||
const TEMPERATURE = 0.3;
|
||||
|
||||
export interface CollectionContext {
|
||||
collection: Collection;
|
||||
tools: Array<
|
||||
Tool & {
|
||||
package: { npmPackageName: string };
|
||||
}
|
||||
>;
|
||||
skillsMarkdown?: string | null;
|
||||
}
|
||||
|
||||
export interface SessionMessage {
|
||||
role: 'user' | 'assistant';
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface GenerateResponseParams {
|
||||
question: string;
|
||||
collectionContext: CollectionContext;
|
||||
similarQuestions: SimilarQuestion[];
|
||||
sessionHistory?: SessionMessage[];
|
||||
tags?: string[];
|
||||
stream?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the system prompt for skill response generation
|
||||
*/
|
||||
function buildSystemPrompt(params: GenerateResponseParams): string {
|
||||
const { collectionContext, similarQuestions, tags } = params;
|
||||
const { collection, tools, skillsMarkdown } = collectionContext;
|
||||
|
||||
// Build tool descriptions
|
||||
const toolDescriptions = tools
|
||||
.map((t) => {
|
||||
return `- **${t.name}** (${t.package.npmPackageName}): ${t.description}`;
|
||||
})
|
||||
.join('\n');
|
||||
|
||||
// Build similar Q&A context for RAG
|
||||
const ragContext =
|
||||
similarQuestions.length > 0
|
||||
? similarQuestions
|
||||
.map((q, i) => {
|
||||
return `### Previous Question ${i + 1} (${Math.round(q.similarity * 100)}% similar)
|
||||
**Q:** ${q.question}
|
||||
**A:** ${q.answer}`;
|
||||
})
|
||||
.join('\n\n')
|
||||
: 'No similar questions have been asked yet.';
|
||||
|
||||
// Optional tag hints
|
||||
const tagHints =
|
||||
tags && tags.length > 0
|
||||
? `\nThe user has tagged this question with: ${tags.join(', ')}`
|
||||
: '';
|
||||
|
||||
return `You are a helpful assistant that answers questions about using the tools in the "${collection.name}" collection.
|
||||
|
||||
## Collection Description
|
||||
${collection.description || 'No description provided.'}
|
||||
|
||||
## Available Tools
|
||||
${toolDescriptions}
|
||||
|
||||
${skillsMarkdown ? `## Skills Documentation\n${skillsMarkdown.slice(0, 4000)}` : ''}
|
||||
|
||||
## Previous Related Questions & Answers (Use for context)
|
||||
${ragContext}
|
||||
${tagHints}
|
||||
|
||||
## Response Guidelines
|
||||
|
||||
1. **Be specific and practical** - Provide concrete examples and code snippets when helpful
|
||||
2. **Reference the tools** - When relevant, mention which tools from the collection can help
|
||||
3. **Build on previous answers** - If similar questions exist, use them as context but provide a fresh, tailored response
|
||||
4. **Use markdown formatting** - Format your response with headers, code blocks, and lists as appropriate
|
||||
5. **Be concise** - Get to the point quickly while being thorough
|
||||
6. **Admit limitations** - If you're unsure or the collection doesn't have tools for something, say so
|
||||
|
||||
Your response will be stored and used to help future users, so make it clear and reusable.`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the user prompt with session context
|
||||
*/
|
||||
function buildUserPrompt(params: GenerateResponseParams): string {
|
||||
const { question, sessionHistory } = params;
|
||||
|
||||
// Include session history for multi-turn conversations
|
||||
if (sessionHistory && sessionHistory.length > 0) {
|
||||
const historyText = sessionHistory
|
||||
.map((m) => `${m.role === 'user' ? 'User' : 'Assistant'}: ${m.content}`)
|
||||
.join('\n\n');
|
||||
|
||||
return `Previous conversation:
|
||||
${historyText}
|
||||
|
||||
Current question: ${question}`;
|
||||
}
|
||||
|
||||
return question;
|
||||
}
|
||||
|
||||
export interface GenerateResponseResult {
|
||||
answer: string;
|
||||
tokensUsed: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a skill response (non-streaming)
|
||||
*/
|
||||
export async function generateSkillResponse(
|
||||
params: GenerateResponseParams
|
||||
): Promise<GenerateResponseResult> {
|
||||
const systemPrompt = buildSystemPrompt(params);
|
||||
const userPrompt = buildUserPrompt(params);
|
||||
|
||||
const { text, usage } = await generateText({
|
||||
model: openai(RESPONSE_MODEL),
|
||||
system: systemPrompt,
|
||||
prompt: userPrompt,
|
||||
temperature: TEMPERATURE,
|
||||
});
|
||||
|
||||
return {
|
||||
answer: text,
|
||||
tokensUsed: usage?.totalTokens ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a skill response with streaming
|
||||
* Returns a ReadableStream for SSE
|
||||
*/
|
||||
export async function generateSkillResponseStream(
|
||||
params: GenerateResponseParams
|
||||
): Promise<ReadableStream> {
|
||||
const systemPrompt = buildSystemPrompt(params);
|
||||
const userPrompt = buildUserPrompt(params);
|
||||
|
||||
const result = streamText({
|
||||
model: openai(RESPONSE_MODEL),
|
||||
system: systemPrompt,
|
||||
prompt: userPrompt,
|
||||
temperature: TEMPERATURE,
|
||||
});
|
||||
|
||||
return result.textStream as unknown as ReadableStream;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate suggested follow-up questions based on the response
|
||||
*/
|
||||
export async function generateFollowupSuggestions(
|
||||
question: string,
|
||||
answer: string,
|
||||
collectionName: string
|
||||
): Promise<string[]> {
|
||||
const { text } = await generateText({
|
||||
model: openai(RESPONSE_MODEL),
|
||||
system: `You suggest follow-up questions based on a Q&A about the "${collectionName}" tool collection.
|
||||
Return exactly 3 short follow-up questions, one per line. No numbering or bullets.`,
|
||||
prompt: `Original question: ${question}
|
||||
|
||||
Answer given: ${answer.slice(0, 1000)}
|
||||
|
||||
Suggest 3 follow-up questions:`,
|
||||
temperature: 0.5,
|
||||
});
|
||||
|
||||
return text
|
||||
.split('\n')
|
||||
.map((q) => q.trim())
|
||||
.filter((q) => q.length > 0 && q.endsWith('?'))
|
||||
.slice(0, 3);
|
||||
}
|
||||
|
||||
/**
|
||||
* Infer confidence score based on RAG context quality
|
||||
*/
|
||||
export function calculateConfidence(
|
||||
similarQuestions: SimilarQuestion[],
|
||||
hasSkillsMarkdown: boolean
|
||||
): number {
|
||||
let confidence = 0.3; // Base confidence
|
||||
|
||||
// Boost for similar questions (RAG context)
|
||||
if (similarQuestions.length > 0) {
|
||||
const avgSimilarity =
|
||||
similarQuestions.reduce((sum, q) => sum + q.similarity, 0) /
|
||||
similarQuestions.length;
|
||||
confidence += avgSimilarity * 0.4; // Up to 0.4 boost
|
||||
}
|
||||
|
||||
// Boost for having skills documentation
|
||||
if (hasSkillsMarkdown) {
|
||||
confidence += 0.2;
|
||||
}
|
||||
|
||||
// Boost for multiple similar questions
|
||||
if (similarQuestions.length >= 3) {
|
||||
confidence += 0.1;
|
||||
}
|
||||
|
||||
return Math.min(1.0, confidence);
|
||||
}
|
||||
331
apps/web/src/lib/ai/skills-seeder.ts
Normal file
331
apps/web/src/lib/ai/skills-seeder.ts
Normal file
|
|
@ -0,0 +1,331 @@
|
|||
/**
|
||||
* Skills Seeder
|
||||
*
|
||||
* Lazy seeding of synthetic questions on first access.
|
||||
* Generates questions from:
|
||||
* - Existing skills.md documentation
|
||||
* - Tool capabilities and descriptions
|
||||
* - Common use case patterns
|
||||
*/
|
||||
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import type { Collection, Tool } from '@prisma/client';
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { generateObject } from 'ai';
|
||||
import { z } from 'zod';
|
||||
import { embedQuestion } from './skills-embedding';
|
||||
import {
|
||||
generateSkillResponse,
|
||||
type CollectionContext,
|
||||
} from './skills-response-generator';
|
||||
import { updateSkillGraph } from './skills-graph-updater';
|
||||
|
||||
const SEED_BATCH_SIZE = 5;
|
||||
const SEEDING_LOCK_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
|
||||
|
||||
export interface CollectionWithTools extends Collection {
|
||||
tools: Array<
|
||||
Tool & {
|
||||
package: { npmPackageName: string };
|
||||
}
|
||||
>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate synthetic questions from skills markdown
|
||||
*/
|
||||
async function generateQuestionsFromSkillsMarkdown(
|
||||
skillsMarkdown: string,
|
||||
collectionName: string,
|
||||
count: number = 5
|
||||
): Promise<string[]> {
|
||||
const { object } = await generateObject({
|
||||
model: openai('gpt-4.1-mini'),
|
||||
schema: z.object({
|
||||
questions: z.array(z.string()),
|
||||
}),
|
||||
system: `You generate realistic questions that users might ask about a tool collection.
|
||||
Generate practical, specific questions based on the skills documentation.
|
||||
Questions should be natural and varied - some simple, some complex.`,
|
||||
prompt: `Collection: ${collectionName}
|
||||
|
||||
Skills Documentation:
|
||||
${skillsMarkdown.slice(0, 4000)}
|
||||
|
||||
Generate ${count} questions that someone using these tools might ask:`,
|
||||
temperature: 0.7,
|
||||
});
|
||||
|
||||
return object.questions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate synthetic questions from tool descriptions
|
||||
*/
|
||||
async function generateQuestionsFromTools(
|
||||
tools: Array<Tool & { package: { npmPackageName: string } }>,
|
||||
count: number = 5
|
||||
): Promise<string[]> {
|
||||
const toolInfo = tools
|
||||
.slice(0, 10) // Limit to first 10 tools
|
||||
.map((t) => `- ${t.name}: ${t.description}`)
|
||||
.join('\n');
|
||||
|
||||
const { object } = await generateObject({
|
||||
model: openai('gpt-4.1-mini'),
|
||||
schema: z.object({
|
||||
questions: z.array(z.string()),
|
||||
}),
|
||||
system: `You generate realistic questions that users might ask when learning to use tools.
|
||||
Questions should cover:
|
||||
- How to use specific tools
|
||||
- Error handling
|
||||
- Common use cases
|
||||
- Integration patterns
|
||||
- Edge cases`,
|
||||
prompt: `Available tools:
|
||||
${toolInfo}
|
||||
|
||||
Generate ${count} practical questions about using these tools:`,
|
||||
temperature: 0.7,
|
||||
});
|
||||
|
||||
return object.questions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate common use case questions
|
||||
*/
|
||||
async function generateCommonUseCaseQuestions(
|
||||
collectionName: string,
|
||||
collectionDescription: string | null,
|
||||
count: number = 3
|
||||
): Promise<string[]> {
|
||||
const { object } = await generateObject({
|
||||
model: openai('gpt-4.1-mini'),
|
||||
schema: z.object({
|
||||
questions: z.array(z.string()),
|
||||
}),
|
||||
system: `You generate common, beginner-friendly questions about tool collections.
|
||||
Focus on:
|
||||
- Getting started
|
||||
- Best practices
|
||||
- Common pitfalls
|
||||
- When to use vs alternatives`,
|
||||
prompt: `Collection: ${collectionName}
|
||||
Description: ${collectionDescription || 'A collection of tools'}
|
||||
|
||||
Generate ${count} common questions someone new might ask:`,
|
||||
temperature: 0.7,
|
||||
});
|
||||
|
||||
return object.questions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if seeding is already in progress (with timeout)
|
||||
*/
|
||||
async function isSeeding(collectionId: string): Promise<boolean> {
|
||||
const collection = await prisma.collection.findUnique({
|
||||
where: { id: collectionId },
|
||||
select: { skillsSeedingAt: true },
|
||||
});
|
||||
|
||||
if (!collection?.skillsSeedingAt) return false;
|
||||
|
||||
// Check if seeding has timed out
|
||||
const elapsed = Date.now() - collection.skillsSeedingAt.getTime();
|
||||
if (elapsed > SEEDING_LOCK_TIMEOUT_MS) {
|
||||
// Clear stale lock
|
||||
await prisma.collection.update({
|
||||
where: { id: collectionId },
|
||||
data: { skillsSeedingAt: null },
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Seed a collection with synthetic questions
|
||||
* Returns true if seeding was performed, false if skipped
|
||||
*/
|
||||
export async function seedCollectionSkills(
|
||||
collection: CollectionWithTools
|
||||
): Promise<{
|
||||
seeded: boolean;
|
||||
questionsCreated: number;
|
||||
reason?: string;
|
||||
}> {
|
||||
// Check if already seeded
|
||||
if (collection.skillsSeeded) {
|
||||
return { seeded: false, questionsCreated: 0, reason: 'Already seeded' };
|
||||
}
|
||||
|
||||
// Check if seeding is in progress
|
||||
if (await isSeeding(collection.id)) {
|
||||
return { seeded: false, questionsCreated: 0, reason: 'Seeding in progress' };
|
||||
}
|
||||
|
||||
// Acquire seeding lock
|
||||
await prisma.collection.update({
|
||||
where: { id: collection.id },
|
||||
data: { skillsSeedingAt: new Date() },
|
||||
});
|
||||
|
||||
try {
|
||||
const allQuestions: string[] = [];
|
||||
|
||||
// 1. Generate from skills.md if available
|
||||
if (collection.skillsMarkdown) {
|
||||
const skillsQuestions = await generateQuestionsFromSkillsMarkdown(
|
||||
collection.skillsMarkdown,
|
||||
collection.name,
|
||||
5
|
||||
);
|
||||
allQuestions.push(...skillsQuestions);
|
||||
}
|
||||
|
||||
// 2. Generate from tool descriptions
|
||||
if (collection.tools.length > 0) {
|
||||
const toolQuestions = await generateQuestionsFromTools(
|
||||
collection.tools,
|
||||
5
|
||||
);
|
||||
allQuestions.push(...toolQuestions);
|
||||
}
|
||||
|
||||
// 3. Generate common use case questions
|
||||
const useCaseQuestions = await generateCommonUseCaseQuestions(
|
||||
collection.name,
|
||||
collection.description,
|
||||
3
|
||||
);
|
||||
allQuestions.push(...useCaseQuestions);
|
||||
|
||||
// Deduplicate questions
|
||||
const uniqueQuestions = [...new Set(allQuestions)];
|
||||
|
||||
// 4. Process questions in batches
|
||||
let questionsCreated = 0;
|
||||
const collectionContext: CollectionContext = {
|
||||
collection,
|
||||
tools: collection.tools,
|
||||
skillsMarkdown: collection.skillsMarkdown,
|
||||
};
|
||||
|
||||
for (let i = 0; i < uniqueQuestions.length; i += SEED_BATCH_SIZE) {
|
||||
const batch = uniqueQuestions.slice(i, i + SEED_BATCH_SIZE);
|
||||
|
||||
for (const question of batch) {
|
||||
try {
|
||||
// Generate embedding
|
||||
const embedding = await embedQuestion(question);
|
||||
|
||||
// Generate answer
|
||||
const { answer, tokensUsed } = await generateSkillResponse({
|
||||
question,
|
||||
collectionContext,
|
||||
similarQuestions: [], // No similar questions for seed
|
||||
stream: false,
|
||||
});
|
||||
|
||||
// Store the question
|
||||
const stored = await prisma.skillQuestion.create({
|
||||
data: {
|
||||
collectionId: collection.id,
|
||||
question,
|
||||
embedding: embedding as unknown as object,
|
||||
answer,
|
||||
answerTokens: tokensUsed,
|
||||
agentName: 'seed-bot',
|
||||
confidence: 0.5, // Medium confidence for synthetic
|
||||
tags: ['synthetic', 'seed'],
|
||||
},
|
||||
});
|
||||
|
||||
// Update skill graph (best-effort)
|
||||
try {
|
||||
await updateSkillGraph({
|
||||
questionId: stored.id,
|
||||
collectionId: collection.id,
|
||||
question,
|
||||
answer,
|
||||
tools: collection.tools,
|
||||
});
|
||||
} catch {
|
||||
// Don't fail seeding if graph update fails
|
||||
}
|
||||
|
||||
questionsCreated++;
|
||||
} catch (error) {
|
||||
console.error(`Failed to seed question: "${question}"`, error);
|
||||
// Continue with other questions
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Mark as seeded
|
||||
await prisma.collection.update({
|
||||
where: { id: collection.id },
|
||||
data: {
|
||||
skillsSeeded: true,
|
||||
skillsSeedingAt: null,
|
||||
},
|
||||
});
|
||||
|
||||
return { seeded: true, questionsCreated };
|
||||
} catch (error) {
|
||||
// Clear seeding lock on error
|
||||
await prisma.collection.update({
|
||||
where: { id: collection.id },
|
||||
data: { skillsSeedingAt: null },
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check seeding status for a collection
|
||||
*/
|
||||
export async function getSeedingStatus(collectionId: string): Promise<{
|
||||
isSeeded: boolean;
|
||||
isSeeding: boolean;
|
||||
questionCount: number;
|
||||
}> {
|
||||
const [collection, questionCount] = await Promise.all([
|
||||
prisma.collection.findUnique({
|
||||
where: { id: collectionId },
|
||||
select: { skillsSeeded: true, skillsSeedingAt: true },
|
||||
}),
|
||||
prisma.skillQuestion.count({ where: { collectionId } }),
|
||||
]);
|
||||
|
||||
if (!collection) {
|
||||
return { isSeeded: false, isSeeding: false, questionCount: 0 };
|
||||
}
|
||||
|
||||
const isCurrentlySeeding =
|
||||
collection.skillsSeedingAt &&
|
||||
Date.now() - collection.skillsSeedingAt.getTime() < SEEDING_LOCK_TIMEOUT_MS;
|
||||
|
||||
return {
|
||||
isSeeded: collection.skillsSeeded,
|
||||
isSeeding: !!isCurrentlySeeding,
|
||||
questionCount,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset seeding status (for manual re-seeding)
|
||||
*/
|
||||
export async function resetSeedingStatus(collectionId: string): Promise<void> {
|
||||
await prisma.collection.update({
|
||||
where: { id: collectionId },
|
||||
data: {
|
||||
skillsSeeded: false,
|
||||
skillsSeedingAt: null,
|
||||
},
|
||||
});
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue