From 1118463e6ba3ef7c1cbfd48163ce73144dc9589d Mon Sep 17 00:00:00 2001 From: Ajax Davis Date: Sun, 25 Jan 2026 02:37:02 +1000 Subject: [PATCH] feat(skills): add RealSkills living endpoint for agent Q&A Implements a skills endpoint that evolves through agent conversations: - GET /:username/collections/:slug/skills - Returns markdown skill summary - POST /:username/collections/:slug/skills - Ask questions, get RAG+LLM responses Features: - OpenAI text-embedding-3-large (3072 dims) for semantic similarity - GPT-4.1-mini for response generation with RAG context - Lazy seeding of synthetic questions on first access - Cache hits for >95% similar questions - Real-time skill graph updates (emergent skill taxonomy) - Session support for multi-turn conversations - Activity feed and stats APIs for UI Database models: SkillQuestion, Skill, SkillSession, SkillQuestionSkill, SkillQuestionTool --- .../[slug]/CollectionDetailClient.tsx | 10 + .../collections/[slug]/skills/route.ts | 537 ++++++++++++++++++ apps/web/src/app/api/skills/activity/route.ts | 81 +++ apps/web/src/app/api/skills/stats/route.ts | 73 +++ apps/web/src/app/docs/skills/page.tsx | 521 +++++++++++++++++ .../components/skills/SkillsActivityFeed.tsx | 174 ++++++ .../src/components/skills/SkillsSection.tsx | 158 ++++++ .../web/src/components/skills/SkillsStats.tsx | 155 +++++ apps/web/src/lib/ai/skills-embedding.ts | 206 +++++++ apps/web/src/lib/ai/skills-graph-updater.ts | 389 +++++++++++++ .../src/lib/ai/skills-response-generator.ts | 221 +++++++ apps/web/src/lib/ai/skills-seeder.ts | 331 +++++++++++ packages/db/prisma/schema.prisma | 153 ++++- 13 files changed, 3001 insertions(+), 8 deletions(-) create mode 100644 apps/web/src/app/(profile)/[username]/collections/[slug]/skills/route.ts create mode 100644 apps/web/src/app/api/skills/activity/route.ts create mode 100644 apps/web/src/app/api/skills/stats/route.ts create mode 100644 apps/web/src/app/docs/skills/page.tsx create mode 100644 apps/web/src/components/skills/SkillsActivityFeed.tsx create mode 100644 apps/web/src/components/skills/SkillsSection.tsx create mode 100644 apps/web/src/components/skills/SkillsStats.tsx create mode 100644 apps/web/src/lib/ai/skills-embedding.ts create mode 100644 apps/web/src/lib/ai/skills-graph-updater.ts create mode 100644 apps/web/src/lib/ai/skills-response-generator.ts create mode 100644 apps/web/src/lib/ai/skills-seeder.ts diff --git a/apps/web/src/app/(profile)/[username]/collections/[slug]/CollectionDetailClient.tsx b/apps/web/src/app/(profile)/[username]/collections/[slug]/CollectionDetailClient.tsx index afca7ee..4d09ea8 100644 --- a/apps/web/src/app/(profile)/[username]/collections/[slug]/CollectionDetailClient.tsx +++ b/apps/web/src/app/(profile)/[username]/collections/[slug]/CollectionDetailClient.tsx @@ -12,6 +12,7 @@ import { ForkedFromBadge } from '~/components/ForkedFromBadge'; import { LikeButton } from '~/components/LikeButton'; import { ScenariosSection } from '~/components/ScenariosSection'; import { ShareButton } from '~/components/ShareButton'; +import { SkillsSection } from '~/components/skills/SkillsSection'; import { UseCasesSection } from '~/components/UseCasesSection'; import { useSession } from '~/lib/auth-client'; @@ -412,6 +413,15 @@ export function CollectionDetailClient({ slug={collection.slug} /> )} + + {/* Skills Section */} + {collection.tools.length > 0 && ( + + )} diff --git a/apps/web/src/app/(profile)/[username]/collections/[slug]/skills/route.ts b/apps/web/src/app/(profile)/[username]/collections/[slug]/skills/route.ts new file mode 100644 index 0000000..bdffd96 --- /dev/null +++ b/apps/web/src/app/(profile)/[username]/collections/[slug]/skills/route.ts @@ -0,0 +1,537 @@ +/** + * RealSkills API Endpoint + * + * A living skills endpoint that evolves through agent conversations. + * Skills emerge organically from question patterns. + * + * GET - Return skill summary markdown (triggers lazy seeding) + * POST - Ask a question (RAG + LLM response) + */ + +import { createHash } from 'crypto'; +import { prisma } from '@tpmjs/db'; +import { NextResponse, type NextRequest } from 'next/server'; +import { z } from 'zod'; + +import { checkQuestionSimilarity } from '~/lib/ai/skills-embedding'; +import { updateSkillGraph, getCollectionSkillsSummary } from '~/lib/ai/skills-graph-updater'; +import { + generateSkillResponse, + generateFollowupSuggestions, + calculateConfidence, + type CollectionContext, +} from '~/lib/ai/skills-response-generator'; +import { + seedCollectionSkills, + getSeedingStatus, + type CollectionWithTools, +} from '~/lib/ai/skills-seeder'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; +export const maxDuration = 300; + +type RouteContext = { + params: Promise<{ username: string; slug: string }>; +}; + +// Request validation schemas +const PostRequestSchema = z.object({ + question: z.string().min(5).max(2000), + sessionId: z.string().optional(), + agentName: z.string().max(100).optional(), + context: z.string().max(2000).optional(), + tags: z.array(z.string().max(50)).max(10).optional(), +}); + +// Session expiry time (24 hours) +const SESSION_EXPIRY_MS = 24 * 60 * 60 * 1000; + +/** + * Hash agent identity for anonymization + */ +function hashAgentIdentity(ip: string, userAgent: string): string { + return createHash('sha256') + .update(`${ip}:${userAgent}`) + .digest('hex') + .slice(0, 16); +} + +/** + * Load collection with tools + */ +async function loadCollection( + username: string, + slug: string +): Promise { + const user = await prisma.user.findUnique({ + where: { username }, + select: { id: true, username: true }, + }); + + if (!user || !user.username) { + return NextResponse.json( + { success: false, error: 'User not found' }, + { status: 404 } + ); + } + + const collection = await prisma.collection.findFirst({ + where: { slug, userId: user.id }, + include: { + tools: { + include: { + tool: { + include: { + package: { + select: { npmPackageName: true, npmVersion: true }, + }, + }, + }, + }, + orderBy: { position: 'asc' }, + take: 100, + }, + }, + }); + + if (!collection) { + return NextResponse.json( + { success: false, error: 'Collection not found' }, + { status: 404 } + ); + } + + if (!collection.isPublic) { + return NextResponse.json( + { success: false, error: 'Collection is not public' }, + { status: 403 } + ); + } + + // Flatten the tools structure + const collectionWithTools = { + ...collection, + tools: collection.tools.map((ct) => ({ + ...ct.tool, + package: ct.tool.package, + })), + }; + + return collectionWithTools as CollectionWithTools; +} + +/** + * Ensure collection is seeded (lazy seeding) + */ +async function ensureSeeded(collection: CollectionWithTools): Promise<{ + isSeeding: boolean; + wasSeeded: boolean; +}> { + const status = await getSeedingStatus(collection.id); + + if (status.isSeeded) { + return { isSeeding: false, wasSeeded: false }; + } + + if (status.isSeeding) { + return { isSeeding: true, wasSeeded: false }; + } + + // Trigger seeding (non-blocking for GET, blocking for POST) + try { + const result = await seedCollectionSkills(collection); + return { isSeeding: false, wasSeeded: result.seeded }; + } catch (error) { + console.error('[Skills] Seeding failed:', error); + // Continue without seeding - endpoint still works + return { isSeeding: false, wasSeeded: false }; + } +} + +/** + * GET /:username/collections/:slug/skills + * + * Returns skill summary as markdown. + * Triggers lazy seeding on first access. + */ +export async function GET(_request: NextRequest, context: RouteContext) { + const startTime = Date.now(); + + try { + const { username: rawUsername, slug } = await context.params; + const username = rawUsername.startsWith('@') + ? rawUsername.slice(1) + : rawUsername; + + // Load collection + const result = await loadCollection(username, slug); + if (result instanceof NextResponse) return result; + const collection = result; + + // Check/trigger seeding + const seedStatus = await ensureSeeded(collection); + + if (seedStatus.isSeeding) { + return NextResponse.json( + { + success: true, + data: { + status: 'seeding', + message: 'Skills are being generated. Please retry in a few seconds.', + }, + }, + { + status: 202, + headers: { 'Retry-After': '10' }, + } + ); + } + + // Get skill summary + const summary = await getCollectionSkillsSummary(collection.id); + + // Build markdown response + const markdown = buildSkillsSummaryMarkdown(collection, summary, username); + + return new Response(markdown, { + status: 200, + headers: { + 'Content-Type': 'text/markdown; charset=utf-8', + 'X-Skills-Total-Questions': summary.totalQuestions.toString(), + 'X-Skills-Total-Skills': summary.totalSkills.toString(), + 'X-Processing-Time-Ms': (Date.now() - startTime).toString(), + }, + }); + } catch (error) { + console.error('[Skills GET Error]:', error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Internal server error', + }, + { status: 500 } + ); + } +} + +/** + * POST /:username/collections/:slug/skills + * + * Submit a question and get a skill-based response. + */ +export async function POST(request: NextRequest, context: RouteContext) { + const startTime = Date.now(); + + try { + const { username: rawUsername, slug } = await context.params; + const username = rawUsername.startsWith('@') + ? rawUsername.slice(1) + : rawUsername; + + // Parse and validate request body + let body: unknown; + try { + body = await request.json(); + } catch { + return NextResponse.json( + { success: false, error: 'Invalid JSON body' }, + { status: 400 } + ); + } + + const parseResult = PostRequestSchema.safeParse(body); + if (!parseResult.success) { + return NextResponse.json( + { + success: false, + error: 'Validation error', + details: parseResult.error.format(), + }, + { status: 400 } + ); + } + + const { question, sessionId, agentName, context: questionContext, tags } = + parseResult.data; + + // Load collection + const result = await loadCollection(username, slug); + if (result instanceof NextResponse) return result; + const collection = result; + + // Ensure seeded + await ensureSeeded(collection); + + // Get agent identity + const ip = request.headers.get('x-forwarded-for') || 'unknown'; + const userAgent = request.headers.get('user-agent') || 'unknown'; + const agentHash = hashAgentIdentity(ip, userAgent); + + // Check for similarity / cache hit + const similarityResult = await checkQuestionSimilarity( + question, + collection.id + ); + + // If very similar question exists (>95%), return cached answer + if (similarityResult.isCacheHit && similarityResult.cachedAnswer) { + const cachedQuestion = similarityResult.similarQuestions[0]; + return NextResponse.json({ + success: true, + data: { + answer: similarityResult.cachedAnswer, + confidence: cachedQuestion?.similarity || 0.95, + basedOn: 1, + skillsIdentified: [], + cached: true, + }, + meta: { + cached: true, + questionId: cachedQuestion?.id || null, + processingMs: Date.now() - startTime, + }, + }); + } + + // Build collection context + const collectionContext: CollectionContext = { + collection, + tools: collection.tools, + skillsMarkdown: collection.skillsMarkdown, + }; + + // Get session history if session exists + let sessionHistory: Array<{ role: 'user' | 'assistant'; content: string }> = + []; + let activeSessionId = sessionId; + + if (sessionId) { + const session = await prisma.skillSession.findUnique({ + where: { id: sessionId }, + }); + if (session && session.collectionId === collection.id) { + sessionHistory = session.context as Array<{ + role: 'user' | 'assistant'; + content: string; + }>; + } + } + + // Generate response + const fullQuestion = questionContext + ? `${question}\n\nContext: ${questionContext}` + : question; + + const { answer, tokensUsed } = await generateSkillResponse({ + question: fullQuestion, + collectionContext, + similarQuestions: similarityResult.similarQuestions, + sessionHistory, + tags, + }); + + // Calculate confidence + const confidence = calculateConfidence( + similarityResult.similarQuestions, + !!collection.skillsMarkdown + ); + + // Store the question + const storedQuestion = await prisma.skillQuestion.create({ + data: { + collectionId: collection.id, + question, + embedding: similarityResult.embedding as unknown as object, + answer, + answerTokens: tokensUsed, + agentHash, + agentName: agentName || null, + sessionId: activeSessionId, + confidence, + tags: tags || [], + }, + }); + + // Update skill graph (best-effort, don't fail request) + let skillLinks: Array<{ skillId: string; skillName: string }> = []; + try { + const graphResult = await updateSkillGraph({ + questionId: storedQuestion.id, + collectionId: collection.id, + question, + answer, + tools: collection.tools, + }); + skillLinks = graphResult.skillLinks; + } catch (error) { + console.error('[Skills] Graph update failed:', error); + } + + // Update or create session + if (sessionId || sessionHistory.length > 0) { + const newHistory = [ + ...sessionHistory, + { role: 'user' as const, content: question }, + { role: 'assistant' as const, content: answer }, + ].slice(-20); // Keep last 20 messages + + if (sessionId) { + await prisma.skillSession.update({ + where: { id: sessionId }, + data: { + context: newHistory, + updatedAt: new Date(), + }, + }); + } else { + const newSession = await prisma.skillSession.create({ + data: { + collectionId: collection.id, + context: newHistory, + agentHash, + agentName, + expiresAt: new Date(Date.now() + SESSION_EXPIRY_MS), + }, + }); + activeSessionId = newSession.id; + } + } + + // Generate follow-up suggestions (optional, don't block) + let suggestedFollowups: string[] = []; + try { + suggestedFollowups = await generateFollowupSuggestions( + question, + answer, + collection.name + ); + } catch { + // Ignore errors for followups + } + + return NextResponse.json({ + success: true, + data: { + answer, + confidence, + basedOn: similarityResult.similarQuestions.length, + skillsIdentified: skillLinks.map((s) => s.skillName), + sessionId: activeSessionId, + suggestedFollowups, + }, + meta: { + cached: false, + questionId: storedQuestion.id, + processingMs: Date.now() - startTime, + }, + }); + } catch (error) { + console.error('[Skills POST Error]:', error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Internal server error', + }, + { status: 500 } + ); + } +} + +/** + * Build markdown summary of skills + */ +function buildSkillsSummaryMarkdown( + collection: CollectionWithTools, + summary: { + totalQuestions: number; + totalSkills: number; + topSkills: Array<{ + name: string; + questionCount: number; + confidence: number; + }>; + }, + username: string +): string { + const baseUrl = process.env.NEXT_PUBLIC_APP_URL || 'https://tpmjs.com'; + const skillsUrl = `${baseUrl}/${username}/collections/${collection.slug}/skills`; + + let markdown = `# Skills: ${collection.name} + +> Skills, proven in the wild — not declared on paper. + +This collection has evolved through **${summary.totalQuestions} questions** from agents, identifying **${summary.totalSkills} distinct skills**. + +## API Usage + +\`\`\`bash +# Ask a question +curl -X POST ${skillsUrl} \\ + -H "Content-Type: application/json" \\ + -d '{"question": "How do I handle errors with these tools?"}' +\`\`\` + +## Top Skills + +`; + + if (summary.topSkills.length > 0) { + for (const skill of summary.topSkills) { + const confidenceBar = '█'.repeat(Math.floor(skill.confidence * 10)); + const confidenceEmpty = '░'.repeat(10 - Math.floor(skill.confidence * 10)); + markdown += `- **${skill.name}** (${skill.questionCount} questions) ${confidenceBar}${confidenceEmpty}\n`; + } + } else { + markdown += `*No skills identified yet. Ask questions to start building the skill graph.*\n`; + } + + markdown += ` +## How It Works + +1. **You ask a question** via POST +2. We find similar past questions (RAG) +3. We generate a tailored response +4. Your question helps evolve the skill graph +5. Future questions get better answers + +## Request Schema + +\`\`\`typescript +interface SkillsRequest { + question: string; // Required (5-2000 chars) + sessionId?: string; // For multi-turn conversations + agentName?: string; // Self-reported agent identity + context?: string; // Additional context (max 2000 chars) + tags?: string[]; // Hint tags (max 10) +} +\`\`\` + +## Response Schema + +\`\`\`typescript +interface SkillsResponse { + success: boolean; + data: { + answer: string; // Markdown response + confidence: number; // 0-1 confidence score + basedOn: number; // Similar questions used + skillsIdentified: string[]; + sessionId?: string; + suggestedFollowups?: string[]; + }; + meta: { + cached: boolean; + questionId: string; + processingMs: number; + }; +} +\`\`\` + +--- + +*Last updated: ${new Date().toISOString()}* +`; + + return markdown; +} diff --git a/apps/web/src/app/api/skills/activity/route.ts b/apps/web/src/app/api/skills/activity/route.ts new file mode 100644 index 0000000..97ffcf8 --- /dev/null +++ b/apps/web/src/app/api/skills/activity/route.ts @@ -0,0 +1,81 @@ +/** + * GET /api/skills/activity + * + * Fetch recent skill questions for a collection (anonymized for activity feed) + */ + +import { prisma } from '@tpmjs/db'; +import { NextResponse, type NextRequest } from 'next/server'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +export async function GET(request: NextRequest) { + const searchParams = request.nextUrl.searchParams; + const collectionId = searchParams.get('collectionId'); + const limitParam = searchParams.get('limit'); + const limit = Math.min(50, Math.max(1, parseInt(limitParam || '10', 10))); + + if (!collectionId) { + return NextResponse.json( + { error: 'collectionId is required' }, + { status: 400 } + ); + } + + try { + // Verify collection exists and is public + const collection = await prisma.collection.findUnique({ + where: { id: collectionId }, + select: { id: true, isPublic: true }, + }); + + if (!collection) { + return NextResponse.json( + { error: 'Collection not found' }, + { status: 404 } + ); + } + + if (!collection.isPublic) { + return NextResponse.json( + { error: 'Collection is not public' }, + { status: 403 } + ); + } + + // Fetch recent questions with skill links (anonymized - no agent info) + const questions = await prisma.skillQuestion.findMany({ + where: { collectionId }, + orderBy: { createdAt: 'desc' }, + take: limit, + select: { + id: true, + question: true, + answer: true, + confidence: true, + similarCount: true, + tags: true, + createdAt: true, + // Include skills but not agent info for privacy + skillNodes: { + select: { + skill: { + select: { + name: true, + }, + }, + }, + }, + }, + }); + + return NextResponse.json({ questions }); + } catch (error) { + console.error('[Skills Activity Error]:', error); + return NextResponse.json( + { error: 'Failed to fetch activity' }, + { status: 500 } + ); + } +} diff --git a/apps/web/src/app/api/skills/stats/route.ts b/apps/web/src/app/api/skills/stats/route.ts new file mode 100644 index 0000000..af09da1 --- /dev/null +++ b/apps/web/src/app/api/skills/stats/route.ts @@ -0,0 +1,73 @@ +/** + * GET /api/skills/stats + * + * Get skill statistics for a collection + */ + +import { prisma } from '@tpmjs/db'; +import { NextResponse, type NextRequest } from 'next/server'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +export async function GET(request: NextRequest) { + const searchParams = request.nextUrl.searchParams; + const collectionId = searchParams.get('collectionId'); + + if (!collectionId) { + return NextResponse.json( + { error: 'collectionId is required' }, + { status: 400 } + ); + } + + try { + // Verify collection exists and is public + const collection = await prisma.collection.findUnique({ + where: { id: collectionId }, + select: { id: true, isPublic: true }, + }); + + if (!collection) { + return NextResponse.json( + { error: 'Collection not found' }, + { status: 404 } + ); + } + + if (!collection.isPublic) { + return NextResponse.json( + { error: 'Collection is not public' }, + { status: 403 } + ); + } + + // Fetch stats in parallel + const [totalQuestions, totalSkills, topSkills] = await Promise.all([ + prisma.skillQuestion.count({ where: { collectionId } }), + prisma.skill.count({ where: { collectionId } }), + prisma.skill.findMany({ + where: { collectionId }, + orderBy: { questionCount: 'desc' }, + take: 10, + select: { + name: true, + questionCount: true, + confidence: true, + }, + }), + ]); + + return NextResponse.json({ + totalQuestions, + totalSkills, + topSkills, + }); + } catch (error) { + console.error('[Skills Stats Error]:', error); + return NextResponse.json( + { error: 'Failed to fetch stats' }, + { status: 500 } + ); + } +} diff --git a/apps/web/src/app/docs/skills/page.tsx b/apps/web/src/app/docs/skills/page.tsx new file mode 100644 index 0000000..01fe823 --- /dev/null +++ b/apps/web/src/app/docs/skills/page.tsx @@ -0,0 +1,521 @@ +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@tpmjs/ui/Card/Card'; +import { CodeBlock } from '@tpmjs/ui/CodeBlock/CodeBlock'; +import type { Metadata } from 'next'; +import Link from 'next/link'; + +export const metadata: Metadata = { + title: 'RealSkills API | TPMJS Docs', + description: + 'A living skills endpoint that evolves through agent conversations. Skills emerge organically from question patterns.', +}; + +const exampleResponseJson = `{ + "success": true, + "data": { + "answer": "To handle errors with these tools...", + "confidence": 0.85, + "basedOn": 3, + "skillsIdentified": ["error-handling", "try-catch-patterns"], + "sessionId": "sess_abc123", + "suggestedFollowups": [ + "What are the retry patterns?", + "How do I log errors?" + ] + }, + "meta": { + "cached": false, + "questionId": "clx123abc456", + "processingMs": 1234 + } +}`; + +export default function SkillsPage(): React.ReactElement { + return ( +
+
+

RealSkills API

+

+ Skills, proven in the wild — not declared on paper. +

+

+ A living skills endpoint that evolves through agent conversations. Unlike static + documentation, skills emerge organically from question patterns and improve over time. +

+
+ + {/* Philosophy */} + + + Philosophy + Why living skills beats static skills.md + + +

+ Traditional documentation is written once and becomes outdated. RealSkills takes a + different approach: +

+
    +
  • + Questions drive discovery — Every agent question reveals what users + actually need +
    • +
  • + Answers compound — Similar questions get better answers based on + previous responses +
    • +
  • + Skills emerge — Patterns in questions automatically create skill + categories +
    • +
  • + Quality improves — More questions = more context = better responses +
    • +
+

+ Think of it as a knowledge base that learns from every interaction. +

+ + + + {/* How It Works */} + + + How It Works + The question → skill inference loop + + +
+ 
{`Agent POSTs question
+        ↓
+┌─────────────────────────────┐
+│   /skills endpoint          │
+│   - Embed question          │
+│   - Check similarity cache  │
+│   - RAG from stored Q&A     │
+│   - Generate response (LLM) │
+│   - Store question + answer │
+│   - Update skill graph      │
+└─────────────────────────────┘
+        ↓
+    Return skill guidance (markdown)
+        ↓
+    Skill graph evolves in real-time`}
+
+ +
    +
  1. + Question Received — Agent submits a question via POST +
    2. +
  2. + Embedding Generated — Question is converted to a 3072-dimensional + vector +
    3. +
  3. + Similarity Check — If >95% similar to existing question, return + cached answer +
    4. +
  4. + RAG Context — Find similar past questions/answers for context +
    5. +
  5. + Response Generation — GPT-4.1-mini generates a tailored response +
    6. +
  6. + Storage & Graph Update — Question stored, skills inferred and linked +
    7. +
+ + + + {/* API Reference */} + + + API Reference + GET and POST endpoints + + +
+

GET /:username/collections/:slug/skills

+

+ Returns the skill summary as markdown. Triggers lazy seeding on first access. +

+ +
+ +
+

POST /:username/collections/:slug/skills

+

+ Submit a question and receive an AI-generated response based on the collection's + tools and previous Q&A. +

+ +
+ + + + {/* Request Schema */} + + + Request Schema + POST request body format + + + + +
+

Multi-Turn Conversations

+

+ To continue a conversation, include the sessionId from a previous + response. Sessions maintain context for up to 24 hours and include the last 20 + messages. +

+
+ + + + {/* Response Schema */} + + + Response Schema + Successful response format + + + + +

Example Response

+ + + + + {/* Integration Guide */} + + + Integration Guide + How agents should use the Skills API + + +

1. Initial Discovery

+

+ When an agent first encounters a collection, fetch the skills summary: +

+ + +

2. Asking Questions

+

+ When the agent needs guidance on using the tools: +

+ + +

3. Multi-Turn Conversations

+

+ For follow-up questions, use the session ID: +

+ + + + + {/* Best Practices */} + + + Best Practices + Effective questioning patterns + + +
+
+
+ Good Questions +
+
    +
  • ✓ "How do I handle rate limiting with the API tool?"
    • +
  • ✓ "What's the best way to batch multiple requests?"
    • +
  • ✓ "Can I use these tools with streaming responses?"
    • +
+
+
+
+ Avoid These +
+
    +
  • ✗ "Tell me everything about this collection"
    • +
  • ✗ Single-word questions like "Help"
    • +
  • ✗ Questions unrelated to the collection's tools
    • +
+
+
+ +
+

Tips for Better Responses

+
    +
  • • Be specific about what you're trying to accomplish
    • +
  • • Include relevant context in the context field
    • +
  • • Use tags to hint at the problem domain
    • +
  • • Use sessions for related follow-up questions
    • +
+
+ + + + {/* Confidence Scores */} + + + Confidence Scores + How confidence is calculated + + +

+ Each response includes a confidence score (0-1) based on: +

+ +
    +
  • + Base confidence (30%) — Minimum for any generated response +
    • +
  • + Similar questions (up to 40%) — More similar past Q&A = higher + confidence +
    • +
  • + Skills documentation (20%) — Collection has generated skills.md +
    • +
  • + Question volume (10%) — 3+ similar questions adds bonus +
    • +
+ +
+

Interpreting Scores

+
    +
  • + >0.8 — High confidence, well-supported by prior Q&A +
    • +
  • + 0.5-0.8 — Moderate confidence, some relevant context +
    • +
  • + <0.5 — Lower confidence, limited prior knowledge +
    • +
+
+ + + + {/* Lazy Seeding */} + + + Lazy Seeding + Automatic bootstrapping on first access + + +

+ When a collection's skills endpoint is accessed for the first time, it automatically + seeds with synthetic questions generated from: +

+ +
    +
  • Existing skills.md documentation (if available)
    • +
  • Tool descriptions and capabilities
    • +
  • Common use case patterns for the tool category
    • +
+ +

+ This ensures the endpoint is useful immediately, even before any real agent interactions. + Seeding typically adds 10-15 synthetic Q&A pairs. +

+ +
+

Seeding Status Response

+

+ If seeding is in progress when you make a request, you'll receive a 202 response: +

+ +
+ + + + {/* Caching */} + + + Caching Behavior + How similar questions are cached + + +

+ Questions with >95% similarity to existing questions return cached answers instantly. + This provides: +

+ +
    +
  • Faster response times (~50ms vs ~1-2s)
    • +
  • Reduced API costs
    • +
  • Consistent answers for equivalent questions
    • +
+ +

+ The meta.cached field indicates whether a cached response was used. Cached + responses increment a similarCount counter for analytics. +

+ + + + {/* Next Steps */} + + + Next Steps + Continue exploring TPMJS + + +
    +
  • + + Scenarios Guide → + +

    + Automated testing for tool collections +

    +
    • +
  • + + Collections API → + +

    + Create and manage tool collections +

    +
    • +
  • + + Agents Documentation → + +

    + Build AI agents with your collections +

    +
    • +
  • + + Browse Tool Registry → + +

    + Discover tools to add to your collections +

    +
    • +
+ + +
+ ); +} diff --git a/apps/web/src/components/skills/SkillsActivityFeed.tsx b/apps/web/src/components/skills/SkillsActivityFeed.tsx new file mode 100644 index 0000000..9eb6824 --- /dev/null +++ b/apps/web/src/components/skills/SkillsActivityFeed.tsx @@ -0,0 +1,174 @@ +'use client'; + +import { Badge } from '@tpmjs/ui/Badge/Badge'; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from '@tpmjs/ui/Card/Card'; +import { Icon } from '@tpmjs/ui/Icon/Icon'; +import { Skeleton } from '@tpmjs/ui/Skeleton/Skeleton'; +// Simple relative time formatter +function formatRelativeTime(date: Date): string { + const now = Date.now(); + const diff = now - date.getTime(); + const seconds = Math.floor(diff / 1000); + const minutes = Math.floor(seconds / 60); + const hours = Math.floor(minutes / 60); + const days = Math.floor(hours / 24); + + if (days > 0) return `${days}d ago`; + if (hours > 0) return `${hours}h ago`; + if (minutes > 0) return `${minutes}m ago`; + return 'just now'; +} +import { useEffect, useState } from 'react'; + +interface SkillQuestion { + id: string; + question: string; + answer: string; + confidence: number; + similarCount: number; + tags: string[]; + createdAt: string; + skillNodes: Array<{ + skill: { + name: string; + }; + }>; +} + +interface SkillsActivityFeedProps { + collectionId: string; + limit?: number; +} + +export function SkillsActivityFeed({ + collectionId, + limit = 10, +}: SkillsActivityFeedProps): React.ReactElement { + const [questions, setQuestions] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + async function fetchQuestions() { + try { + const response = await fetch( + `/api/skills/activity?collectionId=${collectionId}&limit=${limit}` + ); + if (!response.ok) { + throw new Error('Failed to fetch activity'); + } + const data = await response.json(); + setQuestions(data.questions || []); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to load'); + } finally { + setLoading(false); + } + } + + fetchQuestions(); + }, [collectionId, limit]); + + if (loading) { + return ( +
+ {[1, 2, 3].map((i) => ( + + + + + + + ))} +
+ ); + } + + if (error) { + return ( + + +

{error}

+ + + ); + } + + if (questions.length === 0) { + return ( + + + +

+ No questions yet. Be the first to ask! +

+ + + ); + } + + return ( +
+ {questions.map((q) => ( + + +
+ + {q.question} + +
+ = 0.7 ? 'success' : 'secondary'} + size="sm" + > + {Math.round(q.confidence * 100)}% + +
+
+ + + + + {q.answer.slice(0, 150)} + {q.answer.length > 150 ? '...' : ''} + + +
+
+ {q.skillNodes.slice(0, 2).map((sn, i) => ( + + {sn.skill.name} + + ))} + {q.skillNodes.length > 2 && ( + + +{q.skillNodes.length - 2} + + )} +
+
+ {q.similarCount > 0 && ( + + + {q.similarCount} similar + + )} + {formatRelativeTime(new Date(q.createdAt))} +
+
+ + + ))} +
+ ); +} diff --git a/apps/web/src/components/skills/SkillsSection.tsx b/apps/web/src/components/skills/SkillsSection.tsx new file mode 100644 index 0000000..fdc6a13 --- /dev/null +++ b/apps/web/src/components/skills/SkillsSection.tsx @@ -0,0 +1,158 @@ +'use client'; + +import { Button } from '@tpmjs/ui/Button/Button'; +import { CodeBlock } from '@tpmjs/ui/CodeBlock/CodeBlock'; +import { Icon } from '@tpmjs/ui/Icon/Icon'; +import Link from 'next/link'; +import { useState } from 'react'; +import { SkillsActivityFeed } from './SkillsActivityFeed'; +import { SkillsStats } from './SkillsStats'; + +interface SkillsSectionProps { + collectionId: string; + username: string; + slug: string; +} + +export function SkillsSection({ + collectionId, + username, + slug, +}: SkillsSectionProps): React.ReactElement { + const [showApiDocs, setShowApiDocs] = useState(false); + + const baseUrl = typeof window !== 'undefined' ? window.location.origin : 'https://tpmjs.com'; + const skillsUrl = `${baseUrl}/${username}/collections/${slug}/skills`; + + const apiExample = `# Ask a question about this collection's tools +curl -X POST "${skillsUrl}" \\ + -H "Content-Type: application/json" \\ + -d '{ + "question": "How do I handle errors with these tools?", + "agentName": "my-agent" + }'`; + + const responseExample = `{ + "success": true, + "data": { + "answer": "To handle errors with these tools...", + "confidence": 0.85, + "basedOn": 3, + "skillsIdentified": ["error-handling", "debugging"], + "suggestedFollowups": [ + "What are the retry patterns?", + "How do I log errors?" + ] + }, + "meta": { + "cached": false, + "questionId": "clx...", + "processingMs": 1234 + } +}`; + + return ( +
+ {/* Header */} +
+
+
+ +
+
+

Skills

+

+ Proven in the wild — not declared on paper +

+
+
+
+ + + + +
+
+ + {/* API Documentation Toggle */} + {showApiDocs && ( +
+
+

API Endpoint

+
+ POST {skillsUrl} +
+
+ +
+

Example Request

+ +
+ +
+

Example Response

+ +
+ +
+ + View full API documentation + +
+
+ )} + + {/* Main Content */} +
+ {/* Stats Column */} +
+ +
+ + {/* Activity Feed Column */} +
+
+

+ Recent Questions +

+
+ +
+
+ + {/* CTA for empty state */} +
+
+
+ +
+
+

+ Ask questions to build the skill graph +

+

+ Every question helps improve future responses for all agents. +

+
+ +
+
+
+ ); +} diff --git a/apps/web/src/components/skills/SkillsStats.tsx b/apps/web/src/components/skills/SkillsStats.tsx new file mode 100644 index 0000000..0d3e93e --- /dev/null +++ b/apps/web/src/components/skills/SkillsStats.tsx @@ -0,0 +1,155 @@ +'use client'; + +import { Badge } from '@tpmjs/ui/Badge/Badge'; +import { + Card, + CardContent, + CardHeader, + CardTitle, +} from '@tpmjs/ui/Card/Card'; +import { Icon } from '@tpmjs/ui/Icon/Icon'; +import { Skeleton } from '@tpmjs/ui/Skeleton/Skeleton'; +import { useEffect, useState } from 'react'; + +interface SkillStats { + totalQuestions: number; + totalSkills: number; + topSkills: Array<{ + name: string; + questionCount: number; + confidence: number; + }>; +} + +interface SkillsStatsProps { + collectionId: string; +} + +export function SkillsStats({ + collectionId, +}: SkillsStatsProps): React.ReactElement | null { + const [stats, setStats] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + async function fetchStats() { + try { + const response = await fetch( + `/api/skills/stats?collectionId=${collectionId}` + ); + if (!response.ok) { + throw new Error('Failed to fetch stats'); + } + const data = await response.json(); + setStats(data); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to load'); + } finally { + setLoading(false); + } + } + + fetchStats(); + }, [collectionId]); + + if (loading) { + return ( +
+ {[1, 2].map((i) => ( + + + + + + + ))} +
+ ); + } + + if (error) { + return ( + + +

{error}

+ + + ); + } + + if (!stats) { + return null; + } + + return ( +
+ {/* Summary Stats */} +
+ + +
+
+ +
+
+

{stats.totalQuestions}

+

Questions

+
+
+ + + + + +
+
+ +
+
+

{stats.totalSkills}

+

Skills

+
+
+ + +
+ + {/* Top Skills */} + {stats.topSkills.length > 0 && ( + + + + Top Skills + + + +
+ {stats.topSkills.slice(0, 5).map((skill, i) => ( +
+
+ + {skill.name} + +
+
+ {skill.questionCount} Q +
+
+
+
+
+ ))} +
+ + + )} +
+ ); +} diff --git a/apps/web/src/lib/ai/skills-embedding.ts b/apps/web/src/lib/ai/skills-embedding.ts new file mode 100644 index 0000000..35abf52 --- /dev/null +++ b/apps/web/src/lib/ai/skills-embedding.ts @@ -0,0 +1,206 @@ +/** + * Skills Embedding Service + * + * Uses OpenAI text-embedding-3-large (3072 dims) for high-quality + * semantic similarity detection in the RealSkills endpoint. + */ + +import { openai } from '@ai-sdk/openai'; +import { prisma } from '@tpmjs/db'; +import { embed } from 'ai'; + +export const SKILLS_EMBEDDING_MODEL = 'text-embedding-3-large'; +export const SKILLS_EMBEDDING_DIMS = 3072; +export const DEFAULT_SIMILARITY_THRESHOLD = 0.8; +export const CACHE_HIT_THRESHOLD = 0.95; + +/** + * Compute embedding for a question or skill description + */ +export async function embedQuestion(text: string): Promise { + const { embedding } = await embed({ + model: openai.embedding(SKILLS_EMBEDDING_MODEL), + value: text, + }); + return embedding; +} + +/** + * Compute cosine similarity between two vectors + */ +export function cosineSimilarity(a: number[], b: number[]): number { + if (a.length !== b.length) { + throw new Error( + `Vector dimension mismatch: ${a.length} vs ${b.length}` + ); + } + + let dotProduct = 0; + let normA = 0; + let normB = 0; + + for (let i = 0; i < a.length; i++) { + const aVal = a[i] ?? 0; + const bVal = b[i] ?? 0; + dotProduct += aVal * bVal; + normA += aVal * aVal; + normB += bVal * bVal; + } + + const denominator = Math.sqrt(normA) * Math.sqrt(normB); + if (denominator === 0) return 0; + + return dotProduct / denominator; +} + +export interface SimilarQuestion { + id: string; + question: string; + answer: string; + similarity: number; + createdAt: Date; +} + +/** + * Find questions similar to a given query embedding + */ +export async function findSimilarQuestions( + queryEmbedding: number[], + collectionId: string, + options: { + threshold?: number; + limit?: number; + excludeId?: string; + } = {} +): Promise { + const { + threshold = DEFAULT_SIMILARITY_THRESHOLD, + limit = 5, + excludeId, + } = options; + + // Fetch all questions for this collection + const questions = await prisma.skillQuestion.findMany({ + where: { + collectionId, + ...(excludeId && { id: { not: excludeId } }), + }, + select: { + id: true, + question: true, + answer: true, + embedding: true, + createdAt: true, + }, + }); + + // Calculate similarity scores + const similar: SimilarQuestion[] = []; + + for (const q of questions) { + const existingEmbedding = q.embedding as number[]; + if (!existingEmbedding || existingEmbedding.length === 0) continue; + + const similarity = cosineSimilarity(queryEmbedding, existingEmbedding); + + if (similarity >= threshold) { + similar.push({ + id: q.id, + question: q.question, + answer: q.answer, + similarity, + createdAt: q.createdAt, + }); + } + } + + // Sort by similarity descending and limit results + return similar + .sort((a, b) => b.similarity - a.similarity) + .slice(0, limit); +} + +/** + * Check for a cache hit (very similar question already answered) + * Returns the cached answer if similarity > 95% + */ +export async function checkCacheHit( + queryEmbedding: number[], + collectionId: string +): Promise { + const similar = await findSimilarQuestions(queryEmbedding, collectionId, { + threshold: CACHE_HIT_THRESHOLD, + limit: 1, + }); + + if (similar.length > 0 && similar[0]) { + // Increment the similar count for analytics + await prisma.skillQuestion.update({ + where: { id: similar[0].id }, + data: { similarCount: { increment: 1 } }, + }); + return similar[0]; + } + + return null; +} + +/** + * Find questions that match a text query (convenience wrapper) + */ +export async function searchQuestions( + query: string, + collectionId: string, + options: { + threshold?: number; + limit?: number; + } = {} +): Promise { + const embedding = await embedQuestion(query); + return findSimilarQuestions(embedding, collectionId, options); +} + +export interface SimilarityResult { + isCacheHit: boolean; + cachedAnswer: string | null; + similarQuestions: SimilarQuestion[]; + embedding: number[]; +} + +/** + * Full similarity check for a new question + * - Checks for cache hit (>95% similar) + * - Returns similar questions for RAG context + */ +export async function checkQuestionSimilarity( + question: string, + collectionId: string +): Promise { + // Generate embedding for the question + const embedding = await embedQuestion(question); + + // Check for cache hit first + const cacheHit = await checkCacheHit(embedding, collectionId); + if (cacheHit) { + return { + isCacheHit: true, + cachedAnswer: cacheHit.answer, + similarQuestions: [cacheHit], + embedding, + }; + } + + // Find similar questions for RAG context + const similarQuestions = await findSimilarQuestions( + embedding, + collectionId, + { threshold: DEFAULT_SIMILARITY_THRESHOLD, limit: 5 } + ); + + return { + isCacheHit: false, + cachedAnswer: null, + similarQuestions, + embedding, + }; +} diff --git a/apps/web/src/lib/ai/skills-graph-updater.ts b/apps/web/src/lib/ai/skills-graph-updater.ts new file mode 100644 index 0000000..4d5ff80 --- /dev/null +++ b/apps/web/src/lib/ai/skills-graph-updater.ts @@ -0,0 +1,389 @@ +/** + * Skills Graph Updater + * + * Handles real-time skill graph updates: + * - Infers skills from questions using embeddings + * - Creates/matches skill nodes + * - Links questions to skills and tools + * - Updates confidence scores + */ + +import { openai } from '@ai-sdk/openai'; +import type { Skill, Tool } from '@prisma/client'; +import { prisma } from '@tpmjs/db'; +import { generateObject } from 'ai'; +import { z } from 'zod'; +import { + cosineSimilarity, + embedQuestion, +} from './skills-embedding'; + +const SKILL_MATCH_THRESHOLD = 0.75; + +/** + * Generate a URL-safe slug from a skill name + */ +function slugify(text: string): string { + return text + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, '') + .slice(0, 200); +} + +/** + * Extract potential skills from a question using LLM + */ +export async function extractSkillsFromQuestion( + question: string, + tools: Tool[] +): Promise< + Array<{ + name: string; + description: string; + }> +> { + const toolNames = tools.map((t) => t.name).join(', '); + + const { object } = await generateObject({ + model: openai('gpt-4.1-mini'), + schema: z.object({ + skills: z.array( + z.object({ + name: z + .string() + .describe('Short skill name (2-5 words), e.g., "API error handling"'), + description: z + .string() + .describe('One sentence describing what this skill enables'), + }) + ), + }), + system: `You extract skills/capabilities from questions about tool collections. +Tools in this collection: ${toolNames} + +A skill represents a specific capability or use case that the tools enable. +Examples: "API error handling", "Data transformation", "File parsing", "React state management" + +Return 1-3 skills that this question relates to.`, + prompt: question, + temperature: 0.3, + }); + + return object.skills; +} + +/** + * Find existing skills that match by embedding similarity + */ +async function findMatchingSkills( + skillEmbedding: number[], + collectionId: string, + threshold: number = SKILL_MATCH_THRESHOLD +): Promise> { + const existingSkills = await prisma.skill.findMany({ + where: { collectionId }, + select: { + id: true, + name: true, + slug: true, + description: true, + embedding: true, + questionCount: true, + confidence: true, + collectionId: true, + parentSkillId: true, + createdAt: true, + updatedAt: true, + }, + }); + + const matches: Array<{ skill: Skill; similarity: number }> = []; + + for (const skill of existingSkills) { + const existingEmbedding = skill.embedding as number[]; + if (!existingEmbedding || existingEmbedding.length === 0) continue; + + const similarity = cosineSimilarity(skillEmbedding, existingEmbedding); + if (similarity >= threshold) { + matches.push({ + skill: skill as unknown as Skill, + similarity, + }); + } + } + + return matches.sort((a, b) => b.similarity - a.similarity); +} + +/** + * Get or create a skill node + */ +async function getOrCreateSkill( + collectionId: string, + name: string, + description: string +): Promise { + const slug = slugify(name); + + // Try to find existing skill by slug + const existing = await prisma.skill.findUnique({ + where: { + collectionId_slug: { + collectionId, + slug, + }, + }, + }); + + if (existing) { + return existing; + } + + // Check by embedding similarity + const embedding = await embedQuestion(`${name}: ${description}`); + const matches = await findMatchingSkills(embedding, collectionId); + + if (matches.length > 0 && matches[0]) { + // Use existing skill if close match + return matches[0].skill; + } + + // Create new skill + return prisma.skill.create({ + data: { + collectionId, + name, + slug, + description, + embedding: embedding as unknown as object, + questionCount: 0, + confidence: 0, + }, + }); +} + +export interface SkillLink { + skillId: string; + skillName: string; + relevance: number; +} + +export interface ToolLink { + toolId: string; + toolName: string; + relevance: number; +} + +export interface GraphUpdateResult { + skillLinks: SkillLink[]; + toolLinks: ToolLink[]; +} + +/** + * Extract tool mentions from question/answer text + */ +function extractToolMentions( + text: string, + tools: Tool[] +): Array<{ tool: Tool; relevance: number }> { + const mentions: Array<{ tool: Tool; relevance: number }> = []; + const lowerText = text.toLowerCase(); + + for (const tool of tools) { + // Check if tool name is mentioned + if (lowerText.includes(tool.name.toLowerCase())) { + mentions.push({ tool, relevance: 1.0 }); + } + } + + return mentions; +} + +/** + * Update the skill graph after storing a new question + * + * This: + * 1. Extracts skills from the question + * 2. Matches or creates skill nodes + * 3. Links the question to skills + * 4. Links the question to mentioned tools + * 5. Updates skill confidence scores + */ +export async function updateSkillGraph(params: { + questionId: string; + collectionId: string; + question: string; + answer: string; + tools: Tool[]; +}): Promise { + const { questionId, collectionId, question, answer, tools } = params; + + const skillLinks: SkillLink[] = []; + const toolLinks: ToolLink[] = []; + + // 1. Extract skills from the question + const extractedSkills = await extractSkillsFromQuestion(question, tools); + + // 2. Get or create skill nodes and link to question + for (const extracted of extractedSkills) { + try { + const skill = await getOrCreateSkill( + collectionId, + extracted.name, + extracted.description + ); + + // Link question to skill + await prisma.skillQuestionSkill.upsert({ + where: { + questionId_skillId: { + questionId, + skillId: skill.id, + }, + }, + create: { + questionId, + skillId: skill.id, + relevance: 1.0, + }, + update: { + relevance: 1.0, + }, + }); + + // Update skill question count + await prisma.skill.update({ + where: { id: skill.id }, + data: { + questionCount: { increment: 1 }, + // Increase confidence with more questions + confidence: { + increment: 0.05, + }, + }, + }); + + skillLinks.push({ + skillId: skill.id, + skillName: skill.name, + relevance: 1.0, + }); + } catch (error) { + // Log but don't fail - graph updates are best-effort + console.error(`Failed to link skill "${extracted.name}":`, error); + } + } + + // 3. Extract and link tool mentions + const combinedText = `${question} ${answer}`; + const toolMentions = extractToolMentions(combinedText, tools); + + for (const { tool, relevance } of toolMentions) { + try { + await prisma.skillQuestionTool.upsert({ + where: { + questionId_toolId: { + questionId, + toolId: tool.id, + }, + }, + create: { + questionId, + toolId: tool.id, + relevance, + }, + update: { + relevance, + }, + }); + + toolLinks.push({ + toolId: tool.id, + toolName: tool.name, + relevance, + }); + } catch (error) { + console.error(`Failed to link tool "${tool.name}":`, error); + } + } + + return { skillLinks, toolLinks }; +} + +/** + * Get skill summary for a collection + */ +export async function getCollectionSkillsSummary( + collectionId: string +): Promise<{ + totalQuestions: number; + totalSkills: number; + topSkills: Array<{ + name: string; + questionCount: number; + confidence: number; + }>; +}> { + const [totalQuestions, totalSkills, topSkills] = await Promise.all([ + prisma.skillQuestion.count({ where: { collectionId } }), + prisma.skill.count({ where: { collectionId } }), + prisma.skill.findMany({ + where: { collectionId }, + orderBy: { questionCount: 'desc' }, + take: 10, + select: { + name: true, + questionCount: true, + confidence: true, + }, + }), + ]); + + return { + totalQuestions, + totalSkills, + topSkills, + }; +} + +/** + * Recalculate confidence scores for all skills in a collection + * (Useful for batch updates or maintenance) + */ +export async function recalculateSkillConfidence( + collectionId: string +): Promise { + const skills = await prisma.skill.findMany({ + where: { collectionId }, + include: { + questions: { + include: { + question: true, + }, + }, + }, + }); + + for (const skill of skills) { + // Base confidence on question count (logarithmic scale) + const questionCount = skill.questions.length; + let confidence = Math.min(1.0, Math.log10(questionCount + 1) / 2); + + // Boost for recent questions + const recentQuestions = skill.questions.filter((q) => { + const daysSinceQuestion = + (Date.now() - q.question.createdAt.getTime()) / (1000 * 60 * 60 * 24); + return daysSinceQuestion < 30; + }); + + if (recentQuestions.length > 0) { + confidence += 0.1; + } + + await prisma.skill.update({ + where: { id: skill.id }, + data: { confidence: Math.min(1.0, confidence) }, + }); + } +} diff --git a/apps/web/src/lib/ai/skills-response-generator.ts b/apps/web/src/lib/ai/skills-response-generator.ts new file mode 100644 index 0000000..c562521 --- /dev/null +++ b/apps/web/src/lib/ai/skills-response-generator.ts @@ -0,0 +1,221 @@ +/** + * Skills Response Generator + * + * Uses GPT-4.1-mini to generate skill responses using RAG + * from stored questions and collection context. + */ + +import { openai } from '@ai-sdk/openai'; +import type { Collection, Tool } from '@prisma/client'; +import { generateText, streamText } from 'ai'; +import type { SimilarQuestion } from './skills-embedding'; + +const RESPONSE_MODEL = 'gpt-4.1-mini'; +const TEMPERATURE = 0.3; + +export interface CollectionContext { + collection: Collection; + tools: Array< + Tool & { + package: { npmPackageName: string }; + } + >; + skillsMarkdown?: string | null; +} + +export interface SessionMessage { + role: 'user' | 'assistant'; + content: string; +} + +export interface GenerateResponseParams { + question: string; + collectionContext: CollectionContext; + similarQuestions: SimilarQuestion[]; + sessionHistory?: SessionMessage[]; + tags?: string[]; + stream?: boolean; +} + +/** + * Build the system prompt for skill response generation + */ +function buildSystemPrompt(params: GenerateResponseParams): string { + const { collectionContext, similarQuestions, tags } = params; + const { collection, tools, skillsMarkdown } = collectionContext; + + // Build tool descriptions + const toolDescriptions = tools + .map((t) => { + return `- **${t.name}** (${t.package.npmPackageName}): ${t.description}`; + }) + .join('\n'); + + // Build similar Q&A context for RAG + const ragContext = + similarQuestions.length > 0 + ? similarQuestions + .map((q, i) => { + return `### Previous Question ${i + 1} (${Math.round(q.similarity * 100)}% similar) +**Q:** ${q.question} +**A:** ${q.answer}`; + }) + .join('\n\n') + : 'No similar questions have been asked yet.'; + + // Optional tag hints + const tagHints = + tags && tags.length > 0 + ? `\nThe user has tagged this question with: ${tags.join(', ')}` + : ''; + + return `You are a helpful assistant that answers questions about using the tools in the "${collection.name}" collection. + +## Collection Description +${collection.description || 'No description provided.'} + +## Available Tools +${toolDescriptions} + +${skillsMarkdown ? `## Skills Documentation\n${skillsMarkdown.slice(0, 4000)}` : ''} + +## Previous Related Questions & Answers (Use for context) +${ragContext} +${tagHints} + +## Response Guidelines + +1. **Be specific and practical** - Provide concrete examples and code snippets when helpful +2. **Reference the tools** - When relevant, mention which tools from the collection can help +3. **Build on previous answers** - If similar questions exist, use them as context but provide a fresh, tailored response +4. **Use markdown formatting** - Format your response with headers, code blocks, and lists as appropriate +5. **Be concise** - Get to the point quickly while being thorough +6. **Admit limitations** - If you're unsure or the collection doesn't have tools for something, say so + +Your response will be stored and used to help future users, so make it clear and reusable.`; +} + +/** + * Build the user prompt with session context + */ +function buildUserPrompt(params: GenerateResponseParams): string { + const { question, sessionHistory } = params; + + // Include session history for multi-turn conversations + if (sessionHistory && sessionHistory.length > 0) { + const historyText = sessionHistory + .map((m) => `${m.role === 'user' ? 'User' : 'Assistant'}: ${m.content}`) + .join('\n\n'); + + return `Previous conversation: +${historyText} + +Current question: ${question}`; + } + + return question; +} + +export interface GenerateResponseResult { + answer: string; + tokensUsed: number; +} + +/** + * Generate a skill response (non-streaming) + */ +export async function generateSkillResponse( + params: GenerateResponseParams +): Promise { + const systemPrompt = buildSystemPrompt(params); + const userPrompt = buildUserPrompt(params); + + const { text, usage } = await generateText({ + model: openai(RESPONSE_MODEL), + system: systemPrompt, + prompt: userPrompt, + temperature: TEMPERATURE, + }); + + return { + answer: text, + tokensUsed: usage?.totalTokens ?? 0, + }; +} + +/** + * Generate a skill response with streaming + * Returns a ReadableStream for SSE + */ +export async function generateSkillResponseStream( + params: GenerateResponseParams +): Promise { + const systemPrompt = buildSystemPrompt(params); + const userPrompt = buildUserPrompt(params); + + const result = streamText({ + model: openai(RESPONSE_MODEL), + system: systemPrompt, + prompt: userPrompt, + temperature: TEMPERATURE, + }); + + return result.textStream as unknown as ReadableStream; +} + +/** + * Generate suggested follow-up questions based on the response + */ +export async function generateFollowupSuggestions( + question: string, + answer: string, + collectionName: string +): Promise { + const { text } = await generateText({ + model: openai(RESPONSE_MODEL), + system: `You suggest follow-up questions based on a Q&A about the "${collectionName}" tool collection. +Return exactly 3 short follow-up questions, one per line. No numbering or bullets.`, + prompt: `Original question: ${question} + +Answer given: ${answer.slice(0, 1000)} + +Suggest 3 follow-up questions:`, + temperature: 0.5, + }); + + return text + .split('\n') + .map((q) => q.trim()) + .filter((q) => q.length > 0 && q.endsWith('?')) + .slice(0, 3); +} + +/** + * Infer confidence score based on RAG context quality + */ +export function calculateConfidence( + similarQuestions: SimilarQuestion[], + hasSkillsMarkdown: boolean +): number { + let confidence = 0.3; // Base confidence + + // Boost for similar questions (RAG context) + if (similarQuestions.length > 0) { + const avgSimilarity = + similarQuestions.reduce((sum, q) => sum + q.similarity, 0) / + similarQuestions.length; + confidence += avgSimilarity * 0.4; // Up to 0.4 boost + } + + // Boost for having skills documentation + if (hasSkillsMarkdown) { + confidence += 0.2; + } + + // Boost for multiple similar questions + if (similarQuestions.length >= 3) { + confidence += 0.1; + } + + return Math.min(1.0, confidence); +} diff --git a/apps/web/src/lib/ai/skills-seeder.ts b/apps/web/src/lib/ai/skills-seeder.ts new file mode 100644 index 0000000..041cb44 --- /dev/null +++ b/apps/web/src/lib/ai/skills-seeder.ts @@ -0,0 +1,331 @@ +/** + * Skills Seeder + * + * Lazy seeding of synthetic questions on first access. + * Generates questions from: + * - Existing skills.md documentation + * - Tool capabilities and descriptions + * - Common use case patterns + */ + +import { openai } from '@ai-sdk/openai'; +import type { Collection, Tool } from '@prisma/client'; +import { prisma } from '@tpmjs/db'; +import { generateObject } from 'ai'; +import { z } from 'zod'; +import { embedQuestion } from './skills-embedding'; +import { + generateSkillResponse, + type CollectionContext, +} from './skills-response-generator'; +import { updateSkillGraph } from './skills-graph-updater'; + +const SEED_BATCH_SIZE = 5; +const SEEDING_LOCK_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes + +export interface CollectionWithTools extends Collection { + tools: Array< + Tool & { + package: { npmPackageName: string }; + } + >; +} + +/** + * Generate synthetic questions from skills markdown + */ +async function generateQuestionsFromSkillsMarkdown( + skillsMarkdown: string, + collectionName: string, + count: number = 5 +): Promise { + const { object } = await generateObject({ + model: openai('gpt-4.1-mini'), + schema: z.object({ + questions: z.array(z.string()), + }), + system: `You generate realistic questions that users might ask about a tool collection. +Generate practical, specific questions based on the skills documentation. +Questions should be natural and varied - some simple, some complex.`, + prompt: `Collection: ${collectionName} + +Skills Documentation: +${skillsMarkdown.slice(0, 4000)} + +Generate ${count} questions that someone using these tools might ask:`, + temperature: 0.7, + }); + + return object.questions; +} + +/** + * Generate synthetic questions from tool descriptions + */ +async function generateQuestionsFromTools( + tools: Array, + count: number = 5 +): Promise { + const toolInfo = tools + .slice(0, 10) // Limit to first 10 tools + .map((t) => `- ${t.name}: ${t.description}`) + .join('\n'); + + const { object } = await generateObject({ + model: openai('gpt-4.1-mini'), + schema: z.object({ + questions: z.array(z.string()), + }), + system: `You generate realistic questions that users might ask when learning to use tools. +Questions should cover: +- How to use specific tools +- Error handling +- Common use cases +- Integration patterns +- Edge cases`, + prompt: `Available tools: +${toolInfo} + +Generate ${count} practical questions about using these tools:`, + temperature: 0.7, + }); + + return object.questions; +} + +/** + * Generate common use case questions + */ +async function generateCommonUseCaseQuestions( + collectionName: string, + collectionDescription: string | null, + count: number = 3 +): Promise { + const { object } = await generateObject({ + model: openai('gpt-4.1-mini'), + schema: z.object({ + questions: z.array(z.string()), + }), + system: `You generate common, beginner-friendly questions about tool collections. +Focus on: +- Getting started +- Best practices +- Common pitfalls +- When to use vs alternatives`, + prompt: `Collection: ${collectionName} +Description: ${collectionDescription || 'A collection of tools'} + +Generate ${count} common questions someone new might ask:`, + temperature: 0.7, + }); + + return object.questions; +} + +/** + * Check if seeding is already in progress (with timeout) + */ +async function isSeeding(collectionId: string): Promise { + const collection = await prisma.collection.findUnique({ + where: { id: collectionId }, + select: { skillsSeedingAt: true }, + }); + + if (!collection?.skillsSeedingAt) return false; + + // Check if seeding has timed out + const elapsed = Date.now() - collection.skillsSeedingAt.getTime(); + if (elapsed > SEEDING_LOCK_TIMEOUT_MS) { + // Clear stale lock + await prisma.collection.update({ + where: { id: collectionId }, + data: { skillsSeedingAt: null }, + }); + return false; + } + + return true; +} + +/** + * Seed a collection with synthetic questions + * Returns true if seeding was performed, false if skipped + */ +export async function seedCollectionSkills( + collection: CollectionWithTools +): Promise<{ + seeded: boolean; + questionsCreated: number; + reason?: string; +}> { + // Check if already seeded + if (collection.skillsSeeded) { + return { seeded: false, questionsCreated: 0, reason: 'Already seeded' }; + } + + // Check if seeding is in progress + if (await isSeeding(collection.id)) { + return { seeded: false, questionsCreated: 0, reason: 'Seeding in progress' }; + } + + // Acquire seeding lock + await prisma.collection.update({ + where: { id: collection.id }, + data: { skillsSeedingAt: new Date() }, + }); + + try { + const allQuestions: string[] = []; + + // 1. Generate from skills.md if available + if (collection.skillsMarkdown) { + const skillsQuestions = await generateQuestionsFromSkillsMarkdown( + collection.skillsMarkdown, + collection.name, + 5 + ); + allQuestions.push(...skillsQuestions); + } + + // 2. Generate from tool descriptions + if (collection.tools.length > 0) { + const toolQuestions = await generateQuestionsFromTools( + collection.tools, + 5 + ); + allQuestions.push(...toolQuestions); + } + + // 3. Generate common use case questions + const useCaseQuestions = await generateCommonUseCaseQuestions( + collection.name, + collection.description, + 3 + ); + allQuestions.push(...useCaseQuestions); + + // Deduplicate questions + const uniqueQuestions = [...new Set(allQuestions)]; + + // 4. Process questions in batches + let questionsCreated = 0; + const collectionContext: CollectionContext = { + collection, + tools: collection.tools, + skillsMarkdown: collection.skillsMarkdown, + }; + + for (let i = 0; i < uniqueQuestions.length; i += SEED_BATCH_SIZE) { + const batch = uniqueQuestions.slice(i, i + SEED_BATCH_SIZE); + + for (const question of batch) { + try { + // Generate embedding + const embedding = await embedQuestion(question); + + // Generate answer + const { answer, tokensUsed } = await generateSkillResponse({ + question, + collectionContext, + similarQuestions: [], // No similar questions for seed + stream: false, + }); + + // Store the question + const stored = await prisma.skillQuestion.create({ + data: { + collectionId: collection.id, + question, + embedding: embedding as unknown as object, + answer, + answerTokens: tokensUsed, + agentName: 'seed-bot', + confidence: 0.5, // Medium confidence for synthetic + tags: ['synthetic', 'seed'], + }, + }); + + // Update skill graph (best-effort) + try { + await updateSkillGraph({ + questionId: stored.id, + collectionId: collection.id, + question, + answer, + tools: collection.tools, + }); + } catch { + // Don't fail seeding if graph update fails + } + + questionsCreated++; + } catch (error) { + console.error(`Failed to seed question: "${question}"`, error); + // Continue with other questions + } + } + } + + // Mark as seeded + await prisma.collection.update({ + where: { id: collection.id }, + data: { + skillsSeeded: true, + skillsSeedingAt: null, + }, + }); + + return { seeded: true, questionsCreated }; + } catch (error) { + // Clear seeding lock on error + await prisma.collection.update({ + where: { id: collection.id }, + data: { skillsSeedingAt: null }, + }); + throw error; + } +} + +/** + * Check seeding status for a collection + */ +export async function getSeedingStatus(collectionId: string): Promise<{ + isSeeded: boolean; + isSeeding: boolean; + questionCount: number; +}> { + const [collection, questionCount] = await Promise.all([ + prisma.collection.findUnique({ + where: { id: collectionId }, + select: { skillsSeeded: true, skillsSeedingAt: true }, + }), + prisma.skillQuestion.count({ where: { collectionId } }), + ]); + + if (!collection) { + return { isSeeded: false, isSeeding: false, questionCount: 0 }; + } + + const isCurrentlySeeding = + collection.skillsSeedingAt && + Date.now() - collection.skillsSeedingAt.getTime() < SEEDING_LOCK_TIMEOUT_MS; + + return { + isSeeded: collection.skillsSeeded, + isSeeding: !!isCurrentlySeeding, + questionCount, + }; +} + +/** + * Reset seeding status (for manual re-seeding) + */ +export async function resetSeedingStatus(collectionId: string): Promise { + await prisma.collection.update({ + where: { id: collectionId }, + data: { + skillsSeeded: false, + skillsSeedingAt: null, + }, + }); +} diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index e9ad609..6c9eda0 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -101,14 +101,15 @@ model Tool { reviewCount Int @default(0) @map("review_count") // Relations - simulations Simulation[] - healthChecks HealthCheck[] - collections CollectionTool[] - agents AgentTool[] - likes ToolLike[] - ratings ToolRating[] - reviews ToolReview[] - skillsCache ToolSkillsCache? + simulations Simulation[] + healthChecks HealthCheck[] + collections CollectionTool[] + agents AgentTool[] + likes ToolLike[] + ratings ToolRating[] + reviews ToolReview[] + skillsCache ToolSkillsCache? + skillQuestions SkillQuestionTool[] @@unique([packageId, name]) @@index([qualityScore]) @@ -463,6 +464,10 @@ model Collection { skillsMarkdown String? @map("skills_markdown") @db.Text skillsGeneratedAt DateTime? @map("skills_generated_at") + // RealSkills lazy seeding state + skillsSeeded Boolean @default(false) @map("skills_seeded") + skillsSeedingAt DateTime? @map("skills_seeding_at") + // Timestamps createdAt DateTime @default(now()) @map("created_at") updatedAt DateTime @updatedAt @map("updated_at") @@ -474,6 +479,9 @@ model Collection { bridgeTools CollectionBridgeTool[] scenarios Scenario[] skillsGenerationJobs SkillsGenerationJob[] + skillQuestions SkillQuestion[] + skills Skill[] + skillSessions SkillSession[] // Unique constraint: user can't have duplicate collection slugs @@unique([userId, slug]) @@ -1615,3 +1623,132 @@ model OmegaUserSettings { @@map("omega_user_settings") } + +// ============================================================================ +// RealSkills Models (Living Skills Endpoint - Question-Driven Evolution) +// ============================================================================ + +/// SkillQuestion - Agent questions stored with embeddings for RAG +model SkillQuestion { + id String @id @default(cuid()) + + // Collection relationship + collectionId String @map("collection_id") + collection Collection @relation(fields: [collectionId], references: [id], onDelete: Cascade) + + // Question content + question String @db.Text + embedding Json @db.JsonB // Array of floats (3072 dims for text-embedding-3-large) + + // Response + answer String @db.Text + answerTokens Int @default(0) @map("answer_tokens") + + // Agent identity (anonymized in UI) + agentHash String? @map("agent_hash") @db.VarChar(64) // Hash of IP + user-agent for deduplication + agentName String? @map("agent_name") @db.VarChar(100) // Self-reported agent name + sessionId String? @map("session_id") // Optional session for multi-turn + + // Metadata + tags String[] @default([]) @db.Text + confidence Float @default(0) + similarCount Int @default(0) @map("similar_count") // Times similar question asked + + // Graph relationships + skillNodes SkillQuestionSkill[] + toolNodes SkillQuestionTool[] + + // Timestamps + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @updatedAt @map("updated_at") + + @@index([collectionId]) + @@index([sessionId]) + @@index([createdAt]) + @@map("skill_questions") +} + +/// Skill - Emergent skill nodes identified from question patterns +model Skill { + id String @id @default(cuid()) + + // Collection relationship + collectionId String @map("collection_id") + collection Collection @relation(fields: [collectionId], references: [id], onDelete: Cascade) + + // Skill identity + name String @db.VarChar(200) // e.g., "React useEffect debugging" + slug String @db.VarChar(200) // URL-safe version + description String @db.Text + embedding Json @db.JsonB // Array of floats (3072 dims for text-embedding-3-large) + + // Stats + questionCount Int @default(0) @map("question_count") + confidence Float @default(0) // Aggregate confidence + + // Graph relationships + questions SkillQuestionSkill[] + parentSkill Skill? @relation("SkillHierarchy", fields: [parentSkillId], references: [id]) + parentSkillId String? @map("parent_skill_id") + childSkills Skill[] @relation("SkillHierarchy") + + // Timestamps + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @updatedAt @map("updated_at") + + @@unique([collectionId, slug]) + @@index([collectionId]) + @@index([questionCount]) + @@map("skills") +} + +/// SkillQuestionSkill - Junction table linking questions to skills +model SkillQuestionSkill { + questionId String @map("question_id") + question SkillQuestion @relation(fields: [questionId], references: [id], onDelete: Cascade) + skillId String @map("skill_id") + skill Skill @relation(fields: [skillId], references: [id], onDelete: Cascade) + relevance Float @default(1.0) + + @@id([questionId, skillId]) + @@index([questionId]) + @@index([skillId]) + @@map("skill_question_skills") +} + +/// SkillQuestionTool - Junction table linking questions to tools mentioned +model SkillQuestionTool { + questionId String @map("question_id") + question SkillQuestion @relation(fields: [questionId], references: [id], onDelete: Cascade) + toolId String @map("tool_id") + tool Tool @relation(fields: [toolId], references: [id], onDelete: Cascade) + relevance Float @default(1.0) + + @@id([questionId, toolId]) + @@index([questionId]) + @@index([toolId]) + @@map("skill_question_tools") +} + +/// SkillSession - Optional multi-turn conversation sessions +model SkillSession { + id String @id @default(cuid()) + + // Collection relationship + collectionId String @map("collection_id") + collection Collection @relation(fields: [collectionId], references: [id], onDelete: Cascade) + + // Session state + context Json @default("[]") @db.JsonB // Message history + agentHash String? @map("agent_hash") @db.VarChar(64) + agentName String? @map("agent_name") @db.VarChar(100) + + // Timestamps + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @updatedAt @map("updated_at") + expiresAt DateTime @map("expires_at") // Auto-expire sessions + + @@index([collectionId]) + @@index([expiresAt]) + @@map("skill_sessions") +}