feat: add dynamic tool loading system with Railway executor

Implements a complete dynamic tool loading system that allows the playground to discover and load tools from the TPMJS registry at runtime.

**Architecture:**
- Search tool package (@tpmjs/search-registry) - Searches registry for tools
- Search API endpoint (/api/tools/search) - Text-based search with scoring
- Pre-flight tool loading - Automatically searches and loads tools on every message
- Railway executor service (Deno) - Loads tools from esm.sh via HTTP imports
- Dynamic tool loader - Calls Railway to load and execute tools remotely

**Key Components:**

1. Railway Executor (apps/railway-executor/)
   - Deno-based service that natively supports HTTP imports
   - Endpoints: /load-and-describe, /execute-tool, /cache/stats, /cache/clear
   - Deploys to Railway with deno run --allow-net --allow-env server.ts

2. Search Tool Package (packages/tools/search-registry/)
   - AI SDK v6 tool for searching TPMJS registry
   - Uses jsonSchema + inputSchema pattern
   - Searches /api/tools/search endpoint

3. Search API (apps/web/src/app/api/tools/search/)
   - Text-based search with composite scoring
   - Scores: text relevance + quality boost + download boost
   - Returns tool metadata with importUrl for dynamic loading

4. Dynamic Tool Loader (apps/playground/src/lib/dynamic-tool-loader.ts)
   - Calls Railway service to load tools from esm.sh
   - Creates tool wrappers that execute remotely
   - Process-level module cache + per-conversation tracking

5. Pre-flight Loading (apps/playground/src/app/api/chat/route.ts)
   - Automatically searches for tools on every user message
   - Loads top 5 matching tools before agent processes request
   - Merges with static tools for seamless experience

**Technical Decisions:**
- Deno over Node.js: Native HTTP import support without flags
- Remote execution: Tools run in Railway sandbox, not Vercel
- Pre-flight loading: Better UX than two-turn search pattern
- Text search: BM25 had dependency issues, simple scoring works well

**Environment Variables:**
- RAILWAY_SERVICE_URL: https://endearing-commitment-production.up.railway.app

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Ajax Davis 2025-12-04 08:25:53 +10:00
parent 0612eac5e2
commit 2158ee6dfd
28 changed files with 37043 additions and 27 deletions

View file

@ -0,0 +1,124 @@
import { prisma } from '@tpmjs/db';
import { NextResponse } from 'next/server';
export const runtime = 'nodejs';
export const dynamic = 'force-dynamic';
export const maxDuration = 60;
// Simple text-based search scoring (fallback until BM25 is fixed)
function calculateTextScore(query: string, document: string): number {
const queryTokens = query.toLowerCase().split(/\s+/);
const docLower = document.toLowerCase();
let score = 0;
for (const token of queryTokens) {
// Exact match in document
if (docLower.includes(token)) {
score += 1;
}
// Boost if token appears in beginning (likely more relevant)
if (docLower.startsWith(token)) {
score += 0.5;
}
}
return score;
}
export async function GET(request: Request) {
console.log('🔎 [SEARCH API] Request received');
try {
const { searchParams } = new URL(request.url);
const query = searchParams.get('q') || '';
const category = searchParams.get('category');
const limit = Math.min(Number.parseInt(searchParams.get('limit') || '10'), 50);
console.log(`🔎 [SEARCH API] Query: "${query}", Category: ${category}, Limit: ${limit}`);
// Fetch all tools with package info
const tools = await prisma.tool.findMany({
include: { package: true },
where: category
? {
package: { category },
}
: undefined,
});
console.log(`📊 [SEARCH API] Found ${tools.length} tools in database`);
// Build searchable documents and calculate scores
const scoredResults = tools.map((tool) => {
const document = [
tool.description,
tool.exportName,
tool.package.npmPackageName,
tool.package.npmDescription || '',
...(tool.package.npmKeywords || []),
].join(' ');
const textScore = calculateTextScore(query, document);
const qualityBoost = Number.parseFloat(tool.qualityScore || '0') * 0.5;
const downloadBoost = Math.log10((tool.package.npmDownloadsLastMonth || 0) + 1) * 0.1;
const finalScore = textScore + qualityBoost + downloadBoost;
return { tool, score: finalScore };
});
// Sort by score and take top N
const topResults = scoredResults
.filter(({ score }) => score > 0) // Only include results with matches
.sort((a, b) => b.score - a.score)
.slice(0, limit + 1);
const hasMore = topResults.length > limit;
const results = hasMore ? topResults.slice(0, limit) : topResults;
console.log(`✅ [SEARCH API] Returning ${results.length} results (hasMore: ${hasMore})`);
// Format response to match existing /api/tools structure
return NextResponse.json({
success: true,
query,
filters: { category },
results: {
total: scoredResults.filter(({ score }) => score > 0).length,
returned: results.length,
tools: results.map(({ tool }) => ({
id: tool.id,
exportName: tool.exportName,
description: tool.description,
qualityScore: tool.qualityScore,
package: {
npmPackageName: tool.package.npmPackageName,
npmVersion: tool.package.npmVersion,
category: tool.package.category,
frameworks: tool.package.frameworks,
env: tool.package.env,
npmRepository: tool.package.npmRepository,
isOfficial: tool.package.isOfficial,
npmDownloadsLastMonth: tool.package.npmDownloadsLastMonth,
},
importUrl: `https://esm.sh/${tool.package.npmPackageName}@${tool.package.npmVersion}`,
cdnUrl: `https://cdn.jsdelivr.net/npm/${tool.package.npmPackageName}@${tool.package.npmVersion}/+esm`,
})),
},
pagination: {
limit,
hasMore,
},
});
} catch (error) {
console.error('❌ [SEARCH API] Error:', error);
return NextResponse.json(
{
success: false,
error: error instanceof Error ? error.message : 'Search failed',
},
{ status: 500 }
);
}
}