feat: add dynamic tool loading system with Railway executor

Implements a complete dynamic tool loading system that allows the playground to discover and load tools from the TPMJS registry at runtime.

**Architecture:**
- Search tool package (@tpmjs/search-registry) - Searches registry for tools
- Search API endpoint (/api/tools/search) - Text-based search with scoring
- Pre-flight tool loading - Automatically searches and loads tools on every message
- Railway executor service (Deno) - Loads tools from esm.sh via HTTP imports
- Dynamic tool loader - Calls Railway to load and execute tools remotely

**Key Components:**

1. Railway Executor (apps/railway-executor/)
   - Deno-based service that natively supports HTTP imports
   - Endpoints: /load-and-describe, /execute-tool, /cache/stats, /cache/clear
   - Deploys to Railway with deno run --allow-net --allow-env server.ts

2. Search Tool Package (packages/tools/search-registry/)
   - AI SDK v6 tool for searching TPMJS registry
   - Uses jsonSchema + inputSchema pattern
   - Searches /api/tools/search endpoint

3. Search API (apps/web/src/app/api/tools/search/)
   - Text-based search with composite scoring
   - Scores: text relevance + quality boost + download boost
   - Returns tool metadata with importUrl for dynamic loading

4. Dynamic Tool Loader (apps/playground/src/lib/dynamic-tool-loader.ts)
   - Calls Railway service to load tools from esm.sh
   - Creates tool wrappers that execute remotely
   - Process-level module cache + per-conversation tracking

5. Pre-flight Loading (apps/playground/src/app/api/chat/route.ts)
   - Automatically searches for tools on every user message
   - Loads top 5 matching tools before agent processes request
   - Merges with static tools for seamless experience

**Technical Decisions:**
- Deno over Node.js: Native HTTP import support without flags
- Remote execution: Tools run in Railway sandbox, not Vercel
- Pre-flight loading: Better UX than two-turn search pattern
- Text search: BM25 had dependency issues, simple scoring works well

**Environment Variables:**
- RAILWAY_SERVICE_URL: https://endearing-commitment-production.up.railway.app

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

apps/playground/next-env.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="next" />
 /// <reference types="next/image-types/global" />
-import "./.next/types/routes.d.ts";
+import "./.next/dev/types/routes.d.ts";
 
 // NOTE: This file should not be edited
 // see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

apps/playground/next.config.ts

@@ -3,6 +3,9 @@ import type { NextConfig } from 'next';
 const nextConfig: NextConfig = {
   transpilePackages: ['@tpmjs/ui', '@tpmjs/utils', '@tpmjs/types', '@tpmjs/env'],
   reactStrictMode: true,
+  experimental: {
+    urlImports: ['https://esm.sh/', 'https://cdn.jsdelivr.net/npm/'],
+  },
 };
 
 export default nextConfig;

apps/playground/package.json

@@ -15,11 +15,13 @@
     "@ai-sdk/react": "^2.0.106",
     "@tpmjs/env": "workspace:*",
     "@tpmjs/hello": "workspace:*",
+    "@tpmjs/search-registry": "workspace:*",
     "@tpmjs/types": "workspace:*",
     "@tpmjs/ui": "workspace:*",
     "@tpmjs/utils": "workspace:*",
     "ai": "6.0.0-beta.124",
     "firecrawl-aisdk": "^0.7.2",
+    "nanoid": "^5.1.6",
     "next": "^16.0.4",
     "next-themes": "^0.4.6",
     "openai": "^6.9.1",

apps/playground/src/app/api/chat/route.ts

@@ -1,8 +1,11 @@
 import { createOpenAI } from '@ai-sdk/openai';
+import { searchTpmjsToolsTool } from '@tpmjs/search-registry';
 import { type UIMessage, convertToModelMessages, stepCountIs, streamText } from 'ai';
 import type { NextRequest } from 'next/server';
+import { NextResponse } from 'next/server';
 import { env } from '~/env';
-import { loadAllTools } from '~/lib/tool-loader';
+import { addConversationTools, loadToolsBatch } from '~/lib/dynamic-tool-loader';
+import { loadAllTools, sanitizeToolName } from '~/lib/tool-loader';
 
 export const runtime = 'nodejs';
 export const dynamic = 'force-dynamic';
@@ -13,6 +16,10 @@ const openai = createOpenAI({
   apiKey: env.OPENAI_API_KEY,
 });
 
+// Add conversation state tracking (in-memory for MVP)
+// biome-ignore lint/suspicious/noExplicitAny: Tool types from AI SDK are complex
+const conversationStates = new Map<string, { loadedTools: Record<string, any> }>();
+
 /**
  * POST /api/chat
  * Chat with AI agent that can execute TPMJS tools
@@ -20,17 +27,117 @@ const openai = createOpenAI({
 export async function POST(request: NextRequest) {
   try {
     const body = await request.json();
-    console.log('Request body:', JSON.stringify(body, null, 2));
+    console.log('📥 Request body:', JSON.stringify(body, null, 2));
 
     const messages: UIMessage[] = body.messages || [];
+    const conversationId: string = body.conversationId || 'default';
 
-    // Load all available TPMJS tools
-    const tools = await loadAllTools();
+    console.log(`🔑 Conversation ID: ${conversationId}`);
 
-    // Create system prompt listing available tools
-    const toolsList = Object.keys(tools)
+    // Get or create conversation state
+    if (!conversationStates.has(conversationId)) {
+      console.log('✨ Creating new conversation state');
+      conversationStates.set(conversationId, { loadedTools: {} });
+    }
+    const state = conversationStates.get(conversationId)!;
+    console.log(
+      `📊 Current loaded tools in conversation: ${Object.keys(state.loadedTools).length}`
+    );
+
+    // 1. Load static tools + search tool
+    const staticTools = await loadAllTools();
+    console.log(`🔧 Loaded ${Object.keys(staticTools).length} static tools`);
+
+    staticTools.searchTpmjsTools = searchTpmjsToolsTool;
+    console.log('✅ Added searchTpmjsTools to static tools');
+
+    // Debug: Check the search tool structure
+    console.log('🔍 Search tool structure:', {
+      description: searchTpmjsToolsTool.description,
+      parameters: typeof searchTpmjsToolsTool.parameters,
+      execute: typeof searchTpmjsToolsTool.execute,
+    });
+
+    // 2. Extract user query from last message for tool search
+    const lastMessage = messages[messages.length - 1];
+    let userQuery = '';
+    if (lastMessage?.role === 'user') {
+      // Extract text from message parts
+      const parts = (lastMessage as any).parts || [];
+      for (const part of parts) {
+        if (part.type === 'text') {
+          userQuery = part.text;
+          break;
+        }
+      }
+    }
+
+    console.log(`💬 User query: "${userQuery}"`);
+
+    // 3. Automatically search for relevant tools based on the user's message
+    if (userQuery && userQuery.trim().length > 0) {
+      console.log('🔎 Searching for relevant tools...');
+
+      try {
+        const searchResult = await searchTpmjsToolsTool.execute(
+          {
+            query: userQuery,
+            limit: 5, // Get top 5 relevant tools
+          },
+          {} as any
+        );
+
+        console.log(`📦 Found ${searchResult.matchCount} matching tools`);
+
+        if (searchResult.tools && searchResult.tools.length > 0) {
+          console.log(
+            `🔧 Tools found:`,
+            searchResult.tools.map((t: any) => `${t.packageName}/${t.exportName}`)
+          );
+
+          // Dynamically load tools from esm.sh
+          console.log(`📥 Loading ${searchResult.tools.length} tools dynamically...`);
+
+          const toolsToLoad = searchResult.tools.map((meta: any) => ({
+            packageName: meta.packageName,
+            exportName: meta.exportName,
+            version: meta.version,
+            importUrl: meta.importUrl,
+          }));
+
+          try {
+            const loadedTools = await loadToolsBatch(toolsToLoad);
+            console.log(`✅ Successfully loaded ${Object.keys(loadedTools).length} tools`);
+
+            // Add sanitized tools to conversation state
+            for (const [key, tool] of Object.entries(loadedTools)) {
+              const [pkg, exp] = key.split('::');
+              const sanitizedKey = sanitizeToolName(`${pkg}-${exp}`);
+              state.loadedTools[sanitizedKey] = tool;
+              console.log(`✅ Added to conversation: ${sanitizedKey}`);
+            }
+
+            // Track for this conversation
+            addConversationTools(conversationId, Object.keys(state.loadedTools));
+          } catch (error) {
+            console.error('❌ Error loading tools:', error);
+          }
+        } else {
+          console.log('ℹ️  No matching tools found for this query');
+        }
+      } catch (error) {
+        console.error('❌ Error searching for tools:', error);
+      }
+    }
+
+    // 4. Merge with conversation's dynamically loaded tools
+    // biome-ignore lint/suspicious/noExplicitAny: Tool types from AI SDK are complex
+    const allTools: Record<string, any> = { ...staticTools, ...state.loadedTools };
+
+    // 5. Build system prompt with available tools
+    const toolsList = Object.keys(allTools)
       .map((name) => {
-        const tool = tools[name] as { description?: string } | undefined;
+        const tool = allTools[name] as { description?: string } | undefined;
         return `- ${name}: ${tool?.description || 'No description'}`;
       })
       .join('\n');
@@ -42,12 +149,12 @@ ${toolsList}
 
 When you use a tool, you MUST always follow up with a natural language answer to the user summarizing the result.`;
 
-    // Stream the response with multi-step tool usage enabled
+    // 6. Stream response with all available tools
     const result = streamText({
       model: openai('gpt-4o-mini'),
       system,
       messages: convertToModelMessages(messages),
-      tools,
+      tools: allTools,
       stopWhen: stepCountIs(5), // Allow model to call tools AND generate text response
     });
 

apps/playground/src/hooks/useChat.ts

@@ -2,15 +2,42 @@
 
 import { useChat as useAISDKChat } from '@ai-sdk/react';
 import { DefaultChatTransport } from 'ai';
+import { useState } from 'react';
+import { nanoid } from 'nanoid';
 
 /**
  * Custom chat hook that wraps the official @ai-sdk/react useChat
  * Handles SSE streaming with tool calls and UI message protocol
+ * Includes conversation ID tracking for dynamic tool loading
  */
 export function useChat() {
-  return useAISDKChat({
+  // Generate stable conversation ID for session
+  const [conversationId] = useState(() => nanoid());
+
+  const chat = useAISDKChat({
     transport: new DefaultChatTransport({
       api: '/api/chat',
     }),
+    body: {
+      conversationId, // Pass conversation ID to API
+    },
+    onResponse: (response: Response) => {
+      // Handle "tools loaded" response
+      if (response.headers.get('content-type')?.includes('application/json')) {
+        response.json().then((data: any) => {
+          if (data.type === 'tools_loaded') {
+            console.log('🔧 Tools loaded:', data.loaded);
+            // Optionally show toast/notification
+          }
+        }).catch(() => {
+          // Ignore JSON parsing errors for non-JSON responses
+        });
+      }
+    },
   });
+
+  return {
+    ...chat,
+    conversationId,
+  };
 }

apps/playground/src/lib/dynamic-tool-loader.ts

@@ -0,0 +1,182 @@
+// Cache for tool wrappers (process-level)
+const moduleCache = new Map<string, any>();
+
+// Cache for per-conversation active tools
+const conversationTools = new Map<string, Set<string>>();
+
+// Railway service URL
+const RAILWAY_SERVICE_URL =
+  process.env.RAILWAY_SERVICE_URL || process.env.SANDBOX_EXECUTOR_URL || 'http://localhost:3001';
+
+/**
+ * Generate cache key for a tool
+ */
+function getCacheKey(packageName: string, exportName: string): string {
+  return `${packageName}::${exportName}`;
+}
+
+/**
+ * Dynamically load a tool via Railway service
+ * Railway service runs with --experimental-network-imports and can import from esm.sh
+ */
+// biome-ignore lint/suspicious/noExplicitAny: Tool types from AI SDK are complex
+export async function loadToolDynamically(
+  packageName: string,
+  exportName: string,
+  version: string,
+  importUrl?: string
+): Promise<any | null> {
+  const cacheKey = getCacheKey(packageName, exportName);
+
+  // Check cache first
+  if (moduleCache.has(cacheKey)) {
+    console.log(`✅ Cache hit: ${cacheKey}`);
+    return moduleCache.get(cacheKey);
+  }
+
+  try {
+    console.log(`📦 Loading from Railway: ${packageName}/${exportName}`);
+    console.log(`🔗 Railway URL: ${RAILWAY_SERVICE_URL}`);
+
+    // Call Railway service to load and describe tool
+    const response = await fetch(`${RAILWAY_SERVICE_URL}/load-and-describe`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        packageName,
+        exportName,
+        version,
+        importUrl: importUrl || `https://esm.sh/${packageName}@${version}`,
+      }),
+    });
+
+    if (!response.ok) {
+      const errorText = await response.text();
+      console.error(`❌ Railway service error (${response.status}): ${errorText}`);
+      return null;
+    }
+
+    const data = await response.json();
+
+    if (!data.success) {
+      console.error(`❌ Failed to load tool: ${data.error}`);
+      return null;
+    }
+
+    console.log(`✅ Tool loaded from Railway: ${cacheKey}`);
+    console.log(`📋 Description: ${data.tool.description}`);
+
+    // Create a tool wrapper that executes remotely
+    const tool = {
+      description: data.tool.description,
+      inputSchema: data.tool.inputSchema,
+      // biome-ignore lint/suspicious/noExplicitAny: Tool params are dynamic
+      execute: async (params: any) => {
+        console.log(`🚀 Executing ${packageName}/${exportName} remotely with params:`, params);
+
+        const execResponse = await fetch(`${RAILWAY_SERVICE_URL}/execute-tool`, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({
+            packageName,
+            exportName,
+            version,
+            importUrl: importUrl || `https://esm.sh/${packageName}@${version}`,
+            params,
+          }),
+        });
+
+        const result = await execResponse.json();
+
+        if (!result.success) {
+          console.error(`❌ Tool execution failed: ${result.error}`);
+          throw new Error(result.error || 'Tool execution failed');
+        }
+
+        console.log(`✅ Tool executed successfully in ${result.executionTimeMs}ms`);
+        return result.output;
+      },
+    };
+
+    // Cache the wrapper
+    moduleCache.set(cacheKey, tool);
+    console.log(`✅ Cached tool wrapper: ${cacheKey}`);
+
+    return tool;
+  } catch (error) {
+    console.error(`❌ Failed to load ${packageName}#${exportName}:`, error);
+    console.error(`   Stack:`, error instanceof Error ? error.stack : 'No stack trace');
+    return null;
+  }
+}
+
+/**
+ * Load multiple tools in parallel
+ */
+// biome-ignore lint/suspicious/noExplicitAny: Tool types from AI SDK are complex
+export async function loadToolsBatch(
+  toolMetadata: Array<{
+    packageName: string;
+    exportName: string;
+    version: string;
+    importUrl?: string;
+  }>
+): Promise<Record<string, any>> {
+  const promises = toolMetadata.map((meta) =>
+    loadToolDynamically(meta.packageName, meta.exportName, meta.version, meta.importUrl).then(
+      (tool) => ({
+        key: getCacheKey(meta.packageName, meta.exportName),
+        tool,
+      })
+    )
+  );
+
+  const results = await Promise.all(promises);
+
+  // biome-ignore lint/suspicious/noExplicitAny: Tool types from AI SDK are complex
+  const tools: Record<string, any> = {};
+  for (const { key, tool } of results) {
+    if (tool) {
+      tools[key] = tool;
+    }
+  }
+
+  return tools;
+}
+
+/**
+ * Track tools for a conversation
+ */
+export function addConversationTools(conversationId: string, toolKeys: string[]): void {
+  if (!conversationTools.has(conversationId)) {
+    conversationTools.set(conversationId, new Set());
+  }
+  const tools = conversationTools.get(conversationId)!;
+  for (const key of toolKeys) {
+    tools.add(key);
+  }
+}
+
+/**
+ * Get all tools for a conversation
+ */
+export function getConversationTools(conversationId: string): string[] {
+  return Array.from(conversationTools.get(conversationId) || []);
+}
+
+/**
+ * Clear conversation tools (on session end)
+ */
+export function clearConversationTools(conversationId: string): void {
+  conversationTools.delete(conversationId);
+}
+
+/**
+ * Get cache statistics
+ */
+export function getCacheStats() {
+  return {
+    moduleCacheSize: moduleCache.size,
+    conversationCount: conversationTools.size,
+  };
+}

apps/playground/src/lib/tool-loader.ts

@@ -1,6 +1,5 @@
 // Static imports for tools (required for Next.js/webpack)
 import { helloNameTool, helloWorldTool } from '@tpmjs/hello';
-import { crawlTool, scrapeTool, searchTool } from 'firecrawl-aisdk';
 
 /**
  * Tool registry mapping package names + export names to actual tool functions
@@ -11,11 +10,6 @@ const TOOL_REGISTRY: Record<string, Record<string, any>> = {
     helloWorldTool,
     helloNameTool,
   },
-  'firecrawl-aisdk': {
-    scrapeTool,
-    crawlTool,
-    searchTool,
-  },
 };
 
 /**
@@ -72,7 +66,7 @@ function isCoreTool(obj: unknown): obj is Record<string, any> {
  * Sanitize tool name to match OpenAI's requirements
  * Pattern: ^[a-zA-Z0-9_-]+$ (only letters, numbers, underscores, hyphens)
  */
-function sanitizeToolName(name: string): string {
+export function sanitizeToolName(name: string): string {
   return name
     .replace(/@/g, '') // Remove @ symbols
     .replace(/\//g, '_') // Replace / with _