diff --git a/apps/web/next-env.d.ts b/apps/web/next-env.d.ts
index 9edff1c..c4b7818 100644
--- a/apps/web/next-env.d.ts
+++ b/apps/web/next-env.d.ts
@@ -1,6 +1,6 @@
///
///
-import "./.next/types/routes.d.ts";
+import "./.next/dev/types/routes.d.ts";
// NOTE: This file should not be edited
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.
diff --git a/docs/SCALING_TO_1M_TOOLS.md b/docs/SCALING_TO_1M_TOOLS.md
new file mode 100644
index 0000000..2382480
--- /dev/null
+++ b/docs/SCALING_TO_1M_TOOLS.md
@@ -0,0 +1,1193 @@
+# Scaling TPMJS & Blocks to 1 Million Tools
+
+## Executive Summary
+
+The current architecture uses a monolithic `blocks.yml` file (~160KB, 4339 lines, ~100 tools) to define tool specifications, domain models, and validation rules. This approach works well for development and small-scale operations but will not scale to 10K or 1M tools.
+
+This proposal outlines a phased migration from file-based to database-first architecture, with considerations for search, validation, governance, and ecosystem growth.
+
+---
+
+## Current State Analysis
+
+### What We Have Today
+
+**1. blocks.yml Structure (160KB)**
+```yaml
+name: "tpmjs-official-tools"
+root: "."
+
+philosophy:
+ - "Every tool MUST be a working, production-ready implementation"
+ - "Tools use AI SDK v6 tool() + jsonSchema() pattern exclusively"
+ # ... 8 more principles
+
+domain:
+ entities:
+ url: { fields: [...], description: "..." }
+ webpage: { fields: [...], description: "..." }
+ # ... ~50 entities
+
+ signals:
+ # ... ~20 signals
+
+ measures:
+ # ... ~15 measures
+
+blocks:
+ tool.name:
+ description: "..."
+ path: "tool-path"
+ domain_rules: [...]
+ inputs: [...]
+ outputs: [...]
+ # ... ~100 tools
+```
+
+**2. Database Schema (PostgreSQL)**
+- `packages` - NPM package metadata (41 packages)
+- `tools` - Individual tools (92 tools)
+- `sync_checkpoints` - Sync progress tracking
+- `sync_logs` - Audit trail
+- `simulations` - Playground executions
+- `health_checks` - Tool health monitoring
+
+**3. Dual Systems**
+- **TPMJS**: NPM registry sync, web frontend, tool discovery
+- **Blocks**: Development-time validation, domain modeling, AI-powered checks
+
+### Why YAML Won't Scale
+
+| Scale | File Size | Parse Time | Git Diffs | Edit Experience |
+|-------|-----------|------------|-----------|-----------------|
+| 100 tools | 160KB | ~50ms | Manageable | OK |
+| 1,000 tools | 1.6MB | ~500ms | Painful | Poor |
+| 10,000 tools | 16MB | ~5s | Unusable | Impossible |
+| 100,000 tools | 160MB | ~50s | N/A | N/A |
+| 1,000,000 tools | 1.6GB | Minutes | N/A | N/A |
+
+**Additional Problems:**
+- No partial loading (must parse entire file)
+- No concurrent editing (merge conflicts)
+- No versioning per-tool
+- No access control
+- No search/indexing
+- No validation caching
+- Memory pressure on CI/CD
+
+---
+
+## Proposed Architecture
+
+### Phase 1: Database-First Tool Registry (Months 1-3)
+
+**Goal:** Move tool definitions from YAML to database while maintaining blocks.yml compatibility for validation.
+
+#### 1.1 Extended Database Schema
+
+```prisma
+/// Tool Specification - replaces blocks.yml tool definitions
+model ToolSpec {
+ id String @id @default(cuid())
+
+ // Identity
+ name String @unique @db.VarChar(100) // e.g., "text.csvParse"
+ slug String @unique @db.VarChar(100) // e.g., "csv-parse"
+ version String @db.VarChar(20) // Spec version, not npm version
+
+ // Classification
+ category String @db.VarChar(50) // e.g., "text", "research", "workflow"
+ subcategory String? @db.VarChar(50)
+ tags String[] @db.Text
+
+ // Specification
+ description String @db.Text
+ longDescription String? @db.Text
+ inputs Json @db.JsonB // Input schema
+ outputs Json @db.JsonB // Output schema
+ domainRules Json? @db.JsonB // Domain validation rules
+ examples Json? @db.JsonB // Usage examples
+
+ // Domain Bindings
+ consumesEntities String[] @db.Text // e.g., ["csv_data", "text_content"]
+ producesEntities String[] @db.Text
+ signalMappings Json? @db.JsonB
+
+ // Governance
+ status ToolStatus @default(DRAFT) // DRAFT, REVIEW, PUBLISHED, DEPRECATED
+ visibility Visibility @default(PRIVATE) // PRIVATE, UNLISTED, PUBLIC
+ ownerId String? @map("owner_id")
+ reviewedBy String? @map("reviewed_by")
+ reviewedAt DateTime? @map("reviewed_at")
+
+ // Metrics (aggregated from implementations)
+ implementationCount Int @default(0)
+ totalDownloads Int @default(0)
+ avgQualityScore Decimal? @db.Decimal(3, 2)
+
+ // Timestamps
+ createdAt DateTime @default(now())
+ updatedAt DateTime @updatedAt
+ publishedAt DateTime?
+
+ // Relations
+ implementations ToolImplementation[]
+
+ @@index([category])
+ @@index([status])
+ @@index([visibility])
+ @@fulltext([name, description])
+}
+
+enum ToolStatus {
+ DRAFT
+ REVIEW
+ PUBLISHED
+ DEPRECATED
+ ARCHIVED
+}
+
+enum Visibility {
+ PRIVATE
+ UNLISTED
+ PUBLIC
+}
+
+/// Tool Implementation - links spec to actual npm package
+model ToolImplementation {
+ id String @id @default(cuid())
+
+ specId String @map("spec_id")
+ spec ToolSpec @relation(fields: [specId], references: [id])
+
+ packageId String @map("package_id")
+ package Package @relation(fields: [packageId], references: [id])
+
+ toolId String @map("tool_id")
+ tool Tool @relation(fields: [toolId], references: [id])
+
+ // Compliance
+ isOfficial Boolean @default(false)
+ isVerified Boolean @default(false)
+ complianceScore Decimal? @db.Decimal(3, 2)
+
+ @@unique([specId, packageId, toolId])
+}
+
+/// Domain Entity - replaces domain.entities in blocks.yml
+model DomainEntity {
+ id String @id @default(cuid())
+
+ name String @unique @db.VarChar(100)
+ fields String[] @db.Text
+ description String @db.Text
+ category String @db.VarChar(50)
+ schema Json? @db.JsonB // Full JSON Schema
+ examples Json? @db.JsonB
+
+ createdAt DateTime @default(now())
+ updatedAt DateTime @updatedAt
+
+ @@index([category])
+ @@fulltext([name, description])
+}
+
+/// Domain Signal - replaces domain.signals
+model DomainSignal {
+ id String @id @default(cuid())
+
+ name String @unique @db.VarChar(100)
+ description String @db.Text
+ extractionHint String? @db.Text
+ valueType String @db.VarChar(50) // "numeric", "categorical", "boolean"
+ validRange Json? @db.JsonB
+
+ createdAt DateTime @default(now())
+ updatedAt DateTime @updatedAt
+}
+
+/// Domain Measure - replaces domain.measures
+model DomainMeasure {
+ id String @id @default(cuid())
+
+ name String @unique @db.VarChar(100)
+ constraints String[] @db.Text
+ description String? @db.Text
+
+ createdAt DateTime @default(now())
+ updatedAt DateTime @updatedAt
+}
+
+/// Philosophy Principles - replaces philosophy array
+model PhilosophyPrinciple {
+ id String @id @default(cuid())
+
+ order Int @unique
+ principle String @db.Text
+ rationale String? @db.Text
+ enforcedBy String[] @db.Text // Which validators enforce this
+
+ createdAt DateTime @default(now())
+ updatedAt DateTime @updatedAt
+}
+```
+
+#### 1.2 YAML Generation (Backward Compatibility)
+
+Generate `blocks.yml` from database for validation:
+
+```typescript
+// packages/blocks-sync/src/generate-blocks-yml.ts
+
+export async function generateBlocksYml(options: {
+ output: string;
+ filter?: { category?: string; status?: ToolStatus[] };
+}): Promise {
+ const specs = await prisma.toolSpec.findMany({
+ where: {
+ status: { in: options.filter?.status ?? ['PUBLISHED'] },
+ category: options.filter?.category,
+ },
+ include: { implementations: true },
+ });
+
+ const entities = await prisma.domainEntity.findMany();
+ const signals = await prisma.domainSignal.findMany();
+ const measures = await prisma.domainMeasure.findMany();
+ const principles = await prisma.philosophyPrinciple.findMany({
+ orderBy: { order: 'asc' },
+ });
+
+ const blocksYml = {
+ name: 'tpmjs-official-tools',
+ root: '.',
+ philosophy: principles.map(p => p.principle),
+ domain: {
+ entities: Object.fromEntries(
+ entities.map(e => [e.name, { fields: e.fields, description: e.description }])
+ ),
+ signals: Object.fromEntries(
+ signals.map(s => [s.name, { description: s.description, extraction_hint: s.extractionHint }])
+ ),
+ measures: Object.fromEntries(
+ measures.map(m => [m.name, { constraints: m.constraints }])
+ ),
+ },
+ blocks: Object.fromEntries(
+ specs.map(s => [s.name, {
+ description: s.description,
+ path: s.slug,
+ domain_rules: s.domainRules,
+ inputs: s.inputs,
+ outputs: s.outputs,
+ }])
+ ),
+ };
+
+ await writeFile(options.output, yaml.stringify(blocksYml));
+}
+```
+
+#### 1.3 Admin API for Tool Management
+
+```typescript
+// apps/web/src/app/api/admin/specs/route.ts
+
+// Create tool spec
+POST /api/admin/specs
+{
+ "name": "text.csvParse",
+ "category": "text",
+ "description": "Parses CSV text into structured rows",
+ "inputs": [...],
+ "outputs": [...],
+ "domainRules": [...]
+}
+
+// Update tool spec
+PATCH /api/admin/specs/:id
+{
+ "description": "Updated description",
+ "status": "PUBLISHED"
+}
+
+// Bulk import from YAML
+POST /api/admin/specs/import
+Content-Type: multipart/form-data
+file: blocks.yml
+
+// Generate YAML for validation
+GET /api/admin/specs/export?format=yaml&category=text
+```
+
+---
+
+### Phase 2: Search & Discovery Infrastructure (Months 3-6)
+
+**Goal:** Enable fast search, filtering, and discovery at 10K+ scale.
+
+#### 2.1 Search Options Comparison
+
+| Solution | 10K Tools | 100K Tools | 1M Tools | Cost | Complexity |
+|----------|-----------|------------|----------|------|------------|
+| PostgreSQL Full-Text | ✅ Great | ⚠️ OK | ❌ Slow | Free | Low |
+| pg_trgm + GIN | ✅ Great | ✅ Good | ⚠️ OK | Free | Low |
+| Meilisearch | ✅ Great | ✅ Great | ✅ Great | $29/mo | Medium |
+| Typesense | ✅ Great | ✅ Great | ✅ Great | $29/mo | Medium |
+| Algolia | ✅ Great | ✅ Great | ✅ Great | $$$ | Low |
+| Elasticsearch | ✅ Great | ✅ Great | ✅ Great | $$$ | High |
+
+**Recommendation:** Start with PostgreSQL full-text + pg_trgm, migrate to Meilisearch/Typesense at 10K+ tools.
+
+#### 2.2 PostgreSQL Search Optimization
+
+```sql
+-- Add full-text search indexes
+CREATE INDEX idx_tool_specs_fts ON tool_specs
+ USING GIN (to_tsvector('english', name || ' ' || description));
+
+-- Add trigram index for fuzzy matching
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+CREATE INDEX idx_tool_specs_trgm ON tool_specs
+ USING GIN (name gin_trgm_ops, description gin_trgm_ops);
+
+-- Materialized view for search
+CREATE MATERIALIZED VIEW tool_search_index AS
+SELECT
+ ts.id,
+ ts.name,
+ ts.slug,
+ ts.category,
+ ts.description,
+ ts.tags,
+ ts.status,
+ ts.visibility,
+ ts.implementation_count,
+ ts.total_downloads,
+ ts.avg_quality_score,
+ ts.published_at,
+ to_tsvector('english',
+ ts.name || ' ' ||
+ ts.description || ' ' ||
+ array_to_string(ts.tags, ' ')
+ ) as search_vector
+FROM tool_specs ts
+WHERE ts.visibility = 'PUBLIC' AND ts.status = 'PUBLISHED';
+
+CREATE INDEX idx_tool_search_fts ON tool_search_index USING GIN (search_vector);
+REFRESH MATERIALIZED VIEW CONCURRENTLY tool_search_index;
+```
+
+#### 2.3 Dedicated Search Service (Meilisearch)
+
+```typescript
+// packages/search/src/meilisearch.ts
+
+import { MeiliSearch } from 'meilisearch';
+
+const client = new MeiliSearch({
+ host: process.env.MEILISEARCH_HOST!,
+ apiKey: process.env.MEILISEARCH_API_KEY!,
+});
+
+export interface ToolSearchDocument {
+ id: string;
+ name: string;
+ slug: string;
+ category: string;
+ subcategory?: string;
+ description: string;
+ tags: string[];
+ inputs: string[]; // Flattened input names for search
+ outputs: string[]; // Flattened output names for search
+ entities: string[]; // Domain entities consumed/produced
+ downloads: number;
+ qualityScore: number;
+ publishedAt: number;
+}
+
+export async function initializeSearchIndex() {
+ const index = client.index('tools');
+
+ await index.updateSettings({
+ searchableAttributes: [
+ 'name',
+ 'description',
+ 'tags',
+ 'category',
+ 'subcategory',
+ 'inputs',
+ 'outputs',
+ 'entities',
+ ],
+ filterableAttributes: [
+ 'category',
+ 'subcategory',
+ 'tags',
+ 'qualityScore',
+ 'downloads',
+ ],
+ sortableAttributes: [
+ 'downloads',
+ 'qualityScore',
+ 'publishedAt',
+ ],
+ rankingRules: [
+ 'words',
+ 'typo',
+ 'proximity',
+ 'attribute',
+ 'sort',
+ 'exactness',
+ 'downloads:desc',
+ 'qualityScore:desc',
+ ],
+ });
+}
+
+export async function indexTools(tools: ToolSearchDocument[]) {
+ const index = client.index('tools');
+ await index.addDocuments(tools, { primaryKey: 'id' });
+}
+
+export async function searchTools(query: string, options: {
+ category?: string;
+ limit?: number;
+ offset?: number;
+}) {
+ const index = client.index('tools');
+
+ return index.search(query, {
+ filter: options.category ? `category = "${options.category}"` : undefined,
+ limit: options.limit ?? 20,
+ offset: options.offset ?? 0,
+ });
+}
+```
+
+#### 2.4 Search API
+
+```typescript
+// apps/web/src/app/api/search/route.ts
+
+export async function GET(request: Request) {
+ const { searchParams } = new URL(request.url);
+ const q = searchParams.get('q') ?? '';
+ const category = searchParams.get('category');
+ const sort = searchParams.get('sort') ?? 'relevance';
+ const limit = Math.min(parseInt(searchParams.get('limit') ?? '20'), 100);
+ const offset = parseInt(searchParams.get('offset') ?? '0');
+
+ // At 10K+ tools, use Meilisearch
+ if (await shouldUseDedicatedSearch()) {
+ const results = await meilisearch.searchTools(q, { category, limit, offset });
+ return NextResponse.json(results);
+ }
+
+ // Under 10K tools, use PostgreSQL
+ const results = await prisma.$queryRaw`
+ SELECT * FROM tool_search_index
+ WHERE search_vector @@ plainto_tsquery('english', ${q})
+ ${category ? Prisma.sql`AND category = ${category}` : Prisma.empty}
+ ORDER BY
+ ts_rank(search_vector, plainto_tsquery('english', ${q})) DESC,
+ total_downloads DESC
+ LIMIT ${limit} OFFSET ${offset}
+ `;
+
+ return NextResponse.json(results);
+}
+```
+
+---
+
+### Phase 3: Distributed Validation (Months 6-9)
+
+**Goal:** Validate tools at scale without blocking on a single YAML file.
+
+#### 3.1 Validation Job Queue
+
+```typescript
+// packages/validation-worker/src/queue.ts
+
+import { Queue, Worker } from 'bullmq';
+
+const validationQueue = new Queue('tool-validation', {
+ connection: redis,
+});
+
+interface ValidationJob {
+ specId: string;
+ validatorIds: string[]; // Which validators to run
+ priority: 'high' | 'normal' | 'low';
+ triggeredBy: 'publish' | 'update' | 'scheduled' | 'manual';
+}
+
+// Enqueue validation
+export async function enqueueValidation(job: ValidationJob) {
+ await validationQueue.add('validate', job, {
+ priority: job.priority === 'high' ? 1 : job.priority === 'normal' ? 5 : 10,
+ removeOnComplete: 1000,
+ removeOnFail: 5000,
+ });
+}
+
+// Process validations
+const worker = new Worker('tool-validation', async (job) => {
+ const { specId, validatorIds } = job.data;
+
+ // Fetch spec from database
+ const spec = await prisma.toolSpec.findUnique({
+ where: { id: specId },
+ include: { implementations: true },
+ });
+
+ // Generate mini blocks.yml for just this tool
+ const miniBlocksYml = generateSingleToolYml(spec);
+
+ // Run validators
+ const results = await Promise.all(
+ validatorIds.map(id => runValidator(id, miniBlocksYml))
+ );
+
+ // Store results
+ await prisma.validationResult.create({
+ data: {
+ specId,
+ results: JSON.stringify(results),
+ valid: results.every(r => r.valid),
+ },
+ });
+
+ return results;
+}, { connection: redis });
+```
+
+#### 3.2 Validation Results Schema
+
+```prisma
+model ValidationResult {
+ id String @id @default(cuid())
+
+ specId String @map("spec_id")
+ spec ToolSpec @relation(fields: [specId], references: [id])
+
+ valid Boolean
+ validators Json @db.JsonB // { validatorId: { valid, issues, context } }
+ summary String? @db.Text
+
+ createdAt DateTime @default(now())
+
+ @@index([specId])
+ @@index([valid])
+ @@index([createdAt])
+}
+```
+
+#### 3.3 Incremental Validation
+
+Only validate what changed:
+
+```typescript
+// packages/validation/src/incremental.ts
+
+export async function validateIncremental(specId: string) {
+ const spec = await prisma.toolSpec.findUnique({ where: { id: specId } });
+ const lastValidation = await prisma.validationResult.findFirst({
+ where: { specId },
+ orderBy: { createdAt: 'desc' },
+ });
+
+ // Check what changed
+ const specHash = hashSpec(spec);
+ const lastHash = lastValidation?.metadata?.specHash;
+
+ if (specHash === lastHash) {
+ return lastValidation; // No changes, return cached
+ }
+
+ // Determine which validators need to re-run
+ const changedFields = diffSpecs(spec, lastValidation?.spec);
+ const validators = selectValidatorsForChanges(changedFields);
+
+ return enqueueValidation({
+ specId,
+ validatorIds: validators,
+ priority: 'normal',
+ triggeredBy: 'update',
+ });
+}
+```
+
+---
+
+### Phase 4: Governance & Quality (Months 9-12)
+
+**Goal:** Maintain quality at scale with automated and human review.
+
+#### 4.1 Multi-Stage Review Pipeline
+
+```
+┌─────────────┐ ┌──────────────┐ ┌───────────────┐ ┌───────────┐
+│ DRAFT │────▶│ AUTOMATED │────▶│ HUMAN REVIEW │────▶│ PUBLISHED │
+│ │ │ REVIEW │ │ (optional) │ │ │
+└─────────────┘ └──────────────┘ └───────────────┘ └───────────┘
+ │ │ │ │
+ │ │ │ │
+ ▼ ▼ ▼ ▼
+ - Author creates - Schema valid? - Trusted author? - Visible in
+ - Saves draft - Domain rules ok? - Skip review - Search index
+ - Philosophy ok? - OR - API available
+ - Tests pass? - Manual approve - Stats tracked
+```
+
+#### 4.2 Automated Quality Gates
+
+```typescript
+// packages/governance/src/quality-gates.ts
+
+export interface QualityGate {
+ id: string;
+ name: string;
+ required: boolean;
+ check: (spec: ToolSpec) => Promise;
+}
+
+export const qualityGates: QualityGate[] = [
+ {
+ id: 'schema-valid',
+ name: 'Schema Validation',
+ required: true,
+ check: async (spec) => {
+ const ajv = new Ajv();
+ const inputValid = ajv.validateSchema(spec.inputs);
+ const outputValid = ajv.validateSchema(spec.outputs);
+ return { valid: inputValid && outputValid, issues: ajv.errors };
+ },
+ },
+ {
+ id: 'has-description',
+ name: 'Has Description',
+ required: true,
+ check: async (spec) => ({
+ valid: spec.description.length >= 20,
+ issues: spec.description.length < 20 ? ['Description too short'] : [],
+ }),
+ },
+ {
+ id: 'has-examples',
+ name: 'Has Examples',
+ required: false,
+ check: async (spec) => ({
+ valid: spec.examples && spec.examples.length > 0,
+ issues: !spec.examples ? ['No examples provided'] : [],
+ }),
+ },
+ {
+ id: 'domain-compliance',
+ name: 'Domain Compliance',
+ required: true,
+ check: async (spec) => {
+ // Run blocks domain validator
+ return runDomainValidator(spec);
+ },
+ },
+ {
+ id: 'naming-convention',
+ name: 'Naming Convention',
+ required: true,
+ check: async (spec) => {
+ const pattern = /^[a-z]+(\.[a-zA-Z]+)+$/;
+ return {
+ valid: pattern.test(spec.name),
+ issues: !pattern.test(spec.name)
+ ? ['Name must be category.toolName format']
+ : [],
+ };
+ },
+ },
+ {
+ id: 'security-scan',
+ name: 'Security Scan',
+ required: true,
+ check: async (spec) => {
+ // Scan for suspicious patterns in domain rules
+ return securityScanner.scan(spec);
+ },
+ },
+];
+
+export async function runQualityGates(spec: ToolSpec): Promise<{
+ passed: boolean;
+ results: Record;
+}> {
+ const results: Record = {};
+
+ for (const gate of qualityGates) {
+ results[gate.id] = await gate.check(spec);
+ }
+
+ const requiredPassed = qualityGates
+ .filter(g => g.required)
+ .every(g => results[g.id].valid);
+
+ return { passed: requiredPassed, results };
+}
+```
+
+#### 4.3 Trusted Publishers
+
+```prisma
+model Publisher {
+ id String @id @default(cuid())
+
+ userId String @unique
+ displayName String
+ verified Boolean @default(false)
+ trustLevel TrustLevel @default(STANDARD)
+
+ // Auto-publish settings
+ autoPublish Boolean @default(false) // Skip manual review
+ autoPublishLimit Int @default(10) // Max auto-publish per day
+
+ // Stats
+ publishedCount Int @default(0)
+ rejectedCount Int @default(0)
+ qualityAvg Decimal? @db.Decimal(3, 2)
+
+ createdAt DateTime @default(now())
+ updatedAt DateTime @updatedAt
+}
+
+enum TrustLevel {
+ NEW // First 10 tools need review
+ STANDARD // Normal review process
+ TRUSTED // Can auto-publish up to limit
+ CORE // Official maintainer, unlimited auto-publish
+}
+```
+
+#### 4.4 Moderation Queue
+
+```typescript
+// apps/web/src/app/api/admin/moderation/route.ts
+
+// Get pending reviews
+GET /api/admin/moderation?status=pending&limit=20
+
+// Approve tool
+POST /api/admin/moderation/:specId/approve
+{
+ "comment": "Looks good, approved"
+}
+
+// Reject tool
+POST /api/admin/moderation/:specId/reject
+{
+ "reason": "Missing required examples",
+ "comment": "Please add at least 2 usage examples"
+}
+
+// Request changes
+POST /api/admin/moderation/:specId/request-changes
+{
+ "changes": [
+ "Add more detailed description",
+ "Fix input schema - missing required field"
+ ]
+}
+```
+
+---
+
+### Phase 5: Scale to 1 Million (Months 12-18)
+
+#### 5.1 Infrastructure Changes
+
+| Component | 10K Tools | 100K Tools | 1M Tools |
+|-----------|-----------|------------|----------|
+| Database | Single PostgreSQL | Read replicas | Sharded PostgreSQL or CockroachDB |
+| Search | Meilisearch single | Meilisearch cluster | Elasticsearch cluster |
+| Cache | Redis single | Redis cluster | Redis cluster + CDN |
+| Validation | Single worker | Worker pool | Distributed workers (k8s) |
+| API | Single region | Multi-region | Global edge (Cloudflare Workers) |
+
+#### 5.2 Caching Strategy
+
+```typescript
+// packages/cache/src/strategy.ts
+
+export const cacheConfig = {
+ // Tool specs - rarely change
+ specs: {
+ ttl: 3600, // 1 hour
+ staleWhileRevalidate: 86400, // 1 day
+ },
+
+ // Search results - personalized, shorter cache
+ search: {
+ ttl: 60, // 1 minute
+ staleWhileRevalidate: 300, // 5 minutes
+ },
+
+ // Domain entities - almost never change
+ domain: {
+ ttl: 86400, // 1 day
+ staleWhileRevalidate: 604800, // 1 week
+ },
+
+ // Validation results - cache until spec changes
+ validation: {
+ ttl: 0, // Invalidate on spec change
+ keyPrefix: (specId: string, specHash: string) =>
+ `validation:${specId}:${specHash}`,
+ },
+};
+
+// Multi-layer caching
+export async function getToolSpec(id: string): Promise {
+ // L1: In-memory (per-request)
+ const memory = memoryCache.get(id);
+ if (memory) return memory;
+
+ // L2: Redis
+ const redis = await redisCache.get(`spec:${id}`);
+ if (redis) {
+ memoryCache.set(id, redis);
+ return redis;
+ }
+
+ // L3: Database
+ const db = await prisma.toolSpec.findUnique({ where: { id } });
+ if (db) {
+ await redisCache.set(`spec:${id}`, db, cacheConfig.specs.ttl);
+ memoryCache.set(id, db);
+ }
+
+ return db;
+}
+```
+
+#### 5.3 Database Sharding Strategy
+
+For 1M+ tools, consider sharding by category:
+
+```typescript
+// Sharding key: category
+// Shard 0: text.*, data.*, research.*
+// Shard 1: workflow.*, automation.*, integration.*
+// Shard 2: security.*, compliance.*, legal.*
+// Shard 3: ml.*, ai.*, analysis.*
+// Shard 4: finance.*, accounting.*, ops.*
+// ...
+
+// Cross-shard queries use scatter-gather
+export async function searchAcrossShards(query: string) {
+ const shards = getAllShards();
+ const results = await Promise.all(
+ shards.map(shard => shard.search(query))
+ );
+ return mergeAndRank(results);
+}
+```
+
+#### 5.4 CDN-First API Design
+
+```typescript
+// Edge function for tool lookup
+// Deployed to Cloudflare Workers / Vercel Edge
+
+export default {
+ async fetch(request: Request) {
+ const url = new URL(request.url);
+ const slug = url.pathname.replace('/api/tools/', '');
+
+ // Check edge cache
+ const cached = await caches.default.match(request);
+ if (cached) return cached;
+
+ // Fetch from origin with cache headers
+ const response = await fetch(`${ORIGIN_URL}/api/tools/${slug}`, {
+ cf: { cacheTtl: 3600 },
+ });
+
+ // Clone and cache
+ const cloned = response.clone();
+ await caches.default.put(request, cloned);
+
+ return response;
+ },
+};
+```
+
+---
+
+### Phase 6: Ecosystem Features (Ongoing)
+
+#### 6.1 Tool Composition / Recipes
+
+```prisma
+model Recipe {
+ id String @id @default(cuid())
+
+ name String @unique
+ description String @db.Text
+
+ // Ordered list of tools in the recipe
+ steps Json @db.JsonB
+ // [
+ // { specId: "...", inputMappings: { ... } },
+ // { specId: "...", inputMappings: { ... } },
+ // ]
+
+ // Computed metrics
+ estimatedDuration Int? // milliseconds
+ complexity String? // "simple" | "moderate" | "complex"
+
+ publisherId String
+ visibility Visibility
+
+ createdAt DateTime @default(now())
+ updatedAt DateTime @updatedAt
+}
+```
+
+#### 6.2 Tool Analytics
+
+```prisma
+model ToolAnalytics {
+ id String @id @default(cuid())
+
+ specId String @map("spec_id")
+ date DateTime @db.Date
+
+ // Usage
+ views Int @default(0)
+ apiCalls Int @default(0)
+ uniqueUsers Int @default(0)
+
+ // Performance
+ avgLatencyMs Int?
+ p95LatencyMs Int?
+ errorRate Decimal? @db.Decimal(5, 4) // 0.0000 to 1.0000
+
+ // Discovery
+ searchImpressions Int @default(0)
+ searchClicks Int @default(0)
+ directLinks Int @default(0)
+
+ @@unique([specId, date])
+ @@index([date])
+}
+```
+
+#### 6.3 Versioning & Changelogs
+
+```prisma
+model ToolSpecVersion {
+ id String @id @default(cuid())
+
+ specId String @map("spec_id")
+ version String @db.VarChar(20) // semver
+
+ // Snapshot of spec at this version
+ snapshot Json @db.JsonB
+
+ // Changelog
+ changelog String? @db.Text
+ breaking Boolean @default(false)
+
+ createdAt DateTime @default(now())
+
+ @@unique([specId, version])
+ @@index([specId])
+}
+```
+
+---
+
+## Migration Path
+
+### Step 1: Import Existing blocks.yml to Database
+
+```typescript
+// scripts/migrate-blocks-yml.ts
+
+import yaml from 'yaml';
+
+async function migrate() {
+ const content = await readFile('packages/tools/official/blocks.yml', 'utf-8');
+ const blocks = yaml.parse(content);
+
+ // Import philosophy
+ for (const [index, principle] of blocks.philosophy.entries()) {
+ await prisma.philosophyPrinciple.upsert({
+ where: { order: index },
+ create: { order: index, principle },
+ update: { principle },
+ });
+ }
+
+ // Import domain entities
+ for (const [name, entity] of Object.entries(blocks.domain.entities)) {
+ await prisma.domainEntity.upsert({
+ where: { name },
+ create: {
+ name,
+ fields: entity.fields,
+ description: entity.description,
+ category: inferCategory(name),
+ },
+ update: { fields: entity.fields, description: entity.description },
+ });
+ }
+
+ // Import tool specs
+ for (const [name, block] of Object.entries(blocks.blocks)) {
+ const [category, ...rest] = name.split('.');
+ await prisma.toolSpec.upsert({
+ where: { name },
+ create: {
+ name,
+ slug: block.path,
+ category,
+ description: block.description,
+ inputs: block.inputs,
+ outputs: block.outputs,
+ domainRules: block.domain_rules,
+ status: 'PUBLISHED',
+ visibility: 'PUBLIC',
+ },
+ update: { /* ... */ },
+ });
+ }
+}
+```
+
+### Step 2: Maintain Dual-Write During Transition
+
+```typescript
+// Write to both database and regenerate YAML
+export async function updateToolSpec(id: string, data: Partial) {
+ // Update database
+ await prisma.toolSpec.update({
+ where: { id },
+ data,
+ });
+
+ // Regenerate YAML for validation
+ await generateBlocksYml({
+ output: 'packages/tools/official/blocks.yml',
+ });
+}
+```
+
+### Step 3: Switch to Database-First
+
+Once validated:
+1. Remove YAML as source of truth
+2. Generate YAML only for blocks validation
+3. Update CI/CD to use database
+4. Archive blocks.yml (keep for reference)
+
+---
+
+## Cost Estimates
+
+| Scale | Database | Search | Cache | Workers | CDN | Total/mo |
+|-------|----------|--------|-------|---------|-----|----------|
+| 1K tools | $20 (Neon) | $0 (PG) | $0 | $0 | $0 | ~$20 |
+| 10K tools | $50 | $29 | $20 | $0 | $0 | ~$100 |
+| 100K tools | $200 | $99 | $100 | $50 | $50 | ~$500 |
+| 1M tools | $1000 | $500 | $500 | $500 | $200 | ~$2,700 |
+
+---
+
+## Open Questions
+
+1. **Blocks validation at scale**: Should each tool have its own mini-blocks.yml, or should we batch validate?
+
+2. **Real-time sync vs eventual consistency**: How fresh do search results need to be?
+
+3. **Multi-tenancy**: Will organizations want private tool registries?
+
+4. **Federation**: Should tools be able to reference tools from other registries?
+
+5. **AI-generated tools**: How do we handle LLM-generated tool specs at scale?
+
+6. **Deprecation policy**: How long to keep deprecated tools available?
+
+7. **Breaking changes**: How to handle breaking changes to popular tools?
+
+---
+
+## Appendix: Schema Migrations
+
+### Migration 1: Add ToolSpec Table
+
+```sql
+CREATE TABLE tool_specs (
+ id TEXT PRIMARY KEY,
+ name VARCHAR(100) UNIQUE NOT NULL,
+ slug VARCHAR(100) UNIQUE NOT NULL,
+ version VARCHAR(20) NOT NULL,
+ category VARCHAR(50) NOT NULL,
+ subcategory VARCHAR(50),
+ tags TEXT[],
+ description TEXT NOT NULL,
+ long_description TEXT,
+ inputs JSONB NOT NULL,
+ outputs JSONB NOT NULL,
+ domain_rules JSONB,
+ examples JSONB,
+ consumes_entities TEXT[],
+ produces_entities TEXT[],
+ signal_mappings JSONB,
+ status VARCHAR(20) DEFAULT 'DRAFT',
+ visibility VARCHAR(20) DEFAULT 'PRIVATE',
+ owner_id TEXT,
+ reviewed_by TEXT,
+ reviewed_at TIMESTAMP,
+ implementation_count INTEGER DEFAULT 0,
+ total_downloads INTEGER DEFAULT 0,
+ avg_quality_score DECIMAL(3,2),
+ created_at TIMESTAMP DEFAULT NOW(),
+ updated_at TIMESTAMP DEFAULT NOW(),
+ published_at TIMESTAMP
+);
+
+CREATE INDEX idx_tool_specs_category ON tool_specs(category);
+CREATE INDEX idx_tool_specs_status ON tool_specs(status);
+CREATE INDEX idx_tool_specs_visibility ON tool_specs(visibility);
+```
+
+### Migration 2: Add Full-Text Search
+
+```sql
+ALTER TABLE tool_specs
+ADD COLUMN search_vector tsvector
+GENERATED ALWAYS AS (
+ to_tsvector('english',
+ coalesce(name, '') || ' ' ||
+ coalesce(description, '') || ' ' ||
+ coalesce(array_to_string(tags, ' '), '')
+ )
+) STORED;
+
+CREATE INDEX idx_tool_specs_search ON tool_specs USING GIN(search_vector);
+```
+
+---
+
+## Summary
+
+Scaling from 100 to 1M tools requires:
+
+1. **Database-first architecture** - Move tool specs from YAML to PostgreSQL
+2. **Search infrastructure** - PostgreSQL full-text → Meilisearch/Typesense
+3. **Distributed validation** - Job queues with incremental validation
+4. **Governance pipeline** - Automated quality gates + human review
+5. **Caching strategy** - Multi-layer caching + CDN
+6. **Sharding/replication** - At 1M+ tools, consider sharding
+
+The migration can be done incrementally, maintaining backward compatibility with the existing blocks.yml workflow throughout.
diff --git a/package.json b/package.json
index e8e034e..e54ac7e 100644
--- a/package.json
+++ b/package.json
@@ -38,6 +38,9 @@
"changeset:publish": "pnpm build && changeset publish",
"prepare": "node -e \"if (!process.env.CI && !process.env.VERCEL && require('fs').existsSync('.git')) { require('child_process').execSync('lefthook install', {stdio: 'inherit'}) }\""
},
+ "pnpm": {
+ "onlyBuiltDependencies": ["better-sqlite3", "esbuild"]
+ },
"devDependencies": {
"@biomejs/biome": "^1.9.4",
"@blocksai/cli": "^0.2.1",
diff --git a/packages/tool-ideas/.gitignore b/packages/tool-ideas/.gitignore
new file mode 100644
index 0000000..759d65e
--- /dev/null
+++ b/packages/tool-ideas/.gitignore
@@ -0,0 +1,8 @@
+# Database files
+data/*.db
+data/*.db-journal
+data/*.db-wal
+data/*.db-shm
+
+# Keep the data directory
+!data/.gitkeep
diff --git a/packages/tool-ideas/data/.gitkeep b/packages/tool-ideas/data/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/packages/tool-ideas/package.json b/packages/tool-ideas/package.json
new file mode 100644
index 0000000..a014bce
--- /dev/null
+++ b/packages/tool-ideas/package.json
@@ -0,0 +1,43 @@
+{
+ "name": "@tpmjs/tool-ideas",
+ "version": "0.1.0",
+ "description": "Generate 10K high-quality AI tool ideas for the TPMJS registry",
+ "private": true,
+ "type": "module",
+ "bin": {
+ "tool-ideas": "./dist/cli.js"
+ },
+ "exports": {
+ ".": {
+ "types": "./dist/index.d.ts",
+ "default": "./dist/index.js"
+ }
+ },
+ "files": ["dist"],
+ "scripts": {
+ "build": "tsup",
+ "dev": "tsup --watch",
+ "type-check": "tsc --noEmit",
+ "cli": "tsx src/cli.ts"
+ },
+ "dependencies": {
+ "ai": "^6.0.3",
+ "@ai-sdk/openai": "^3.0.1",
+ "better-sqlite3": "^11.8.1",
+ "drizzle-orm": "^0.38.3",
+ "commander": "^13.0.0",
+ "p-limit": "^6.2.0",
+ "zod": "^3.24.1",
+ "ora": "^8.1.1",
+ "chalk": "^5.4.1"
+ },
+ "devDependencies": {
+ "@tpmjs/tsconfig": "workspace:*",
+ "@types/better-sqlite3": "^7.6.12",
+ "@types/node": "^22.10.5",
+ "drizzle-kit": "^0.30.1",
+ "tsup": "^8.3.5",
+ "tsx": "^4.19.2",
+ "typescript": "^5.7.2"
+ }
+}
diff --git a/packages/tool-ideas/src/cli.ts b/packages/tool-ideas/src/cli.ts
new file mode 100644
index 0000000..15e488e
--- /dev/null
+++ b/packages/tool-ideas/src/cli.ts
@@ -0,0 +1,21 @@
+import { Command } from 'commander';
+import { enrichCommand } from './commands/enrich.js';
+import { exportCommand } from './commands/export.js';
+import { generateCommand } from './commands/generate.js';
+import { statsCommand } from './commands/stats.js';
+import { vocabCommand } from './commands/vocab.js';
+
+const program = new Command();
+
+program
+ .name('tool-ideas')
+ .description('Generate and enrich AI tool ideas for TPMJS')
+ .version('0.1.0');
+
+program.addCommand(vocabCommand);
+program.addCommand(generateCommand);
+program.addCommand(enrichCommand);
+program.addCommand(statsCommand);
+program.addCommand(exportCommand);
+
+program.parse();
diff --git a/packages/tool-ideas/src/commands/enrich.ts b/packages/tool-ideas/src/commands/enrich.ts
new file mode 100644
index 0000000..c1ab1e4
--- /dev/null
+++ b/packages/tool-ideas/src/commands/enrich.ts
@@ -0,0 +1,85 @@
+import chalk from 'chalk';
+import { Command } from 'commander';
+import ora from 'ora';
+import { BatchProcessor, getEnrichmentStats } from '../enrichment/batch-processor.js';
+import { getSkeletonStats } from '../generators/skeleton-generator.js';
+
+export const enrichCommand = new Command('enrich')
+ .description('Enrich tool skeletons with GPT-4.1-mini')
+ .option('--db ', 'Database path', './data/tool-ideas.db')
+ .option('--batch ', 'Batch size', '100')
+ .option('--concurrency ', 'Concurrent API calls', '5')
+ .option('--cost-limit ', 'Max cost in USD', '50')
+ .option('--continuous', 'Process all pending skeletons', false)
+ .option('--model ', 'OpenAI model to use', 'gpt-4.1-mini')
+ .action(async (options) => {
+ const spinner = ora('Starting enrichment...').start();
+
+ try {
+ // Check pending count
+ const stats = getSkeletonStats(options.db);
+ if (stats.pending === 0) {
+ spinner.info(chalk.yellow('No pending skeletons to process'));
+ return;
+ }
+
+ spinner.text = `Found ${stats.pending} pending skeletons`;
+
+ const processor = new BatchProcessor({
+ dbPath: options.db,
+ batchSize: Number.parseInt(options.batch),
+ concurrency: Number.parseInt(options.concurrency),
+ costLimitUsd: Number.parseFloat(options.costLimit),
+ model: options.model,
+ onProgress: (processed, total, cost) => {
+ const pct = Math.round((processed / total) * 100);
+ spinner.text = `Enriching: ${processed}/${total} (${pct}%) | Cost: $${cost.toFixed(4)}`;
+ },
+ onError: (error, skeletonId) => {
+ console.log(chalk.red(`\n Error processing skeleton ${skeletonId}: ${error.message}`));
+ },
+ });
+
+ if (options.continuous) {
+ // Process all
+ spinner.text = 'Processing all pending skeletons...';
+ const result = await processor.processAll();
+
+ spinner.succeed(chalk.green('Enrichment complete!'));
+ console.log(chalk.dim('─'.repeat(50)));
+ console.log(` Processed: ${chalk.cyan(result.totalProcessed)}`);
+ console.log(` Failed: ${chalk.red(result.totalFailed)}`);
+ console.log(` Nonsensical: ${chalk.yellow(result.totalNonsensical)}`);
+ console.log(` Total cost: ${chalk.green(`$${result.totalCost.toFixed(4)}`)}`);
+ } else {
+ // Process single batch
+ const result = await processor.processNextBatch();
+
+ if (result.success) {
+ spinner.succeed(chalk.green('Batch processed!'));
+ console.log(chalk.dim('─'.repeat(50)));
+ console.log(` Processed: ${chalk.cyan(result.processed)}`);
+ console.log(` Failed: ${chalk.red(result.failed)}`);
+ console.log(` Nonsensical: ${chalk.yellow(result.nonsensical)}`);
+ console.log(` Batch cost: ${chalk.green(`$${result.cost.toFixed(4)}`)}`);
+ console.log(chalk.dim('\n Run with --continuous to process all pending'));
+ } else {
+ spinner.warn(chalk.yellow(result.message));
+ }
+ }
+
+ // Show enrichment stats
+ const enrichStats = getEnrichmentStats(options.db);
+ console.log(chalk.dim('─'.repeat(50)));
+ console.log(chalk.bold('Enrichment Stats:'));
+ console.log(` Total ideas: ${chalk.cyan(enrichStats.totalIdeas)}`);
+ console.log(` Quality ideas: ${chalk.green(enrichStats.quality)}`);
+ console.log(` Nonsensical: ${chalk.yellow(enrichStats.nonsensical)}`);
+ console.log(` Avg quality: ${chalk.cyan(enrichStats.avgQualityScore.toFixed(2))}`);
+ console.log(` Total cost: ${chalk.green(`$${enrichStats.totalCost.toFixed(4)}`)}`);
+ } catch (error) {
+ spinner.fail(chalk.red('Enrichment failed'));
+ console.error(error);
+ process.exit(1);
+ }
+ });
diff --git a/packages/tool-ideas/src/commands/export.ts b/packages/tool-ideas/src/commands/export.ts
new file mode 100644
index 0000000..c45038c
--- /dev/null
+++ b/packages/tool-ideas/src/commands/export.ts
@@ -0,0 +1,231 @@
+import { writeFileSync } from 'node:fs';
+import chalk from 'chalk';
+import { Command } from 'commander';
+import { and, desc, eq, gte, inArray, sql } from 'drizzle-orm';
+import ora from 'ora';
+import { getDatabase } from '../db/client.js';
+import { categories, contexts, objects, toolIdeas, toolSkeletons, verbs } from '../db/schema.js';
+
+interface ExportedTool {
+ name: string;
+ description: string;
+ category: string;
+ parameters: unknown[];
+ returns: { type: string; description: string };
+ aiAgent: { useCase: string; limitations?: string; examples?: string[] };
+ tags: string[];
+ examples: { input: Record; description: string }[];
+ qualityScore: number;
+ skeleton: {
+ verb: string;
+ object: string;
+ context: string | null;
+ };
+}
+
+export const exportCommand = new Command('export')
+ .description('Export enriched tools to JSON')
+ .option('--db ', 'Database path', './data/tool-ideas.db')
+ .option('--output ', 'Output file path', './data/tools-export.json')
+ .option('--min-quality ', 'Minimum quality score', '0.5')
+ .option('--exclude-nonsensical', 'Exclude nonsensical tools', false)
+ .option('--limit ', 'Maximum tools to export', '0')
+ .option('--format ', 'Export format: json, jsonl, prisma', 'json')
+ .action(async (options) => {
+ const spinner = ora('Exporting tools...').start();
+
+ try {
+ const db = getDatabase(options.db);
+ const minQuality = Number.parseFloat(options.minQuality);
+ const limit = Number.parseInt(options.limit);
+
+ // Build query conditions
+ const conditions = [gte(toolIdeas.qualityScore, minQuality)];
+ if (options.excludeNonsensical) {
+ conditions.push(eq(toolIdeas.isNonsensical, false));
+ }
+
+ // Query tools with skeleton relations
+ let query = db
+ .select()
+ .from(toolIdeas)
+ .where(and(...conditions))
+ .orderBy(desc(toolIdeas.qualityScore));
+
+ if (limit > 0) {
+ query = query.limit(limit);
+ }
+
+ const tools = query.all();
+
+ spinner.text = `Found ${tools.length} tools to export`;
+
+ // Load skeleton data for context
+ const skeletonIds = [...new Set(tools.map((t) => t.skeletonId))];
+ const skeletons = db
+ .select()
+ .from(toolSkeletons)
+ .where(inArray(toolSkeletons.id, skeletonIds))
+ .all();
+ const skeletonMap = new Map(skeletons.map((s) => [s.id, s]));
+
+ // Load related vocabulary
+ const verbIds = [...new Set(skeletons.map((s) => s.verbId))];
+ const objectIds = [...new Set(skeletons.map((s) => s.objectId))];
+ const contextIds = [
+ ...new Set(skeletons.map((s) => s.contextId).filter(Boolean)),
+ ] as number[];
+
+ const verbMap = new Map(
+ db
+ .select()
+ .from(verbs)
+ .where(inArray(verbs.id, verbIds))
+ .all()
+ .map((v) => [v.id, v])
+ );
+ const objectMap = new Map(
+ db
+ .select()
+ .from(objects)
+ .where(inArray(objects.id, objectIds))
+ .all()
+ .map((o) => [o.id, o])
+ );
+ const contextMap =
+ contextIds.length > 0
+ ? new Map(
+ db
+ .select()
+ .from(contexts)
+ .where(inArray(contexts.id, contextIds))
+ .all()
+ .map((c) => [c.id, c])
+ )
+ : new Map();
+
+ // Transform to export format
+ const exported: ExportedTool[] = tools
+ .map((tool) => {
+ const skeleton = skeletonMap.get(tool.skeletonId);
+ if (!skeleton) return null;
+ const verb = verbMap.get(skeleton.verbId);
+ const object = objectMap.get(skeleton.objectId);
+ if (!verb || !object) return null;
+ const context = skeleton.contextId ? contextMap.get(skeleton.contextId) : null;
+
+ // Parse category from name (e.g., "data.parseCSV" -> "data")
+ const category = tool.name.split('.')[0];
+
+ return {
+ name: tool.name,
+ description: tool.description,
+ category,
+ parameters: JSON.parse(tool.parametersJson),
+ returns: JSON.parse(tool.returnsJson),
+ aiAgent: JSON.parse(tool.aiAgentJson),
+ tags: JSON.parse(tool.tagsJson),
+ examples: JSON.parse(tool.examplesJson),
+ qualityScore: tool.qualityScore,
+ skeleton: {
+ verb: verb.name,
+ object: object.name,
+ context: context?.name ?? null,
+ },
+ };
+ })
+ .filter((t): t is ExportedTool => t !== null);
+
+ // Write output based on format
+ let output: string;
+ let outputPath = options.output;
+
+ switch (options.format) {
+ case 'jsonl':
+ output = exported.map((t) => JSON.stringify(t)).join('\n');
+ if (!outputPath.endsWith('.jsonl')) {
+ outputPath = outputPath.replace(/\.json$/, '.jsonl');
+ }
+ break;
+
+ case 'prisma': {
+ // Export in format ready for Prisma seed
+ const prismaData = exported.map((t) => ({
+ name: t.name,
+ slug: t.name.replace('.', '-').toLowerCase(),
+ description: t.description,
+ category: t.category,
+ isOfficial: false,
+ tier: 'rich',
+ toolSpec: {
+ name: t.name,
+ description: t.description,
+ parameters: t.parameters,
+ returns: t.returns,
+ aiAgent: t.aiAgent,
+ tags: t.tags,
+ examples: t.examples,
+ },
+ qualityScore: t.qualityScore,
+ }));
+ output = JSON.stringify(prismaData, null, 2);
+ if (!outputPath.includes('prisma')) {
+ outputPath = outputPath.replace(/\.json$/, '-prisma.json');
+ }
+ break;
+ }
+ default:
+ output = JSON.stringify(
+ {
+ metadata: {
+ exportedAt: new Date().toISOString(),
+ count: exported.length,
+ minQuality,
+ excludeNonsensical: options.excludeNonsensical,
+ },
+ tools: exported,
+ },
+ null,
+ 2
+ );
+ }
+
+ writeFileSync(outputPath, output);
+
+ spinner.succeed(chalk.green(`Exported ${exported.length} tools to ${outputPath}`));
+
+ // Show summary
+ console.log(chalk.dim('─'.repeat(50)));
+ console.log(` Format: ${chalk.cyan(options.format)}`);
+ console.log(` Min quality: ${chalk.cyan(minQuality)}`);
+ console.log(
+ ` Nonsensical: ${chalk.cyan(options.excludeNonsensical ? 'excluded' : 'included')}`
+ );
+ console.log(` File size: ${chalk.cyan(formatBytes(Buffer.byteLength(output)))}`);
+
+ // Quality distribution
+ const qualityDist = {
+ excellent: exported.filter((t) => t.qualityScore >= 0.9).length,
+ good: exported.filter((t) => t.qualityScore >= 0.7 && t.qualityScore < 0.9).length,
+ fair: exported.filter((t) => t.qualityScore >= 0.5 && t.qualityScore < 0.7).length,
+ poor: exported.filter((t) => t.qualityScore < 0.5).length,
+ };
+
+ console.log(chalk.dim('─'.repeat(50)));
+ console.log(chalk.bold('Quality Distribution:'));
+ console.log(` Excellent (≥0.9): ${chalk.green(qualityDist.excellent)}`);
+ console.log(` Good (0.7-0.9): ${chalk.cyan(qualityDist.good)}`);
+ console.log(` Fair (0.5-0.7): ${chalk.yellow(qualityDist.fair)}`);
+ console.log(` Poor (<0.5): ${chalk.red(qualityDist.poor)}`);
+ } catch (error) {
+ spinner.fail(chalk.red('Export failed'));
+ console.error(error);
+ process.exit(1);
+ }
+ });
+
+function formatBytes(bytes: number): string {
+ if (bytes < 1024) return `${bytes} B`;
+ if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+ return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
diff --git a/packages/tool-ideas/src/commands/generate.ts b/packages/tool-ideas/src/commands/generate.ts
new file mode 100644
index 0000000..0d4c30a
--- /dev/null
+++ b/packages/tool-ideas/src/commands/generate.ts
@@ -0,0 +1,53 @@
+import chalk from 'chalk';
+import { Command } from 'commander';
+import ora from 'ora';
+import { generateSkeletons, getSkeletonStats } from '../generators/skeleton-generator.js';
+
+export const generateCommand = new Command('generate')
+ .description('Generate tool skeletons from vocabulary')
+ .option('--db ', 'Database path', './data/tool-ideas.db')
+ .option('--count ', 'Number of skeletons to generate', '10000')
+ .option('--threshold ', 'Minimum compatibility score', '0.5')
+ .option('--seed ', 'Random seed for reproducibility', '42')
+ .option('--contexts', 'Include context variations', false)
+ .action(async (options) => {
+ const spinner = ora('Generating tool skeletons...').start();
+
+ try {
+ const count = Number.parseInt(options.count);
+ const threshold = Number.parseFloat(options.threshold);
+ const seed = Number.parseInt(options.seed);
+
+ spinner.text = `Generating up to ${count} skeletons (threshold: ${threshold})...`;
+
+ const result = await generateSkeletons({
+ dbPath: options.db,
+ count,
+ threshold,
+ seed,
+ includeContexts: options.contexts,
+ onProgress: (current, total) => {
+ const pct = Math.round((current / total) * 100);
+ spinner.text = `Generating skeletons: ${current}/${total} (${pct}%)`;
+ },
+ });
+
+ spinner.succeed(chalk.green('Skeleton generation complete!'));
+ console.log(chalk.dim('─'.repeat(40)));
+ console.log(` Generated: ${chalk.cyan(result.generated)}`);
+ console.log(` Skipped: ${chalk.yellow(result.skipped)} (duplicates)`);
+
+ // Show overall stats
+ const stats = getSkeletonStats(options.db);
+ console.log(chalk.dim('─'.repeat(40)));
+ console.log(chalk.bold('Total Skeletons:'));
+ console.log(` Total: ${chalk.cyan(stats.total)}`);
+ console.log(` Pending: ${chalk.yellow(stats.pending)}`);
+ console.log(` Completed: ${chalk.green(stats.completed)}`);
+ console.log(` Failed: ${chalk.red(stats.failed)}`);
+ } catch (error) {
+ spinner.fail(chalk.red('Failed to generate skeletons'));
+ console.error(error);
+ process.exit(1);
+ }
+ });
diff --git a/packages/tool-ideas/src/commands/stats.ts b/packages/tool-ideas/src/commands/stats.ts
new file mode 100644
index 0000000..9ba4b8f
--- /dev/null
+++ b/packages/tool-ideas/src/commands/stats.ts
@@ -0,0 +1,126 @@
+import chalk from 'chalk';
+import { Command } from 'commander';
+import { desc, eq, sql } from 'drizzle-orm';
+import { getDatabase } from '../db/client.js';
+import { toolIdeas, toolSkeletons } from '../db/schema.js';
+import { getEnrichmentStats } from '../enrichment/batch-processor.js';
+import { getSkeletonStats } from '../generators/skeleton-generator.js';
+import { getVocabularyStats } from '../generators/vocabulary.js';
+
+export const statsCommand = new Command('stats')
+ .description('Show statistics for all stages')
+ .option('--db ', 'Database path', './data/tool-ideas.db')
+ .option('--detailed', 'Show detailed breakdowns', false)
+ .action(async (options) => {
+ try {
+ const db = getDatabase(options.db);
+
+ console.log(chalk.bold('\n📊 Tool Ideas Statistics\n'));
+
+ // Vocabulary stats
+ const vocabStats = getVocabularyStats(options.db);
+ console.log(chalk.bold.blue('Vocabulary'));
+ console.log(chalk.dim('─'.repeat(50)));
+ console.log(` Categories: ${chalk.cyan(vocabStats.categories.toString().padStart(6))}`);
+ console.log(` Verbs: ${chalk.cyan(vocabStats.verbs.toString().padStart(6))}`);
+ console.log(` Objects: ${chalk.cyan(vocabStats.objects.toString().padStart(6))}`);
+ console.log(` Contexts: ${chalk.cyan(vocabStats.contexts.toString().padStart(6))}`);
+ console.log(` Qualifiers: ${chalk.cyan(vocabStats.qualifiers.toString().padStart(6))}`);
+ console.log(
+ ` ${chalk.bold('Total:')} ${chalk.bold(vocabStats.total.toString().padStart(7))}`
+ );
+
+ // Skeleton stats
+ const skelStats = getSkeletonStats(options.db);
+ console.log(chalk.bold.blue('\nSkeletons'));
+ console.log(chalk.dim('─'.repeat(50)));
+ console.log(` Pending: ${chalk.yellow(skelStats.pending.toString().padStart(6))}`);
+ console.log(` Completed: ${chalk.green(skelStats.completed.toString().padStart(6))}`);
+ console.log(` Failed: ${chalk.red(skelStats.failed.toString().padStart(6))}`);
+ console.log(
+ ` ${chalk.bold('Total:')} ${chalk.bold(skelStats.total.toString().padStart(7))}`
+ );
+
+ // Enrichment stats
+ const enrichStats = getEnrichmentStats(options.db);
+ console.log(chalk.bold.blue('\nEnriched Tools'));
+ console.log(chalk.dim('─'.repeat(50)));
+ console.log(` Quality: ${chalk.green(enrichStats.quality.toString().padStart(6))}`);
+ console.log(` Nonsensical: ${chalk.yellow(enrichStats.nonsensical.toString().padStart(6))}`);
+ console.log(
+ ` ${chalk.bold('Total:')} ${chalk.bold(enrichStats.totalIdeas.toString().padStart(7))}`
+ );
+ console.log(
+ ` Avg Score: ${chalk.cyan(enrichStats.avgQualityScore.toFixed(2).padStart(6))}`
+ );
+ console.log(
+ ` Total Cost: ${chalk.green(`$${enrichStats.totalCost.toFixed(2)}`.padStart(6))}`
+ );
+
+ // Progress bar
+ const progress =
+ skelStats.total > 0 ? Math.round((skelStats.completed / skelStats.total) * 100) : 0;
+ const filled = Math.round(progress / 2);
+ const bar = '█'.repeat(filled) + '░'.repeat(50 - filled);
+ console.log(chalk.bold.blue('\nProgress'));
+ console.log(chalk.dim('─'.repeat(50)));
+ console.log(` [${bar}] ${progress}%`);
+
+ // Detailed breakdowns
+ if (options.detailed) {
+ console.log(chalk.bold.blue('\nQuality Distribution'));
+ console.log(chalk.dim('─'.repeat(50)));
+
+ const qualityDist = db
+ .select({
+ bucket: sql`
+ CASE
+ WHEN quality_score >= 0.9 THEN '0.9-1.0'
+ WHEN quality_score >= 0.8 THEN '0.8-0.9'
+ WHEN quality_score >= 0.7 THEN '0.7-0.8'
+ WHEN quality_score >= 0.6 THEN '0.6-0.7'
+ WHEN quality_score >= 0.5 THEN '0.5-0.6'
+ ELSE '< 0.5'
+ END
+ `,
+ count: sql`count(*)`,
+ })
+ .from(toolIdeas)
+ .where(eq(toolIdeas.isNonsensical, false))
+ .groupBy(sql`1`)
+ .orderBy(desc(sql`1`))
+ .all();
+
+ for (const row of qualityDist) {
+ const barLen = Math.round((row.count / enrichStats.quality) * 30);
+ const bar = '█'.repeat(barLen);
+ console.log(` ${row.bucket}: ${bar} ${row.count}`);
+ }
+
+ // Top categories
+ console.log(chalk.bold.blue('\nTop Categories'));
+ console.log(chalk.dim('─'.repeat(50)));
+
+ const topCats = db
+ .select({
+ name: sql`substr(name, 1, instr(name, '.') - 1)`,
+ count: sql`count(*)`,
+ })
+ .from(toolIdeas)
+ .where(eq(toolIdeas.isNonsensical, false))
+ .groupBy(sql`1`)
+ .orderBy(desc(sql`2`))
+ .limit(10)
+ .all();
+
+ for (const row of topCats) {
+ console.log(` ${row.name.padEnd(20)} ${row.count}`);
+ }
+ }
+
+ console.log('');
+ } catch (error) {
+ console.error(chalk.red('Failed to get stats:'), error);
+ process.exit(1);
+ }
+ });
diff --git a/packages/tool-ideas/src/commands/vocab.ts b/packages/tool-ideas/src/commands/vocab.ts
new file mode 100644
index 0000000..d5c736a
--- /dev/null
+++ b/packages/tool-ideas/src/commands/vocab.ts
@@ -0,0 +1,88 @@
+import chalk from 'chalk';
+import { Command } from 'commander';
+import ora from 'ora';
+import { getDatabase } from '../db/client.js';
+import { seedCompatibilityRules } from '../generators/compatibility.js';
+import { getVocabularyStats, seedVocabulary } from '../generators/vocabulary.js';
+
+export const vocabCommand = new Command('vocab').description(
+ 'Manage vocabulary (categories, verbs, objects, contexts)'
+);
+
+vocabCommand
+ .command('generate')
+ .description('Generate vocabulary using AI (GPT-4.1-mini)')
+ .option('--db ', 'Database path', './data/tool-ideas.db')
+ .option('--categories ', 'Number of categories', '40')
+ .option('--verbs ', 'Number of verbs', '60')
+ .option('--objects ', 'Number of objects', '250')
+ .option('--contexts ', 'Number of contexts', '50')
+ .option('--qualifiers ', 'Number of qualifiers', '30')
+ .action(async (options) => {
+ const spinner = ora('Generating vocabulary with AI...').start();
+
+ try {
+ // Ensure database is initialized
+ getDatabase(options.db);
+
+ const counts = {
+ categories: Number.parseInt(options.categories),
+ verbs: Number.parseInt(options.verbs),
+ objects: Number.parseInt(options.objects),
+ contexts: Number.parseInt(options.contexts),
+ qualifiers: Number.parseInt(options.qualifiers),
+ };
+
+ spinner.text = `Generating ${counts.categories} categories...`;
+ const result = await seedVocabulary({
+ dbPath: options.db,
+ counts,
+ onProgress: (type, current, total) => {
+ spinner.text = `Generating ${type}: ${current}/${total}`;
+ },
+ });
+
+ spinner.succeed(chalk.green('Vocabulary generated successfully!'));
+ console.log(chalk.dim('Results:'));
+ console.log(` Categories: ${result.categories}`);
+ console.log(` Verbs: ${result.verbs}`);
+ console.log(` Objects: ${result.objects}`);
+ console.log(` Contexts: ${result.contexts}`);
+ console.log(` Qualifiers: ${result.qualifiers}`);
+ console.log(chalk.dim(` Total cost: $${result.totalCost.toFixed(4)}`));
+
+ // Generate compatibility rules
+ spinner.start('Generating compatibility rules...');
+ const compatResult = await seedCompatibilityRules({ dbPath: options.db });
+ spinner.succeed(chalk.green('Compatibility rules generated!'));
+ console.log(` Verb-object rules: ${compatResult.verbObjectRules}`);
+ console.log(` Category-verb rules: ${compatResult.categoryVerbRules}`);
+ } catch (error) {
+ spinner.fail(chalk.red('Failed to generate vocabulary'));
+ console.error(error);
+ process.exit(1);
+ }
+ });
+
+vocabCommand
+ .command('stats')
+ .description('Show vocabulary statistics')
+ .option('--db ', 'Database path', './data/tool-ideas.db')
+ .action(async (options) => {
+ try {
+ const stats = getVocabularyStats(options.db);
+
+ console.log(chalk.bold('\nVocabulary Statistics'));
+ console.log(chalk.dim('─'.repeat(40)));
+ console.log(` Categories: ${chalk.cyan(stats.categories)}`);
+ console.log(` Verbs: ${chalk.cyan(stats.verbs)}`);
+ console.log(` Objects: ${chalk.cyan(stats.objects)}`);
+ console.log(` Contexts: ${chalk.cyan(stats.contexts)}`);
+ console.log(` Qualifiers: ${chalk.cyan(stats.qualifiers)}`);
+ console.log(chalk.dim('─'.repeat(40)));
+ console.log(` Total: ${chalk.bold(stats.total)}`);
+ } catch (error) {
+ console.error(chalk.red('Failed to get stats:'), error);
+ process.exit(1);
+ }
+ });
diff --git a/packages/tool-ideas/src/db/client.ts b/packages/tool-ideas/src/db/client.ts
new file mode 100644
index 0000000..f9f5de2
--- /dev/null
+++ b/packages/tool-ideas/src/db/client.ts
@@ -0,0 +1,192 @@
+import { existsSync, mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import Database from 'better-sqlite3';
+import { drizzle } from 'drizzle-orm/better-sqlite3';
+import * as schema from './schema.js';
+
+let db: ReturnType> | null = null;
+let sqlite: Database.Database | null = null;
+
+/**
+ * Get or create the database connection
+ */
+export function getDatabase(dbPath = './data/tool-ideas.db') {
+ if (db) return db;
+
+ // Ensure directory exists
+ const dir = dirname(dbPath);
+ if (!existsSync(dir)) {
+ mkdirSync(dir, { recursive: true });
+ }
+
+ sqlite = new Database(dbPath);
+ sqlite.pragma('journal_mode = WAL');
+ sqlite.pragma('foreign_keys = ON');
+
+ db = drizzle(sqlite, { schema });
+
+ // Initialize tables if they don't exist
+ initializeTables(sqlite);
+
+ return db;
+}
+
+/**
+ * Close the database connection
+ */
+export function closeDatabase() {
+ if (sqlite) {
+ sqlite.close();
+ sqlite = null;
+ db = null;
+ }
+}
+
+/**
+ * Initialize database tables
+ */
+function initializeTables(sqlite: Database.Database) {
+ sqlite.exec(`
+ -- Categories
+ CREATE TABLE IF NOT EXISTS categories (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ name TEXT NOT NULL UNIQUE,
+ tpmjs_category TEXT NOT NULL,
+ description TEXT NOT NULL,
+ priority INTEGER NOT NULL DEFAULT 0
+ );
+ CREATE INDEX IF NOT EXISTS idx_categories_tpmjs ON categories(tpmjs_category);
+
+ -- Verbs
+ CREATE TABLE IF NOT EXISTS verbs (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ name TEXT NOT NULL UNIQUE,
+ past_tense TEXT,
+ gerund TEXT,
+ verb_type TEXT NOT NULL,
+ priority INTEGER NOT NULL DEFAULT 0
+ );
+ CREATE INDEX IF NOT EXISTS idx_verbs_type ON verbs(verb_type);
+
+ -- Objects
+ CREATE TABLE IF NOT EXISTS objects (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ name TEXT NOT NULL UNIQUE,
+ plural TEXT,
+ domain TEXT NOT NULL,
+ priority INTEGER NOT NULL DEFAULT 0
+ );
+ CREATE INDEX IF NOT EXISTS idx_objects_domain ON objects(domain);
+
+ -- Contexts
+ CREATE TABLE IF NOT EXISTS contexts (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ name TEXT NOT NULL UNIQUE,
+ context_type TEXT NOT NULL,
+ description TEXT
+ );
+ CREATE INDEX IF NOT EXISTS idx_contexts_type ON contexts(context_type);
+
+ -- Qualifiers
+ CREATE TABLE IF NOT EXISTS qualifiers (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ name TEXT NOT NULL UNIQUE,
+ qualifier_type TEXT NOT NULL,
+ description TEXT
+ );
+
+ -- Verb-Object Compatibility
+ CREATE TABLE IF NOT EXISTS verb_object_compatibility (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ verb_id INTEGER NOT NULL REFERENCES verbs(id),
+ object_id INTEGER NOT NULL REFERENCES objects(id),
+ score REAL NOT NULL DEFAULT 1.0,
+ reasoning TEXT
+ );
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_verb_object_unique ON verb_object_compatibility(verb_id, object_id);
+
+ -- Category-Verb Affinity
+ CREATE TABLE IF NOT EXISTS category_verb_affinity (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ category_id INTEGER NOT NULL REFERENCES categories(id),
+ verb_id INTEGER NOT NULL REFERENCES verbs(id),
+ score REAL NOT NULL DEFAULT 1.0
+ );
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_category_verb_unique ON category_verb_affinity(category_id, verb_id);
+
+ -- Tool Skeletons
+ CREATE TABLE IF NOT EXISTS tool_skeletons (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ hash TEXT NOT NULL UNIQUE,
+ category_id INTEGER NOT NULL REFERENCES categories(id),
+ verb_id INTEGER NOT NULL REFERENCES verbs(id),
+ object_id INTEGER NOT NULL REFERENCES objects(id),
+ context_id INTEGER REFERENCES contexts(id),
+ qualifier_ids TEXT,
+ raw_name TEXT NOT NULL,
+ compatibility_score REAL NOT NULL,
+ status TEXT NOT NULL DEFAULT 'pending',
+ generated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+ );
+ CREATE INDEX IF NOT EXISTS idx_skeletons_status ON tool_skeletons(status);
+ CREATE INDEX IF NOT EXISTS idx_skeletons_score ON tool_skeletons(compatibility_score);
+ CREATE INDEX IF NOT EXISTS idx_skeletons_category ON tool_skeletons(category_id);
+
+ -- Tool Ideas (enriched)
+ CREATE TABLE IF NOT EXISTS tool_ideas (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ skeleton_id INTEGER NOT NULL UNIQUE REFERENCES tool_skeletons(id),
+ name TEXT NOT NULL,
+ description TEXT NOT NULL,
+ parameters_json TEXT NOT NULL,
+ returns_json TEXT NOT NULL,
+ ai_agent_json TEXT,
+ tags_json TEXT,
+ examples_json TEXT,
+ is_nonsensical INTEGER NOT NULL DEFAULT 0,
+ nonsense_reason TEXT,
+ quality_score REAL,
+ model_used TEXT NOT NULL,
+ prompt_tokens INTEGER,
+ completion_tokens INTEGER,
+ processing_time_ms INTEGER,
+ enriched_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+ );
+ CREATE INDEX IF NOT EXISTS idx_ideas_quality ON tool_ideas(quality_score);
+ CREATE INDEX IF NOT EXISTS idx_ideas_nonsensical ON tool_ideas(is_nonsensical);
+
+ -- Processing Batches
+ CREATE TABLE IF NOT EXISTS processing_batches (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ batch_number INTEGER NOT NULL,
+ status TEXT NOT NULL DEFAULT 'pending',
+ skeleton_start_id INTEGER NOT NULL,
+ skeleton_end_id INTEGER NOT NULL,
+ total_count INTEGER NOT NULL,
+ processed_count INTEGER NOT NULL DEFAULT 0,
+ success_count INTEGER NOT NULL DEFAULT 0,
+ failed_count INTEGER NOT NULL DEFAULT 0,
+ nonsensical_count INTEGER NOT NULL DEFAULT 0,
+ started_at TEXT,
+ completed_at TEXT,
+ error_message TEXT,
+ cost_usd REAL
+ );
+ CREATE INDEX IF NOT EXISTS idx_batches_status ON processing_batches(status);
+
+ -- Processing Errors
+ CREATE TABLE IF NOT EXISTS processing_errors (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ skeleton_id INTEGER NOT NULL REFERENCES tool_skeletons(id),
+ batch_id INTEGER REFERENCES processing_batches(id),
+ error_type TEXT NOT NULL,
+ error_message TEXT NOT NULL,
+ retry_count INTEGER NOT NULL DEFAULT 0,
+ created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+ );
+ CREATE INDEX IF NOT EXISTS idx_errors_skeleton ON processing_errors(skeleton_id);
+ `);
+}
+
+export type Database = ReturnType;
+export { schema };
diff --git a/packages/tool-ideas/src/db/schema.ts b/packages/tool-ideas/src/db/schema.ts
new file mode 100644
index 0000000..e48a19e
--- /dev/null
+++ b/packages/tool-ideas/src/db/schema.ts
@@ -0,0 +1,262 @@
+import { sql } from 'drizzle-orm';
+import { index, integer, real, sqliteTable, text, uniqueIndex } from 'drizzle-orm/sqlite-core';
+
+// =============================================================================
+// VOCABULARY TABLES
+// =============================================================================
+
+/**
+ * Categories - maps to TPMJS_CATEGORIES
+ */
+export const categories = sqliteTable(
+ 'categories',
+ {
+ id: integer('id').primaryKey({ autoIncrement: true }),
+ name: text('name').notNull().unique(),
+ tpmjsCategory: text('tpmjs_category').notNull(),
+ description: text('description').notNull(),
+ priority: integer('priority').notNull().default(0), // Higher = more important
+ },
+ (table) => [index('idx_categories_tpmjs').on(table.tpmjsCategory)]
+);
+
+/**
+ * Verbs - action words for tools
+ */
+export const verbs = sqliteTable(
+ 'verbs',
+ {
+ id: integer('id').primaryKey({ autoIncrement: true }),
+ name: text('name').notNull().unique(),
+ pastTense: text('past_tense'),
+ gerund: text('gerund'),
+ verbType: text('verb_type').notNull(), // action, analysis, transformation, detection, extraction, validation, aggregation, prediction, management
+ priority: integer('priority').notNull().default(0),
+ },
+ (table) => [index('idx_verbs_type').on(table.verbType)]
+);
+
+/**
+ * Objects - nouns that tools operate on
+ */
+export const objects = sqliteTable(
+ 'objects',
+ {
+ id: integer('id').primaryKey({ autoIncrement: true }),
+ name: text('name').notNull().unique(),
+ plural: text('plural'),
+ domain: text('domain').notNull(), // document, code, data, media, business, security, communication, etc.
+ priority: integer('priority').notNull().default(0),
+ },
+ (table) => [index('idx_objects_domain').on(table.domain)]
+);
+
+/**
+ * Contexts - optional modifiers for tools
+ */
+export const contexts = sqliteTable(
+ 'contexts',
+ {
+ id: integer('id').primaryKey({ autoIncrement: true }),
+ name: text('name').notNull().unique(),
+ contextType: text('context_type').notNull(), // workflow, platform, industry, constraint
+ description: text('description'),
+ },
+ (table) => [index('idx_contexts_type').on(table.contextType)]
+);
+
+/**
+ * Qualifiers - additional modifiers
+ */
+export const qualifiers = sqliteTable('qualifiers', {
+ id: integer('id').primaryKey({ autoIncrement: true }),
+ name: text('name').notNull().unique(),
+ qualifierType: text('qualifier_type').notNull(), // temporal, scope, format, source, mode
+ description: text('description'),
+});
+
+// =============================================================================
+// COMPATIBILITY TABLES
+// =============================================================================
+
+/**
+ * Verb-Object compatibility - which verbs work with which objects
+ */
+export const verbObjectCompatibility = sqliteTable(
+ 'verb_object_compatibility',
+ {
+ id: integer('id').primaryKey({ autoIncrement: true }),
+ verbId: integer('verb_id')
+ .notNull()
+ .references(() => verbs.id),
+ objectId: integer('object_id')
+ .notNull()
+ .references(() => objects.id),
+ score: real('score').notNull().default(1.0), // 0.0 to 1.0
+ reasoning: text('reasoning'),
+ },
+ (table) => [uniqueIndex('idx_verb_object_unique').on(table.verbId, table.objectId)]
+);
+
+/**
+ * Category-Verb affinity - which verbs fit which categories
+ */
+export const categoryVerbAffinity = sqliteTable(
+ 'category_verb_affinity',
+ {
+ id: integer('id').primaryKey({ autoIncrement: true }),
+ categoryId: integer('category_id')
+ .notNull()
+ .references(() => categories.id),
+ verbId: integer('verb_id')
+ .notNull()
+ .references(() => verbs.id),
+ score: real('score').notNull().default(1.0),
+ },
+ (table) => [uniqueIndex('idx_category_verb_unique').on(table.categoryId, table.verbId)]
+);
+
+// =============================================================================
+// GENERATED DATA TABLES
+// =============================================================================
+
+/**
+ * Tool skeletons - raw generated combinations before enrichment
+ */
+export const toolSkeletons = sqliteTable(
+ 'tool_skeletons',
+ {
+ id: integer('id').primaryKey({ autoIncrement: true }),
+ hash: text('hash').notNull().unique(), // SHA256 for deduplication
+ categoryId: integer('category_id')
+ .notNull()
+ .references(() => categories.id),
+ verbId: integer('verb_id')
+ .notNull()
+ .references(() => verbs.id),
+ objectId: integer('object_id')
+ .notNull()
+ .references(() => objects.id),
+ contextId: integer('context_id').references(() => contexts.id),
+ qualifierIds: text('qualifier_ids'), // JSON array of qualifier IDs
+ rawName: text('raw_name').notNull(), // e.g., "data.parseCSV"
+ compatibilityScore: real('compatibility_score').notNull(),
+ status: text('status').notNull().default('pending'), // pending, processing, completed, failed, skipped
+ generatedAt: text('generated_at').notNull().default(sql`CURRENT_TIMESTAMP`),
+ },
+ (table) => [
+ index('idx_skeletons_status').on(table.status),
+ index('idx_skeletons_score').on(table.compatibilityScore),
+ index('idx_skeletons_category').on(table.categoryId),
+ ]
+);
+
+/**
+ * Enriched tool ideas - fully fleshed out by GPT
+ */
+export const toolIdeas = sqliteTable(
+ 'tool_ideas',
+ {
+ id: integer('id').primaryKey({ autoIncrement: true }),
+ skeletonId: integer('skeleton_id')
+ .notNull()
+ .unique()
+ .references(() => toolSkeletons.id),
+
+ // Core tool spec fields
+ name: text('name').notNull(), // category.verbObject
+ description: text('description').notNull(),
+ parametersJson: text('parameters_json').notNull(), // JSON array
+ returnsJson: text('returns_json').notNull(), // JSON object
+ aiAgentJson: text('ai_agent_json'), // JSON object: useCase, limitations, examples
+ tagsJson: text('tags_json'), // JSON array
+ examplesJson: text('examples_json'), // JSON array
+
+ // Quality metadata
+ isNonsensical: integer('is_nonsensical', { mode: 'boolean' }).notNull().default(false),
+ nonsenseReason: text('nonsense_reason'),
+ qualityScore: real('quality_score'),
+
+ // Processing metadata
+ modelUsed: text('model_used').notNull(),
+ promptTokens: integer('prompt_tokens'),
+ completionTokens: integer('completion_tokens'),
+ processingTimeMs: integer('processing_time_ms'),
+ enrichedAt: text('enriched_at').notNull().default(sql`CURRENT_TIMESTAMP`),
+ },
+ (table) => [
+ index('idx_ideas_quality').on(table.qualityScore),
+ index('idx_ideas_nonsensical').on(table.isNonsensical),
+ ]
+);
+
+/**
+ * Processing batches - track enrichment progress
+ */
+export const processingBatches = sqliteTable(
+ 'processing_batches',
+ {
+ id: integer('id').primaryKey({ autoIncrement: true }),
+ batchNumber: integer('batch_number').notNull(),
+ status: text('status').notNull().default('pending'), // pending, processing, completed, failed
+ skeletonStartId: integer('skeleton_start_id').notNull(),
+ skeletonEndId: integer('skeleton_end_id').notNull(),
+ totalCount: integer('total_count').notNull(),
+ processedCount: integer('processed_count').notNull().default(0),
+ successCount: integer('success_count').notNull().default(0),
+ failedCount: integer('failed_count').notNull().default(0),
+ nonsensicalCount: integer('nonsensical_count').notNull().default(0),
+ startedAt: text('started_at'),
+ completedAt: text('completed_at'),
+ errorMessage: text('error_message'),
+ costUsd: real('cost_usd'),
+ },
+ (table) => [index('idx_batches_status').on(table.status)]
+);
+
+/**
+ * Processing errors - for retry logic
+ */
+export const processingErrors = sqliteTable(
+ 'processing_errors',
+ {
+ id: integer('id').primaryKey({ autoIncrement: true }),
+ skeletonId: integer('skeleton_id')
+ .notNull()
+ .references(() => toolSkeletons.id),
+ batchId: integer('batch_id').references(() => processingBatches.id),
+ errorType: text('error_type').notNull(),
+ errorMessage: text('error_message').notNull(),
+ retryCount: integer('retry_count').notNull().default(0),
+ createdAt: text('created_at').notNull().default(sql`CURRENT_TIMESTAMP`),
+ },
+ (table) => [index('idx_errors_skeleton').on(table.skeletonId)]
+);
+
+// =============================================================================
+// TYPES
+// =============================================================================
+
+export type Category = typeof categories.$inferSelect;
+export type NewCategory = typeof categories.$inferInsert;
+
+export type Verb = typeof verbs.$inferSelect;
+export type NewVerb = typeof verbs.$inferInsert;
+
+export type ToolObject = typeof objects.$inferSelect;
+export type NewToolObject = typeof objects.$inferInsert;
+
+export type Context = typeof contexts.$inferSelect;
+export type NewContext = typeof contexts.$inferInsert;
+
+export type Qualifier = typeof qualifiers.$inferSelect;
+export type NewQualifier = typeof qualifiers.$inferInsert;
+
+export type ToolSkeleton = typeof toolSkeletons.$inferSelect;
+export type NewToolSkeleton = typeof toolSkeletons.$inferInsert;
+
+export type ToolIdea = typeof toolIdeas.$inferSelect;
+export type NewToolIdea = typeof toolIdeas.$inferInsert;
+
+export type ProcessingBatch = typeof processingBatches.$inferSelect;
+export type NewProcessingBatch = typeof processingBatches.$inferInsert;
diff --git a/packages/tool-ideas/src/enrichment/batch-processor.ts b/packages/tool-ideas/src/enrichment/batch-processor.ts
new file mode 100644
index 0000000..38155f8
--- /dev/null
+++ b/packages/tool-ideas/src/enrichment/batch-processor.ts
@@ -0,0 +1,411 @@
+import { openai } from '@ai-sdk/openai';
+import { generateObject } from 'ai';
+import { and, eq, inArray, sql } from 'drizzle-orm';
+import pLimit from 'p-limit';
+import { getDatabase } from '../db/client.js';
+import {
+ type NewToolIdea,
+ type ToolSkeleton,
+ categories,
+ contexts,
+ objects,
+ processingBatches,
+ processingErrors,
+ toolIdeas,
+ toolSkeletons,
+ verbs,
+} from '../db/schema.js';
+import { type SkeletonWithRelations, createEnrichmentPrompt } from './prompts.js';
+import { type EnrichedTool, EnrichedToolSchema } from './schemas.js';
+
+// =============================================================================
+// BATCH PROCESSOR OPTIONS
+// =============================================================================
+
+export interface BatchProcessorOptions {
+ dbPath?: string;
+ batchSize?: number;
+ concurrency?: number;
+ maxRetries?: number;
+ retryDelayMs?: number;
+ costLimitUsd?: number;
+ model?: string;
+ onProgress?: (processed: number, total: number, cost: number) => void;
+ onError?: (error: Error, skeletonId: number) => void;
+}
+
+const DEFAULT_OPTIONS: Required> =
+ {
+ batchSize: 100,
+ concurrency: 5,
+ maxRetries: 3,
+ retryDelayMs: 1000,
+ costLimitUsd: 50,
+ model: 'gpt-4.1-mini',
+ };
+
+// =============================================================================
+// PRICING (GPT-4.1-mini)
+// =============================================================================
+
+const PRICING = {
+ inputPerToken: 0.15 / 1_000_000, // $0.15 per 1M tokens
+ outputPerToken: 0.6 / 1_000_000, // $0.60 per 1M tokens
+};
+
+function calculateCost(promptTokens: number, completionTokens: number): number {
+ return promptTokens * PRICING.inputPerToken + completionTokens * PRICING.outputPerToken;
+}
+
+// =============================================================================
+// BATCH PROCESSOR
+// =============================================================================
+
+export class BatchProcessor {
+ private options: Required>;
+ private dbPath?: string;
+ private onProgress?: (processed: number, total: number, cost: number) => void;
+ private onError?: (error: Error, skeletonId: number) => void;
+ private totalCost = 0;
+ private processedCount = 0;
+
+ constructor(options: BatchProcessorOptions = {}) {
+ this.options = { ...DEFAULT_OPTIONS, ...options };
+ this.dbPath = options.dbPath;
+ this.onProgress = options.onProgress;
+ this.onError = options.onError;
+ }
+
+ /**
+ * Process the next batch of pending skeletons
+ */
+ async processNextBatch(): Promise<{
+ success: boolean;
+ processed: number;
+ failed: number;
+ nonsensical: number;
+ cost: number;
+ message: string;
+ }> {
+ const db = getDatabase(this.dbPath);
+
+ // Check cost limit
+ if (this.totalCost >= this.options.costLimitUsd) {
+ return {
+ success: false,
+ processed: 0,
+ failed: 0,
+ nonsensical: 0,
+ cost: this.totalCost,
+ message: `Cost limit reached: $${this.totalCost.toFixed(2)}`,
+ };
+ }
+
+ // Get pending skeletons
+ const pendingSkeletons = db
+ .select()
+ .from(toolSkeletons)
+ .where(eq(toolSkeletons.status, 'pending'))
+ .limit(this.options.batchSize)
+ .all();
+
+ if (pendingSkeletons.length === 0) {
+ return {
+ success: true,
+ processed: 0,
+ failed: 0,
+ nonsensical: 0,
+ cost: this.totalCost,
+ message: 'No pending skeletons',
+ };
+ }
+
+ // Load relations for each skeleton
+ const skeletonsWithRelations = await this.loadSkeletonRelations(db, pendingSkeletons);
+
+ // Process with concurrency limit
+ const limit = pLimit(this.options.concurrency);
+ let successCount = 0;
+ let failedCount = 0;
+ let nonsensicalCount = 0;
+ let batchCost = 0;
+
+ await Promise.all(
+ skeletonsWithRelations.map((skeleton) =>
+ limit(async () => {
+ try {
+ const result = await this.processSkeleton(db, skeleton);
+ successCount++;
+ batchCost += result.cost;
+ if (result.isNonsensical) nonsensicalCount++;
+ } catch (error) {
+ failedCount++;
+ if (this.onError) {
+ this.onError(error as Error, skeleton.id);
+ }
+ await this.logError(db, skeleton.id, error as Error);
+ }
+ })
+ )
+ );
+
+ this.totalCost += batchCost;
+ this.processedCount += successCount;
+
+ if (this.onProgress) {
+ const totalPending =
+ db
+ .select({ count: sql`count(*)` })
+ .from(toolSkeletons)
+ .where(eq(toolSkeletons.status, 'pending'))
+ .get()?.count ?? 0;
+ this.onProgress(this.processedCount, this.processedCount + totalPending, this.totalCost);
+ }
+
+ return {
+ success: true,
+ processed: successCount,
+ failed: failedCount,
+ nonsensical: nonsensicalCount,
+ cost: batchCost,
+ message: `Processed ${successCount}/${pendingSkeletons.length}, cost: $${batchCost.toFixed(4)}`,
+ };
+ }
+
+ /**
+ * Process continuously until done or cost limit reached
+ */
+ async processAll(): Promise<{
+ totalProcessed: number;
+ totalFailed: number;
+ totalNonsensical: number;
+ totalCost: number;
+ }> {
+ let totalProcessed = 0;
+ let totalFailed = 0;
+ let totalNonsensical = 0;
+
+ while (true) {
+ const result = await this.processNextBatch();
+
+ totalProcessed += result.processed;
+ totalFailed += result.failed;
+ totalNonsensical += result.nonsensical;
+
+ if (!result.success || result.processed === 0) {
+ break;
+ }
+
+ // Check cost limit
+ if (this.totalCost >= this.options.costLimitUsd) {
+ console.log(`Cost limit reached: $${this.totalCost.toFixed(2)}`);
+ break;
+ }
+ }
+
+ return {
+ totalProcessed,
+ totalFailed,
+ totalNonsensical,
+ totalCost: this.totalCost,
+ };
+ }
+
+ /**
+ * Load skeleton relations from database
+ */
+ private async loadSkeletonRelations(
+ db: ReturnType,
+ skeletons: ToolSkeleton[]
+ ): Promise {
+ const categoryIds = [...new Set(skeletons.map((s) => s.categoryId))];
+ const verbIds = [...new Set(skeletons.map((s) => s.verbId))];
+ const objectIds = [...new Set(skeletons.map((s) => s.objectId))];
+ const contextIds = [...new Set(skeletons.map((s) => s.contextId).filter(Boolean))] as number[];
+
+ const categoryMap = new Map(
+ db
+ .select()
+ .from(categories)
+ .where(inArray(categories.id, categoryIds))
+ .all()
+ .map((c) => [c.id, c])
+ );
+ const verbMap = new Map(
+ db
+ .select()
+ .from(verbs)
+ .where(inArray(verbs.id, verbIds))
+ .all()
+ .map((v) => [v.id, v])
+ );
+ const objectMap = new Map(
+ db
+ .select()
+ .from(objects)
+ .where(inArray(objects.id, objectIds))
+ .all()
+ .map((o) => [o.id, o])
+ );
+ const contextMap =
+ contextIds.length > 0
+ ? new Map(
+ db
+ .select()
+ .from(contexts)
+ .where(inArray(contexts.id, contextIds))
+ .all()
+ .map((c) => [c.id, c])
+ )
+ : new Map();
+
+ return skeletons.map((s) => ({
+ ...s,
+ category: categoryMap.get(s.categoryId)!,
+ verb: verbMap.get(s.verbId)!,
+ object: objectMap.get(s.objectId)!,
+ context: s.contextId ? (contextMap.get(s.contextId) ?? null) : null,
+ }));
+ }
+
+ /**
+ * Process a single skeleton
+ */
+ private async processSkeleton(
+ db: ReturnType,
+ skeleton: SkeletonWithRelations
+ ): Promise<{ cost: number; isNonsensical: boolean }> {
+ const startTime = Date.now();
+
+ // Mark as processing
+ db.update(toolSkeletons)
+ .set({ status: 'processing' })
+ .where(eq(toolSkeletons.id, skeleton.id))
+ .run();
+
+ try {
+ const prompt = createEnrichmentPrompt(skeleton);
+
+ const result = await generateObject({
+ model: openai(this.options.model),
+ schema: EnrichedToolSchema,
+ prompt,
+ maxRetries: this.options.maxRetries,
+ });
+
+ const enriched = result.object;
+ const processingTime = Date.now() - startTime;
+ const promptTokens = result.usage?.promptTokens ?? 0;
+ const completionTokens = result.usage?.completionTokens ?? 0;
+ const cost = calculateCost(promptTokens, completionTokens);
+
+ // Save enriched tool
+ const toolIdea: NewToolIdea = {
+ skeletonId: skeleton.id,
+ name: enriched.name,
+ description: enriched.description,
+ parametersJson: JSON.stringify(enriched.parameters),
+ returnsJson: JSON.stringify(enriched.returns),
+ aiAgentJson: JSON.stringify(enriched.aiAgent),
+ tagsJson: JSON.stringify(enriched.tags),
+ examplesJson: JSON.stringify(enriched.examples),
+ isNonsensical: enriched.isNonsensical,
+ nonsenseReason: enriched.nonsenseReason ?? null,
+ qualityScore: enriched.qualityScore,
+ modelUsed: this.options.model,
+ promptTokens,
+ completionTokens,
+ processingTimeMs: processingTime,
+ enrichedAt: new Date().toISOString(),
+ };
+
+ db.insert(toolIdeas).values(toolIdea).run();
+
+ // Mark skeleton as completed
+ db.update(toolSkeletons)
+ .set({ status: 'completed' })
+ .where(eq(toolSkeletons.id, skeleton.id))
+ .run();
+
+ return { cost, isNonsensical: enriched.isNonsensical };
+ } catch (error) {
+ // Mark skeleton as failed
+ db.update(toolSkeletons)
+ .set({ status: 'failed' })
+ .where(eq(toolSkeletons.id, skeleton.id))
+ .run();
+
+ throw error;
+ }
+ }
+
+ /**
+ * Log processing error
+ */
+ private async logError(db: ReturnType, skeletonId: number, error: Error) {
+ db.insert(processingErrors)
+ .values({
+ skeletonId,
+ batchId: null,
+ errorType: error.name,
+ errorMessage: error.message,
+ retryCount: 0,
+ createdAt: new Date().toISOString(),
+ })
+ .run();
+ }
+
+ /**
+ * Get processing stats
+ */
+ getStats() {
+ return {
+ processedCount: this.processedCount,
+ totalCost: this.totalCost,
+ };
+ }
+}
+
+// =============================================================================
+// STATS HELPER
+// =============================================================================
+
+export function getEnrichmentStats(dbPath?: string) {
+ const db = getDatabase(dbPath);
+
+ const totalIdeas = db.select({ count: sql`count(*)` }).from(toolIdeas).get()?.count ?? 0;
+ const nonsensical =
+ db
+ .select({ count: sql`count(*)` })
+ .from(toolIdeas)
+ .where(eq(toolIdeas.isNonsensical, true))
+ .get()?.count ?? 0;
+
+ const avgQuality =
+ db
+ .select({ avg: sql`avg(quality_score)` })
+ .from(toolIdeas)
+ .where(eq(toolIdeas.isNonsensical, false))
+ .get()?.avg ?? 0;
+
+ const totalTokens = db
+ .select({
+ promptTokens: sql`sum(prompt_tokens)`,
+ completionTokens: sql`sum(completion_tokens)`,
+ })
+ .from(toolIdeas)
+ .get();
+
+ const totalCost = calculateCost(
+ totalTokens?.promptTokens ?? 0,
+ totalTokens?.completionTokens ?? 0
+ );
+
+ return {
+ totalIdeas,
+ nonsensical,
+ quality: totalIdeas - nonsensical,
+ avgQualityScore: avgQuality,
+ totalCost,
+ };
+}
diff --git a/packages/tool-ideas/src/enrichment/prompts.ts b/packages/tool-ideas/src/enrichment/prompts.ts
new file mode 100644
index 0000000..772d341
--- /dev/null
+++ b/packages/tool-ideas/src/enrichment/prompts.ts
@@ -0,0 +1,132 @@
+import type { Category, Context, ToolObject, Verb } from '../db/schema.js';
+
+// =============================================================================
+// SKELETON WITH LOADED RELATIONS
+// =============================================================================
+
+export interface SkeletonWithRelations {
+ id: number;
+ hash: string;
+ rawName: string;
+ compatibilityScore: number;
+ category: Category;
+ verb: Verb;
+ object: ToolObject;
+ context: Context | null;
+}
+
+// =============================================================================
+// SINGLE TOOL ENRICHMENT PROMPT
+// =============================================================================
+
+export function createEnrichmentPrompt(skeleton: SkeletonWithRelations): string {
+ return `You are designing a realistic AI tool for the TPMJS tool registry. Create a complete, practical tool specification.
+
+## Tool Skeleton
+- **Category**: ${skeleton.category.name} (${skeleton.category.description})
+- **Verb**: ${skeleton.verb.name} (${skeleton.verb.verbType} verb, gerund: ${skeleton.verb.gerund})
+- **Object**: ${skeleton.object.name} (${skeleton.object.domain} domain)
+${skeleton.context ? `- **Context**: ${skeleton.context.name} (${skeleton.context.contextType})` : ''}
+- **Raw Name**: ${skeleton.rawName}
+- **Compatibility Score**: ${skeleton.compatibilityScore.toFixed(2)}
+
+## Requirements
+
+### 1. Name (MUST follow format)
+Use format: \`category.verbObject\`
+Examples: \`data.parseCSV\`, \`security.scanVulnerabilities\`, \`docs.generateChangelog\`
+
+### 2. Description (50-500 chars)
+Explain what the tool does in clear, practical terms. Focus on:
+- What input it accepts
+- What processing it performs
+- What output it produces
+
+### 3. Parameters (1-10 typed inputs)
+Design practical parameters an agent would need:
+- Use camelCase names
+- Include types: string, number, boolean, array, object
+- Mark required: true or false
+- defaultValue: string representation of default (use empty string "" if none)
+
+### 4. Returns
+Describe the output structure the tool produces.
+
+### 5. AI Agent Guidance
+Help AI agents understand when to use this tool:
+- useCase: Detailed explanation of scenarios (30-500 chars)
+- limitations: What it can't do (optional)
+- examples: 1-3 example user requests
+
+### 6. Tags (2-8)
+Keywords for discovery: action type, domain, use case.
+
+### 7. Examples (1-3)
+Realistic input examples with descriptions. Use inputJson field with valid JSON string.
+
+## Quality Assessment
+
+Evaluate if this tool makes practical sense:
+- **1.0**: Highly practical, clear use case, well-defined I/O
+- **0.7-0.9**: Practical with some edge cases
+- **0.4-0.6**: Niche use case but valid
+- **0.1-0.3**: Marginal utility
+- **0.0**: Nonsensical combination
+
+If the combination doesn't make sense (e.g., "parse + Meeting" or "transcribe + JSON"):
+- Set \`isNonsensical: true\`
+- Set \`nonsenseReason\` to explain why
+- Still fill all other fields with best effort
+
+Note: ALL fields are required. Use empty string "" for optional text fields when not applicable.
+
+Respond with valid JSON matching the schema.`;
+}
+
+// =============================================================================
+// BATCH ENRICHMENT PROMPT
+// =============================================================================
+
+export function createBatchEnrichmentPrompt(skeletons: SkeletonWithRelations[]): string {
+ const skeletonList = skeletons
+ .map(
+ (s, i) => `
+### Tool ${i + 1}
+- **ID**: ${s.id}
+- **Category**: ${s.category.name} (${s.category.description})
+- **Verb**: ${s.verb.name} (${s.verb.verbType})
+- **Object**: ${s.object.name} (${s.object.domain})
+${s.context ? `- **Context**: ${s.context.name}` : ''}
+- **Raw Name**: ${s.rawName}
+- **Score**: ${s.compatibilityScore.toFixed(2)}
+`
+ )
+ .join('\n');
+
+ return `You are designing realistic AI tools for the TPMJS tool registry. Create complete specifications for ${skeletons.length} tools.
+
+## Tool Skeletons
+${skeletonList}
+
+## Requirements for EACH tool
+
+1. **Name**: category.verbObject format (e.g., data.parseCSV)
+2. **Description**: 50-500 chars, practical explanation
+3. **Parameters**: 1-10 typed inputs with descriptions
+4. **Returns**: Output type and description
+5. **AI Agent Guidance**: useCase (when to use), limitations, examples
+6. **Tags**: 2-8 keywords for discovery
+7. **Examples**: 1-3 realistic usage examples
+
+## Quality Scoring
+
+- 1.0: Highly practical, production-ready concept
+- 0.7-0.9: Good use case with minor limitations
+- 0.4-0.6: Niche but valid
+- 0.1-0.3: Marginal utility
+- 0.0: Nonsensical
+
+Mark nonsensical combinations with isNonsensical=true and explain why.
+
+Return an array of ${skeletons.length} tool specifications.`;
+}
diff --git a/packages/tool-ideas/src/enrichment/schemas.ts b/packages/tool-ideas/src/enrichment/schemas.ts
new file mode 100644
index 0000000..87486d3
--- /dev/null
+++ b/packages/tool-ideas/src/enrichment/schemas.ts
@@ -0,0 +1,89 @@
+import { z } from 'zod';
+
+// =============================================================================
+// TOOL PARAMETER SCHEMA (all fields required for OpenAI structured output)
+// =============================================================================
+
+export const ToolParameterSchema = z.object({
+ name: z.string().describe('camelCase parameter name'),
+ type: z.enum(['string', 'number', 'boolean', 'array', 'object']).describe('Parameter type'),
+ description: z.string().describe('What this parameter does (10-200 chars)'),
+ required: z.boolean().describe('Whether this parameter is required'),
+ defaultValue: z.string().describe('Default value as string, or empty string if none'),
+});
+
+export type ToolParameter = z.infer;
+
+// =============================================================================
+// TOOL RETURNS SCHEMA
+// =============================================================================
+
+export const ToolReturnsSchema = z.object({
+ type: z.string().describe('Return type name (e.g., ParsedData, ValidationResult)'),
+ description: z.string().describe('What the tool returns (10-200 chars)'),
+});
+
+export type ToolReturns = z.infer;
+
+// =============================================================================
+// AI AGENT GUIDANCE SCHEMA (all fields required)
+// =============================================================================
+
+export const AIAgentSchema = z.object({
+ useCase: z.string().describe('When and why an AI agent should use this tool (30-500 chars)'),
+ limitations: z.string().describe('What this tool cannot do, or empty string if none'),
+ examples: z.array(z.string()).describe('1-3 example user requests'),
+});
+
+export type AIAgent = z.infer;
+
+// =============================================================================
+// TOOL EXAMPLE SCHEMA
+// =============================================================================
+
+export const ToolExampleSchema = z.object({
+ inputJson: z.string().describe('Example input parameters as JSON string'),
+ description: z.string().describe('What this example demonstrates (max 100 chars)'),
+});
+
+export type ToolExample = z.infer;
+
+// =============================================================================
+// ENRICHED TOOL SCHEMA (main output from GPT - all fields required)
+// =============================================================================
+
+export const EnrichedToolSchema = z.object({
+ name: z.string().describe('Tool name in category.verbObject format (e.g., data.parseCSV)'),
+
+ description: z.string().describe('Clear description of what the tool does (50-500 chars)'),
+
+ parameters: z.array(ToolParameterSchema).describe('1-10 input parameters for the tool'),
+
+ returns: ToolReturnsSchema.describe('What the tool returns'),
+
+ aiAgent: AIAgentSchema.describe('Guidance for AI agents using this tool'),
+
+ tags: z.array(z.string()).describe('2-8 descriptive tags for discovery'),
+
+ examples: z.array(ToolExampleSchema).describe('1-3 usage examples'),
+
+ isNonsensical: z.boolean().describe('True if this tool concept does not make practical sense'),
+
+ nonsenseReason: z
+ .string()
+ .describe('If nonsensical, explain why. Empty string if not nonsensical'),
+
+ qualityScore: z.number().describe('Quality score 0-1: 1=highly practical, 0=nonsensical'),
+});
+
+export type EnrichedTool = z.infer;
+
+// =============================================================================
+// BATCH ENRICHMENT SCHEMA (for processing multiple skeletons)
+// =============================================================================
+
+export const BatchEnrichmentSchema = z.object({
+ tools: z.array(EnrichedToolSchema).min(1).max(10),
+});
+
+export type BatchEnrichment = z.infer;
diff --git a/packages/tool-ideas/src/generators/compatibility.ts b/packages/tool-ideas/src/generators/compatibility.ts
new file mode 100644
index 0000000..706e386
--- /dev/null
+++ b/packages/tool-ideas/src/generators/compatibility.ts
@@ -0,0 +1,307 @@
+import { openai } from '@ai-sdk/openai';
+import { generateObject } from 'ai';
+import { z } from 'zod';
+import { getDatabase } from '../db/client.js';
+import {
+ type Category,
+ type ToolObject,
+ type Verb,
+ categories,
+ categoryVerbAffinity,
+ objects,
+ verbObjectCompatibility,
+ verbs,
+} from '../db/schema.js';
+// Drizzle operators imported as needed
+
+// =============================================================================
+// COMPATIBILITY SCHEMAS
+// =============================================================================
+
+const VerbObjectRuleSchema = z.object({
+ verbName: z.string(),
+ compatibleObjects: z.array(z.string()).describe('Objects this verb works well with'),
+ incompatibleObjects: z.array(z.string()).describe('Objects this verb does NOT work with'),
+});
+
+const CategoryVerbRuleSchema = z.object({
+ categoryName: z.string(),
+ preferredVerbs: z.array(z.string()).describe('Verbs that fit well with this category'),
+ score: z.number().min(0.5).max(1.0).describe('Affinity score'),
+});
+
+// =============================================================================
+// GENERATE COMPATIBILITY RULES WITH AI
+// =============================================================================
+
+export async function generateVerbObjectRules(
+ verbList: Verb[],
+ objectList: ToolObject[]
+): Promise