From 3b955025776415b0c098e48c4aab609fecb578b2 Mon Sep 17 00:00:00 2001 From: Ajax Davis Date: Fri, 28 Nov 2025 02:57:54 +1000 Subject: [PATCH] feat(sync): implement Phase 3 Sync Workers with Vercel Cron integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add complete sync worker system for NPM package discovery and metrics: **Sync Workers:** - Changes Feed Sync (/api/sync/changes) - Polls NPM changes every 2 min - Keyword Search Sync (/api/sync/keyword) - Searches tpmjs-tool keyword every 15 min - Metrics Sync (/api/sync/metrics) - Updates downloads & quality scores hourly **Features:** - Secure CRON_SECRET authentication for all sync endpoints - Comprehensive error handling with sync logs and checkpoints - Smart package validation and filtering (skip invalid tpmjs fields) - Automatic tool upsert with discovery method tracking - Quality score calculation based on tier, downloads, and GitHub stars - 5-minute timeout support for long-running sync operations **Dependencies:** - Add @tpmjs/npm-client to web app for NPM API integration - Use barrel exports from npm-client package (no subpath imports) - Add CRON_SECRET env variable validation - Add ~/src path alias to tsconfig **Infrastructure:** - Configure Vercel Cron jobs in vercel.json for automated syncing - Add publishedAt field to PackageVersion schema - Fix Prisma JSON field handling (use undefined instead of null) - Proper null checks for fetchLatestPackageVersion return values **Type Safety:** - Cast TpmjsField union type to access optional rich-tier properties - Handle searchByKeyword array return type correctly - Fix fetchDownloadStats to return number directly All API routes follow Next.js 16 conventions with proper type checking. Type-check and full build successful. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- NPM_MIRROR.md | 500 +++++++++++++++++++++ apps/web/package.json | 1 + apps/web/src/app/api/sync/changes/route.ts | 218 +++++++++ apps/web/src/app/api/sync/keyword/route.ts | 206 +++++++++ apps/web/src/app/api/sync/metrics/route.ts | 177 ++++++++ apps/web/src/env.ts | 1 + apps/web/tsconfig.json | 3 +- packages/npm-client/package.json | 8 + packages/npm-client/src/package.ts | 15 +- pnpm-lock.yaml | 214 ++------- supabase/.gitignore | 8 + supabase/config.toml | 347 ++++++++++++++ vercel.json | 16 +- 13 files changed, 1540 insertions(+), 174 deletions(-) create mode 100644 NPM_MIRROR.md create mode 100644 apps/web/src/app/api/sync/changes/route.ts create mode 100644 apps/web/src/app/api/sync/keyword/route.ts create mode 100644 apps/web/src/app/api/sync/metrics/route.ts create mode 100644 supabase/.gitignore create mode 100644 supabase/config.toml diff --git a/NPM_MIRROR.md b/NPM_MIRROR.md new file mode 100644 index 0000000..61d05cb --- /dev/null +++ b/NPM_MIRROR.md @@ -0,0 +1,500 @@ +# TPMJS NPM-Integrated Registry Architecture + +> **Automated tool discovery from NPM with zero-click submission** + +## Vision + +Transform TPMJS from a manual directory into an **automated NPM-integrated registry** where package authors simply publish to NPM with a `tpmjs` field in their `package.json` and their tools are discovered and listed within secondsβ€”no manual submission, no forms, no waiting. + +## Quick Start for Package Authors + +```json +{ + "name": "my-awesome-tool", + "version": "1.0.0", + "keywords": ["tpmjs-tool"], + "tpmjs": { + "category": "web-scraping", + "description": "Extract product data from e-commerce websites with ease", + "example": "const data = await scraper.extract('https://shop.com')" + } +} +``` + +```bash +npm publish +# ✨ Listed automatically within 15 minutes (keyword) or seconds (changes feed) +``` + +--- + +## Architecture Overview + +``` +NPM Ecosystem + ↓ +Changes Feed + Keyword Search + ↓ +Package Validator (Zod) + ↓ +PostgreSQL Database + ↓ +Next.js API Routes + ↓ +TPMJS Web App +``` + +### Core Components + +1. **NPM Sync Service** (Node.js) - Monitors NPM registry for new packages +2. **PostgreSQL Database** - Stores validated tool metadata +3. **Next.js API** - Serves tool data with search/filtering +4. **Web Frontend** - Browse, search, and discover tools + +--- + +## Discovery Mechanism: Hybrid Approach + +### Method 1: Keyword Search (Official) +- Search NPM for packages with `tpmjs-tool` keyword +- Runs every 15 minutes via cron +- Packages marked as "Official" + +### Method 2: Changes Feed (Automatic) +- Monitors `replicate.npmjs.com/registry/_changes` in real-time +- Detects packages with `tpmjs` field instantly +- Packages marked as "Community" (unless they also have keyword) + +### Why Hybrid? +- **Keywords** = Clear opt-in, queryable, respects NPM conventions +- **Changes Feed** = Real-time, catches packages without keywords +- **Together** = Best discoverability with fallback + +--- + +## The "tpmjs" Field: Tiered Schema + +### Minimal Tier (Required) + +```json +{ + "tpmjs": { + "category": "web-scraping", + "description": "Extract structured data from websites using CSS selectors", + "example": "const data = await tool.scrape({ url: 'https://example.com', selector: '.price' })" + } +} +``` + +**Categories:** +- web-scraping +- data-processing +- file-operations +- communication +- database +- api-integration +- image-processing +- text-analysis +- automation +- ai-ml +- security +- monitoring + +### Rich Tier (Optional) + +Extend with any of these optional fields: + +```json +{ + "tpmjs": { + // ... Required fields ... + + "parameters": [ + { + "name": "url", + "type": "string", + "description": "Target URL to scrape", + "required": true + } + ], + "returns": { + "type": "object", + "description": "Extracted data matching the selector" + }, + "authentication": { + "required": false, + "type": "api-key", + "envVar": "SCRAPER_API_KEY", + "docsUrl": "https://docs.example.com/auth" + }, + "pricing": { + "model": "freemium", + "freeLimit": "100 requests/month", + "paidUrl": "https://example.com/pricing" + }, + "frameworks": ["vercel-ai", "langchain", "llamaindex"], + "links": { + "documentation": "https://docs.example.com", + "playground": "https://example.com/try", + "repository": "https://github.com/user/repo" + }, + "tags": ["web", "scraping", "html", "css"], + "status": "stable", + "aiAgent": { + "useCase": "Use when agent needs to extract data from websites", + "limitations": "Cannot handle JavaScript-heavy SPAs" + } + } +} +``` + +--- + +## Database Schema + +### Tools Table + +```sql +CREATE TABLE tools ( + -- NPM Metadata + npm_package_name VARCHAR(214) UNIQUE NOT NULL, + npm_version VARCHAR(50) NOT NULL, + npm_published_at TIMESTAMP NOT NULL, + npm_description TEXT, + npm_repository JSONB, + npm_homepage TEXT, + npm_license VARCHAR(50), + + -- TPMJS Metadata + category VARCHAR(50) NOT NULL, + description TEXT NOT NULL, + example TEXT NOT NULL, + parameters JSONB, + authentication JSONB, + pricing JSONB, + frameworks TEXT[], + links JSONB, + tags TEXT[], + status VARCHAR(20), + + -- Discovery + discovery_method VARCHAR(20) NOT NULL, -- 'keyword' | 'changes-feed' + is_official BOOLEAN DEFAULT false, + tier VARCHAR(20) NOT NULL, -- 'minimal' | 'rich' + + -- Metrics + npm_downloads_last_month INTEGER DEFAULT 0, + github_stars INTEGER DEFAULT 0, + quality_score DECIMAL(3,2), -- 0.00 to 1.00 + + -- Timestamps + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() +); +``` + +--- + +## Sync Service Architecture + +### Workers + +**1. Changes Feed Worker** +- Connects to `replicate.npmjs.com/registry/_changes` +- Receives real-time change events +- Fetches package metadata for each change +- Checks for `tpmjs` field +- Validates and inserts to database + +**2. Keyword Search Worker** +- Runs every 15 minutes (cron) +- Searches `/-/v1/search?text=keywords:tpmjs-tool` +- Processes all results +- Marks as "Official" + +**3. Metrics Worker** (Optional Phase 4) +- Updates download counts from NPM API +- Fetches GitHub stars +- Calculates quality scores + +### Package Processing Pipeline + +``` +1. Fetch package metadata from NPM +2. Extract `tpmjs` field from latest version +3. Validate against Zod schema +4. If valid β†’ Insert/Update database +5. If invalid β†’ Log error +6. If no field β†’ Skip +``` + +--- + +## API Routes + +### GET /api/tools +Search and list tools + +**Query Parameters:** +- `q` - Search query +- `category` - Filter by category +- `official` - Only official tools (true/false) +- `limit` - Results per page (default 20) +- `offset` - Pagination offset + +**Response:** +```json +{ + "tools": [...], + "pagination": { + "total": 150, + "limit": 20, + "offset": 0, + "hasMore": true + } +} +``` + +### GET /api/tools/[id] +Get tool details by ID + +### POST /api/tools/validate +Validate a `tpmjs` field before publishing + +**Request:** +```json +{ + "category": "web-scraping", + "description": "...", + "example": "..." +} +``` + +**Response:** +```json +{ + "valid": true, + "tier": "minimal", + "errors": [] +} +``` + +### GET /api/stats +Registry statistics + +```json +{ + "totalTools": 2847, + "officialTools": 150, + "categories": { + "web-scraping": 320, + "communication": 280, + ... + } +} +``` + +--- + +## Quality Scoring Algorithm + +Tools are scored 0.00 to 1.00 based on: + +- **Base validity** (0.3) - Has valid schema +- **Tier** (0.1-0.2) - Rich tier > Minimal tier +- **NPM downloads** (0.2) - Based on monthly downloads +- **GitHub stars** (0.15) - Repository popularity +- **Documentation** (0.1) - Has docs URL +- **Example quality** (0.05) - Example length > 100 chars + +Score is used for default sorting and quality indicators. + +--- + +## Implementation Phases + +### Phase 1: Foundation (Week 1-2) +- Set up PostgreSQL + Prisma +- Create Zod schemas in `@tpmjs/types` +- Build sync service structure +- Implement NPM API client + +### Phase 2: Discovery (Week 2-3) +- Implement changes feed worker +- Implement keyword search worker +- Deploy sync service (Railway/Fly.io) +- Test with real packages + +### Phase 3: API & Frontend (Week 3-4) +- Build Next.js API routes +- Update tool listing page +- Update tool detail pages +- Add validation endpoint + +### Phase 4: Polish (Week 4-5) +- Add metrics worker +- Create documentation +- Build CLI validator +- Launch to community + +### Phase 5: Enhancements (Post-Launch) +- Semantic search (embeddings) +- Usage analytics +- Tool recommendations +- GitHub Actions integration + +--- + +## Infrastructure Requirements + +### Sync Service +- **Platform:** Railway or Fly.io +- **Runtime:** Node.js 22+ +- **Resources:** 512MB RAM, 1 CPU +- **Cost:** ~$5-10/month + +### Database +- **Platform:** Neon Postgres (serverless) +- **Size:** Free tier (start), scale as needed +- **Backups:** Automatic with Neon +- **Cost:** Free tier available, ~$10-20/month for production + +### Web App +- **Platform:** Vercel (existing) +- **No changes required** + +--- + +## Monitoring & Health + +### Metrics to Track + +1. **Sync Health** + - Changes feed uptime + - Packages processed per hour + - Validation success rate + +2. **Database** + - Total tools + - Official vs community ratio + - Tier distribution + +3. **API** + - Request latency (p95 < 200ms) + - Search performance + - Error rates + +### Alerts + +- Sync service down > 5 minutes +- Database connection failures +- Validation error rate > 10% + +--- + +## Developer Experience + +### Validation Before Publishing + +```bash +# Using TPMJS CLI (to be built) +npx tpmjs validate + +# Or via API +curl -X POST https://tpmjs.com/api/tools/validate \ + -H "Content-Type: application/json" \ + -d '{"category":"web-scraping","description":"...","example":"..."}' +``` + +### Documentation Pages Needed + +1. **Getting Started** - Adding TPMJS support +2. **Schema Reference** - Complete field docs +3. **Best Practices** - Tips for quality tools +4. **Examples** - Sample configurations +5. **FAQ** - Common questions + +--- + +## Migration from Mock Data + +### Current State +- 12 mock tools in `toolData.ts` +- Client-side search +- Hard-coded categories + +### Migration Strategy + +1. **Publish Real Packages** + - Create NPM packages for mock tools + - Add `tpmjs` fields + - Publish with `tpmjs-tool` keyword + +2. **Update Frontend** + - Replace mock data with API calls + - Keep existing UI components + - Update types to match Prisma models + +3. **Gradual Rollout** + - Dual mode (mock + real) + - Real data primary, mock fallback + - Remove mock entirely + +--- + +## Success Metrics + +### Technical +- βœ“ Discovery latency < 60 seconds +- βœ“ API response time < 200ms p95 +- βœ“ Support 10,000+ tools +- βœ“ 99.9% uptime + +### User Experience +- βœ“ 0-click submission (automatic) +- βœ“ Instant validation feedback +- βœ“ <100ms search speed +- βœ“ 100% mobile features + +### Business +- Week 1: 10 official tools +- Month 1: 50 official tools +- Month 3: 200+ tools +- Month 6: 1000+ tools +- 50+ active package authors + +--- + +## Comparison to Vercel's Approach + +| Feature | Vercel AI SDK | TPMJS | +|---------|---------------|-------| +| **Submission** | Manual file edit + PR | Automatic via NPM | +| **Discovery** | None | Real-time changes feed | +| **Validation** | Manual review | Automated Zod schema | +| **Updates** | New PR required | Automatic on publish | +| **Search** | Static array | Full-text + categories | +| **Scale** | 6 tools | 1000+ tools ready | + +--- + +## Next Steps + +1. Review this architecture plan +2. Approve database schema and API design +3. Set up infrastructure (Railway + Postgres) +4. Start Phase 1: Foundation +5. Launch MVP in 4-5 weeks + +--- + +## References + +- [NPM Registry API Docs](https://github.com/npm/registry/blob/main/docs/REGISTRY-API.md) +- [NPM Changes Feed](https://github.com/npm/registry/blob/main/docs/REPLICATE-API.md) +- [Vercel AI Tools Registry](https://github.com/vercel/ai/blob/main/content/tools-registry/registry.ts) +- [TPMJS Architecture Plan](/.claude/plans/goofy-inventing-stearns.md) (Full details) + +--- + +**Built with ❀️ for the AI agent ecosystem** diff --git a/apps/web/package.json b/apps/web/package.json index 3f142f6..44caf7b 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -13,6 +13,7 @@ "dependencies": { "@tpmjs/db": "workspace:*", "@tpmjs/env": "workspace:*", + "@tpmjs/npm-client": "workspace:*", "@tpmjs/types": "workspace:*", "@tpmjs/ui": "workspace:*", "@tpmjs/utils": "workspace:*", diff --git a/apps/web/src/app/api/sync/changes/route.ts b/apps/web/src/app/api/sync/changes/route.ts new file mode 100644 index 0000000..77b04c0 --- /dev/null +++ b/apps/web/src/app/api/sync/changes/route.ts @@ -0,0 +1,218 @@ +import { prisma } from '@tpmjs/db'; +import { fetchChanges, fetchLatestPackageVersion } from '@tpmjs/npm-client'; +import { validateTpmjsField } from '@tpmjs/types/tpmjs'; +import { type NextRequest, NextResponse } from 'next/server'; +import { env } from '~/env'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; +export const maxDuration = 300; // 5 minutes max for cron jobs + +/** + * POST /api/sync/changes + * Sync tools from NPM changes feed + * + * This endpoint is called by Vercel Cron (every 2 minutes) + * Requires Authorization: Bearer + */ +// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Complex but straightforward CRUD operation +export async function POST(request: NextRequest) { + // Verify cron secret for security + const authHeader = request.headers.get('authorization'); + const token = authHeader?.replace('Bearer ', ''); + + if (env.CRON_SECRET && token !== env.CRON_SECRET) { + return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }); + } + + const startTime = Date.now(); + let processed = 0; + let skipped = 0; + let errors = 0; + const errorMessages: string[] = []; + + try { + // Get last checkpoint + const checkpoint = await prisma.syncCheckpoint.findUnique({ + where: { source: 'changes-feed' }, + }); + + const lastSeq = checkpoint?.checkpoint + ? String((checkpoint.checkpoint as { lastSeq?: string })?.lastSeq || '0') + : '0'; + + // Fetch changes from NPM (limit to 100 per run to avoid timeouts) + const changesResult = await fetchChanges({ + since: lastSeq, + limit: 100, + includeDocs: false, + }); + + // Process each change + for (const change of changesResult.results) { + try { + // Fetch full package metadata + const pkg = await fetchLatestPackageVersion(change.id); + + // Skip if package not found + if (!pkg) { + skipped++; + continue; + } + + // Check if package has tpmjs field + if (!pkg.tpmjs) { + skipped++; + continue; + } + + // Validate tpmjs field + const validation = validateTpmjsField(pkg.tpmjs); + if (!validation.valid || !validation.data) { + skipped++; + continue; + } + + // Extract repository URL and GitHub stars + const githubStars: number | null = null; + + // Cast to TpmjsRich to access optional fields (they'll be undefined if not present) + const tpmjsData = validation.data as { + category: string; + description: string; + example: string; + parameters?: unknown; + returns?: unknown; + authentication?: unknown; + pricing?: unknown; + frameworks?: string[]; + links?: unknown; + tags?: string[]; + status?: string; + aiAgent?: unknown; + }; + + // Prepare data for upsert + const toolData = { + npmVersion: pkg.version, + npmPublishedAt: pkg.publishedAt ? new Date(pkg.publishedAt) : new Date(), + npmDescription: pkg.description ?? undefined, + npmRepository: pkg.repository ?? undefined, + npmHomepage: pkg.homepage ?? undefined, + npmLicense: pkg.license ?? undefined, + category: tpmjsData.category, + description: tpmjsData.description, + example: tpmjsData.example, + parameters: tpmjsData.parameters ?? undefined, + returns: tpmjsData.returns ?? undefined, + authentication: tpmjsData.authentication ?? undefined, + pricing: tpmjsData.pricing ?? undefined, + frameworks: tpmjsData.frameworks || [], + links: tpmjsData.links ?? undefined, + tags: tpmjsData.tags || [], + status: tpmjsData.status ?? undefined, + aiAgent: tpmjsData.aiAgent ?? undefined, + isOfficial: pkg.keywords?.includes('tpmjs-tool') || false, + tier: validation.tier || 'minimal', + }; + + // Upsert tool to database + await prisma.tool.upsert({ + where: { npmPackageName: pkg.name }, + create: { + npmPackageName: pkg.name, + ...toolData, + discoveryMethod: 'changes-feed', + npmDownloadsLastMonth: 0, // Will be updated by metrics sync + githubStars: githubStars, + qualityScore: null, // Will be calculated by metrics sync + }, + update: toolData, + }); + + processed++; + } catch (error) { + errors++; + const errorMsg = `Failed to process ${change.id}: ${error instanceof Error ? error.message : 'Unknown error'}`; + errorMessages.push(errorMsg); + console.error(errorMsg); + } + } + + // Update checkpoint with new sequence + await prisma.syncCheckpoint.upsert({ + where: { source: 'changes-feed' }, + create: { + source: 'changes-feed', + checkpoint: { + lastSeq: changesResult.lastSeq, + lastRun: new Date().toISOString(), + }, + }, + update: { + checkpoint: { + lastSeq: changesResult.lastSeq, + lastRun: new Date().toISOString(), + }, + }, + }); + + // Log sync operation + await prisma.syncLog.create({ + data: { + source: 'changes-feed', + status: errors > 0 ? 'partial' : 'success', + processed, + skipped, + errors, + message: + errors > 0 + ? `Processed with errors: ${errorMessages.slice(0, 3).join('; ')}` + : `Successfully processed ${processed} packages`, + metadata: { + durationMs: Date.now() - startTime, + lastSeq: changesResult.lastSeq, + pending: changesResult.pending, + }, + }, + }); + + return NextResponse.json({ + success: true, + data: { + processed, + skipped, + errors, + lastSeq: changesResult.lastSeq, + pending: changesResult.pending, + durationMs: Date.now() - startTime, + }, + }); + } catch (error) { + console.error('Changes feed sync failed:', error); + + // Log failed sync + await prisma.syncLog.create({ + data: { + source: 'changes-feed', + status: 'error', + processed, + skipped, + errors: errors + 1, + message: error instanceof Error ? error.message : 'Unknown error', + metadata: { + durationMs: Date.now() - startTime, + }, + }, + }); + + return NextResponse.json( + { + success: false, + error: 'Sync failed', + message: error instanceof Error ? error.message : 'Unknown error', + }, + { status: 500 } + ); + } +} diff --git a/apps/web/src/app/api/sync/keyword/route.ts b/apps/web/src/app/api/sync/keyword/route.ts new file mode 100644 index 0000000..55f50a5 --- /dev/null +++ b/apps/web/src/app/api/sync/keyword/route.ts @@ -0,0 +1,206 @@ +import { prisma } from '@tpmjs/db'; +import { fetchLatestPackageVersion, searchByKeyword } from '@tpmjs/npm-client'; +import { validateTpmjsField } from '@tpmjs/types/tpmjs'; +import { type NextRequest, NextResponse } from 'next/server'; +import { env } from '~/env'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; +export const maxDuration = 300; // 5 minutes max for cron jobs + +/** + * POST /api/sync/keyword + * Sync tools by searching NPM for 'tpmjs-tool' keyword + * + * This endpoint is called by Vercel Cron (every 15 minutes) + * Requires Authorization: Bearer + */ +// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Complex but straightforward CRUD operation +export async function POST(request: NextRequest) { + // Verify cron secret for security + const authHeader = request.headers.get('authorization'); + const token = authHeader?.replace('Bearer ', ''); + + if (env.CRON_SECRET && token !== env.CRON_SECRET) { + return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }); + } + + const startTime = Date.now(); + let processed = 0; + let skipped = 0; + let errors = 0; + const errorMessages: string[] = []; + + try { + // Search for packages with 'tpmjs-tool' keyword + const searchResults = await searchByKeyword({ + keyword: 'tpmjs-tool', + size: 250, // Get up to 250 packages per sync + }); + + // Process each package + for (const result of searchResults) { + try { + // Fetch full package metadata + const pkg = await fetchLatestPackageVersion(result.package.name); + + // Skip if package not found + if (!pkg) { + skipped++; + continue; + } + + // Check if package has tpmjs field + if (!pkg.tpmjs) { + skipped++; + continue; + } + + // Validate tpmjs field + const validation = validateTpmjsField(pkg.tpmjs); + if (!validation.valid || !validation.data) { + skipped++; + continue; + } + + // Extract repository URL and GitHub stars + const githubStars: number | null = null; + + // Cast to TpmjsRich to access optional fields (they'll be undefined if not present) + const tpmjsData = validation.data as { + category: string; + description: string; + example: string; + parameters?: unknown; + returns?: unknown; + authentication?: unknown; + pricing?: unknown; + frameworks?: string[]; + links?: unknown; + tags?: string[]; + status?: string; + aiAgent?: unknown; + }; + + // Prepare data for upsert + const toolData = { + npmVersion: pkg.version, + npmPublishedAt: pkg.publishedAt ? new Date(pkg.publishedAt) : new Date(), + npmDescription: pkg.description ?? undefined, + npmRepository: pkg.repository ?? undefined, + npmHomepage: pkg.homepage ?? undefined, + npmLicense: pkg.license ?? undefined, + category: tpmjsData.category, + description: tpmjsData.description, + example: tpmjsData.example, + parameters: tpmjsData.parameters ?? undefined, + returns: tpmjsData.returns ?? undefined, + authentication: tpmjsData.authentication ?? undefined, + pricing: tpmjsData.pricing ?? undefined, + frameworks: tpmjsData.frameworks || [], + links: tpmjsData.links ?? undefined, + tags: tpmjsData.tags || [], + status: tpmjsData.status ?? undefined, + aiAgent: tpmjsData.aiAgent ?? undefined, + isOfficial: pkg.keywords?.includes('tpmjs-tool') || false, + tier: validation.tier || 'minimal', + }; + + // Upsert tool to database + await prisma.tool.upsert({ + where: { npmPackageName: pkg.name }, + create: { + npmPackageName: pkg.name, + ...toolData, + discoveryMethod: 'keyword', + npmDownloadsLastMonth: 0, // Will be updated by metrics sync + githubStars: githubStars, + qualityScore: null, // Will be calculated by metrics sync + }, + update: toolData, + }); + + processed++; + } catch (error) { + errors++; + const errorMsg = `Failed to process ${result.package.name}: ${error instanceof Error ? error.message : 'Unknown error'}`; + errorMessages.push(errorMsg); + console.error(errorMsg); + } + } + + // Update checkpoint with last run timestamp + await prisma.syncCheckpoint.upsert({ + where: { source: 'keyword-search' }, + create: { + source: 'keyword-search', + checkpoint: { + lastRun: new Date().toISOString(), + packagesFound: searchResults.length, + }, + }, + update: { + checkpoint: { + lastRun: new Date().toISOString(), + packagesFound: searchResults.length, + }, + }, + }); + + // Log sync operation + await prisma.syncLog.create({ + data: { + source: 'keyword-search', + status: errors > 0 ? 'partial' : 'success', + processed, + skipped, + errors, + message: + errors > 0 + ? `Processed with errors: ${errorMessages.slice(0, 3).join('; ')}` + : `Successfully processed ${processed} packages`, + metadata: { + durationMs: Date.now() - startTime, + packagesFound: searchResults.length, + }, + }, + }); + + return NextResponse.json({ + success: true, + data: { + processed, + skipped, + errors, + packagesFound: searchResults.length, + durationMs: Date.now() - startTime, + }, + }); + } catch (error) { + console.error('Keyword search sync failed:', error); + + // Log failed sync + await prisma.syncLog.create({ + data: { + source: 'keyword-search', + status: 'error', + processed, + skipped, + errors: errors + 1, + message: error instanceof Error ? error.message : 'Unknown error', + metadata: { + durationMs: Date.now() - startTime, + }, + }, + }); + + return NextResponse.json( + { + success: false, + error: 'Sync failed', + message: error instanceof Error ? error.message : 'Unknown error', + }, + { status: 500 } + ); + } +} diff --git a/apps/web/src/app/api/sync/metrics/route.ts b/apps/web/src/app/api/sync/metrics/route.ts new file mode 100644 index 0000000..0b452b4 --- /dev/null +++ b/apps/web/src/app/api/sync/metrics/route.ts @@ -0,0 +1,177 @@ +import { prisma } from '@tpmjs/db'; +import { fetchDownloadStats } from '@tpmjs/npm-client'; +import { type NextRequest, NextResponse } from 'next/server'; +import { env } from '~/env'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; +export const maxDuration = 300; // 5 minutes max for cron jobs + +/** + * POST /api/sync/metrics + * Update download stats and quality scores for all tools + * + * This endpoint is called by Vercel Cron (every hour) + * Requires Authorization: Bearer + */ +// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Complex but straightforward CRUD operation +export async function POST(request: NextRequest) { + // Verify cron secret for security + const authHeader = request.headers.get('authorization'); + const token = authHeader?.replace('Bearer ', ''); + + if (env.CRON_SECRET && token !== env.CRON_SECRET) { + return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }); + } + + const startTime = Date.now(); + let processed = 0; + const skipped = 0; + let errors = 0; + const errorMessages: string[] = []; + + try { + // Get all tools from database + const tools = await prisma.tool.findMany({ + select: { + id: true, + npmPackageName: true, + tier: true, + npmDownloadsLastMonth: true, + githubStars: true, + }, + }); + + // Process each tool + for (const tool of tools) { + try { + // Fetch download stats from NPM + const downloads = await fetchDownloadStats(tool.npmPackageName); + + // Calculate quality score (0.00 to 1.00) + const qualityScore = calculateQualityScore({ + tier: tool.tier, + downloads, + githubStars: tool.githubStars || 0, + }); + + // Update tool metrics + await prisma.tool.update({ + where: { id: tool.id }, + data: { + npmDownloadsLastMonth: downloads, + qualityScore, + }, + }); + + processed++; + } catch (error) { + errors++; + const errorMsg = `Failed to process ${tool.npmPackageName}: ${error instanceof Error ? error.message : 'Unknown error'}`; + errorMessages.push(errorMsg); + console.error(errorMsg); + } + } + + // Update checkpoint with last run timestamp + await prisma.syncCheckpoint.upsert({ + where: { source: 'metrics' }, + create: { + source: 'metrics', + checkpoint: { + lastRun: new Date().toISOString(), + totalTools: tools.length, + }, + }, + update: { + checkpoint: { + lastRun: new Date().toISOString(), + totalTools: tools.length, + }, + }, + }); + + // Log sync operation + await prisma.syncLog.create({ + data: { + source: 'metrics', + status: errors > 0 ? 'partial' : 'success', + processed, + skipped, + errors, + message: + errors > 0 + ? `Processed with errors: ${errorMessages.slice(0, 3).join('; ')}` + : `Successfully updated metrics for ${processed} tools`, + metadata: { + durationMs: Date.now() - startTime, + totalTools: tools.length, + }, + }, + }); + + return NextResponse.json({ + success: true, + data: { + processed, + skipped, + errors, + totalTools: tools.length, + durationMs: Date.now() - startTime, + }, + }); + } catch (error) { + console.error('Metrics sync failed:', error); + + // Log failed sync + await prisma.syncLog.create({ + data: { + source: 'metrics', + status: 'error', + processed, + skipped, + errors: errors + 1, + message: error instanceof Error ? error.message : 'Unknown error', + metadata: { + durationMs: Date.now() - startTime, + }, + }, + }); + + return NextResponse.json( + { + success: false, + error: 'Sync failed', + message: error instanceof Error ? error.message : 'Unknown error', + }, + { status: 500 } + ); + } +} + +/** + * Calculate quality score based on multiple factors + * Returns a value between 0.00 and 1.00 + */ +function calculateQualityScore(params: { + tier: string; + downloads: number; + githubStars: number; +}): number { + const { tier, downloads, githubStars } = params; + + // Base score from tier + const tierScore = tier === 'rich' ? 0.6 : 0.4; + + // Downloads score (logarithmic scale, max 0.3) + const downloadsScore = Math.min(0.3, Math.log10(downloads + 1) / 10); + + // GitHub stars score (logarithmic scale, max 0.1) + const starsScore = Math.min(0.1, Math.log10(githubStars + 1) / 10); + + // Total score (capped at 1.00) + const totalScore = Math.min(1.0, tierScore + downloadsScore + starsScore); + + // Round to 2 decimal places + return Math.round(totalScore * 100) / 100; +} diff --git a/apps/web/src/env.ts b/apps/web/src/env.ts index 2ac0dde..13dda6c 100644 --- a/apps/web/src/env.ts +++ b/apps/web/src/env.ts @@ -4,4 +4,5 @@ import { z } from 'zod'; export const env = createEnv({ NODE_ENV: z.enum(['development', 'production', 'test']).default('development'), NEXT_PUBLIC_API_URL: z.string().url().optional(), + CRON_SECRET: z.string().min(32).optional(), // Required for Vercel Cron security }); diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json index 00b1618..cff0c6e 100644 --- a/apps/web/tsconfig.json +++ b/apps/web/tsconfig.json @@ -3,7 +3,8 @@ "compilerOptions": { "baseUrl": ".", "paths": { - "@/*": ["./src/*"] + "@/*": ["./src/*"], + "~/*": ["./src/*"] } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], diff --git a/packages/npm-client/package.json b/packages/npm-client/package.json index 1cea51a..61e95aa 100644 --- a/packages/npm-client/package.json +++ b/packages/npm-client/package.json @@ -5,6 +5,14 @@ "type": "module", "main": "./src/index.ts", "types": "./src/index.ts", + "exports": { + ".": "./src/index.ts", + "./changes": "./src/changes.ts", + "./search": "./src/search.ts", + "./package": "./src/package.ts", + "./stats": "./src/stats.ts", + "./rate-limiter": "./src/rate-limiter.ts" + }, "scripts": { "type-check": "tsc --noEmit" }, diff --git a/packages/npm-client/src/package.ts b/packages/npm-client/src/package.ts index 4086c5a..338bb32 100644 --- a/packages/npm-client/src/package.ts +++ b/packages/npm-client/src/package.ts @@ -53,6 +53,8 @@ const PackageVersionSchema = z.object({ tarball: z.string(), }) .optional(), + // Publishing metadata + publishedAt: z.string().optional(), }); /** @@ -144,7 +146,18 @@ export async function fetchLatestPackageVersion( return null; } - return metadata.versions[latestTag] || null; + const version = metadata.versions[latestTag]; + if (!version) { + return null; + } + + // Add publishedAt from metadata.time + const publishedAt = metadata.time?.[latestTag]; + + return { + ...version, + publishedAt, + }; } /** diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 47e7452..a4e727f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -41,9 +41,15 @@ importers: apps/web: dependencies: + '@tpmjs/db': + specifier: workspace:* + version: link:../../packages/db '@tpmjs/env': specifier: workspace:* version: link:../../packages/env + '@tpmjs/npm-client': + specifier: workspace:* + version: link:../../packages/npm-client '@tpmjs/types': specifier: workspace:* version: link:../../packages/types @@ -92,10 +98,10 @@ importers: version: 10.4.22(postcss@8.5.6) eslint: specifier: ^9.39.1 - version: 9.39.1(jiti@1.21.7) + version: 9.39.1(jiti@2.6.1) eslint-config-next: specifier: ^16.0.4 - version: 16.0.4(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3) + version: 16.0.4(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3) postcss: specifier: ^8.5.1 version: 8.5.6 @@ -201,6 +207,22 @@ importers: specifier: ^5.9.3 version: 5.9.3 + packages/npm-client: + dependencies: + zod: + specifier: ^3.24.1 + version: 3.25.76 + devDependencies: + '@tpmjs/tsconfig': + specifier: workspace:* + version: link:../config/tsconfig + '@types/node': + specifier: ^22.10.2 + version: 22.19.1 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + packages/storybook: dependencies: '@tpmjs/ui': @@ -5119,11 +5141,6 @@ snapshots: '@esbuild/win32-x64@0.27.0': optional: true - '@eslint-community/eslint-utils@4.9.0(eslint@9.39.1(jiti@1.21.7))': - dependencies: - eslint: 9.39.1(jiti@1.21.7) - eslint-visitor-keys: 3.4.3 - '@eslint-community/eslint-utils@4.9.0(eslint@9.39.1(jiti@2.6.1))': dependencies: eslint: 9.39.1(jiti@2.6.1) @@ -5962,23 +5979,6 @@ snapshots: '@types/uuid@9.0.8': {} - '@typescript-eslint/eslint-plugin@8.48.0(@typescript-eslint/parser@8.48.0(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3))(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3)': - dependencies: - '@eslint-community/regexpp': 4.12.2 - '@typescript-eslint/parser': 8.48.0(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3) - '@typescript-eslint/scope-manager': 8.48.0 - '@typescript-eslint/type-utils': 8.48.0(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3) - '@typescript-eslint/utils': 8.48.0(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3) - '@typescript-eslint/visitor-keys': 8.48.0 - eslint: 9.39.1(jiti@1.21.7) - graphemer: 1.4.0 - ignore: 7.0.5 - natural-compare: 1.4.0 - ts-api-utils: 2.1.0(typescript@5.9.3) - typescript: 5.9.3 - transitivePeerDependencies: - - supports-color - '@typescript-eslint/eslint-plugin@8.48.0(@typescript-eslint/parser@8.48.0(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3)': dependencies: '@eslint-community/regexpp': 4.12.2 @@ -5996,18 +5996,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@8.48.0(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3)': - dependencies: - '@typescript-eslint/scope-manager': 8.48.0 - '@typescript-eslint/types': 8.48.0 - '@typescript-eslint/typescript-estree': 8.48.0(typescript@5.9.3) - '@typescript-eslint/visitor-keys': 8.48.0 - debug: 4.4.3 - eslint: 9.39.1(jiti@1.21.7) - typescript: 5.9.3 - transitivePeerDependencies: - - supports-color - '@typescript-eslint/parser@8.48.0(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3)': dependencies: '@typescript-eslint/scope-manager': 8.48.0 @@ -6038,18 +6026,6 @@ snapshots: dependencies: typescript: 5.9.3 - '@typescript-eslint/type-utils@8.48.0(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3)': - dependencies: - '@typescript-eslint/types': 8.48.0 - '@typescript-eslint/typescript-estree': 8.48.0(typescript@5.9.3) - '@typescript-eslint/utils': 8.48.0(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3) - debug: 4.4.3 - eslint: 9.39.1(jiti@1.21.7) - ts-api-utils: 2.1.0(typescript@5.9.3) - typescript: 5.9.3 - transitivePeerDependencies: - - supports-color - '@typescript-eslint/type-utils@8.48.0(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3)': dependencies: '@typescript-eslint/types': 8.48.0 @@ -6079,17 +6055,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.48.0(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3)': - dependencies: - '@eslint-community/eslint-utils': 4.9.0(eslint@9.39.1(jiti@1.21.7)) - '@typescript-eslint/scope-manager': 8.48.0 - '@typescript-eslint/types': 8.48.0 - '@typescript-eslint/typescript-estree': 8.48.0(typescript@5.9.3) - eslint: 9.39.1(jiti@1.21.7) - typescript: 5.9.3 - transitivePeerDependencies: - - supports-color - '@typescript-eslint/utils@8.48.0(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3)': dependencies: '@eslint-community/eslint-utils': 4.9.0(eslint@9.39.1(jiti@2.6.1)) @@ -6897,18 +6862,18 @@ snapshots: escape-string-regexp@4.0.0: {} - eslint-config-next@16.0.4(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3): + eslint-config-next@16.0.4(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3): dependencies: '@next/eslint-plugin-next': 16.0.4 - eslint: 9.39.1(jiti@1.21.7) + eslint: 9.39.1(jiti@2.6.1) eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.1(jiti@1.21.7)) - eslint-plugin-import: 2.32.0(eslint@9.39.1(jiti@1.21.7)) - eslint-plugin-jsx-a11y: 6.10.2(eslint@9.39.1(jiti@1.21.7)) - eslint-plugin-react: 7.37.5(eslint@9.39.1(jiti@1.21.7)) - eslint-plugin-react-hooks: 7.0.1(eslint@9.39.1(jiti@1.21.7)) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(eslint@9.39.1(jiti@2.6.1)))(eslint@9.39.1(jiti@2.6.1)) + eslint-plugin-import: 2.32.0(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.1(jiti@2.6.1)) + eslint-plugin-jsx-a11y: 6.10.2(eslint@9.39.1(jiti@2.6.1)) + eslint-plugin-react: 7.37.5(eslint@9.39.1(jiti@2.6.1)) + eslint-plugin-react-hooks: 7.0.1(eslint@9.39.1(jiti@2.6.1)) globals: 16.4.0 - typescript-eslint: 8.48.0(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3) + typescript-eslint: 8.48.0(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3) optionalDependencies: typescript: 5.9.3 transitivePeerDependencies: @@ -6925,18 +6890,18 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.1(jiti@1.21.7)): + eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(eslint@9.39.1(jiti@2.6.1)))(eslint@9.39.1(jiti@2.6.1)): dependencies: '@nolyfill/is-core-module': 1.0.39 debug: 4.4.3 - eslint: 9.39.1(jiti@1.21.7) + eslint: 9.39.1(jiti@2.6.1) get-tsconfig: 4.13.0 is-bun-module: 2.0.0 stable-hash: 0.0.5 tinyglobby: 0.2.15 unrs-resolver: 1.11.1 optionalDependencies: - eslint-plugin-import: 2.32.0(eslint@9.39.1(jiti@1.21.7)) + eslint-plugin-import: 2.32.0(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.1(jiti@2.6.1)) transitivePeerDependencies: - supports-color @@ -6950,13 +6915,13 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.1(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.1(jiti@1.21.7)): + eslint-module-utils@2.12.1(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(eslint@9.39.1(jiti@2.6.1)))(eslint@9.39.1(jiti@2.6.1)))(eslint@9.39.1(jiti@2.6.1)): dependencies: debug: 3.2.7 optionalDependencies: - eslint: 9.39.1(jiti@1.21.7) + eslint: 9.39.1(jiti@2.6.1) eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.1(jiti@1.21.7)) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(eslint@9.39.1(jiti@2.6.1)))(eslint@9.39.1(jiti@2.6.1)) transitivePeerDependencies: - supports-color @@ -6989,7 +6954,7 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-plugin-import@2.32.0(eslint@9.39.1(jiti@1.21.7)): + eslint-plugin-import@2.32.0(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.1(jiti@2.6.1)): dependencies: '@rtsao/scc': 1.1.0 array-includes: 3.1.9 @@ -6998,9 +6963,9 @@ snapshots: array.prototype.flatmap: 1.3.3 debug: 3.2.7 doctrine: 2.1.0 - eslint: 9.39.1(jiti@1.21.7) + eslint: 9.39.1(jiti@2.6.1) eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.12.1(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.1(jiti@1.21.7)) + eslint-module-utils: 2.12.1(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(eslint@9.39.1(jiti@2.6.1)))(eslint@9.39.1(jiti@2.6.1)))(eslint@9.39.1(jiti@2.6.1)) hasown: 2.0.2 is-core-module: 2.16.1 is-glob: 4.0.3 @@ -7016,25 +6981,6 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-plugin-jsx-a11y@6.10.2(eslint@9.39.1(jiti@1.21.7)): - dependencies: - aria-query: 5.3.2 - array-includes: 3.1.9 - array.prototype.flatmap: 1.3.3 - ast-types-flow: 0.0.8 - axe-core: 4.11.0 - axobject-query: 4.1.0 - damerau-levenshtein: 1.0.8 - emoji-regex: 9.2.2 - eslint: 9.39.1(jiti@1.21.7) - hasown: 2.0.2 - jsx-ast-utils: 3.3.5 - language-tags: 1.0.9 - minimatch: 3.1.2 - object.fromentries: 2.0.8 - safe-regex-test: 1.1.0 - string.prototype.includes: 2.0.1 - eslint-plugin-jsx-a11y@6.10.2(eslint@9.39.1(jiti@2.6.1)): dependencies: aria-query: 5.3.2 @@ -7058,39 +7004,17 @@ snapshots: dependencies: eslint: 9.39.1(jiti@2.6.1) - eslint-plugin-react-hooks@7.0.1(eslint@9.39.1(jiti@1.21.7)): + eslint-plugin-react-hooks@7.0.1(eslint@9.39.1(jiti@2.6.1)): dependencies: '@babel/core': 7.28.5 '@babel/parser': 7.28.5 - eslint: 9.39.1(jiti@1.21.7) + eslint: 9.39.1(jiti@2.6.1) hermes-parser: 0.25.1 zod: 3.25.76 zod-validation-error: 4.0.2(zod@3.25.76) transitivePeerDependencies: - supports-color - eslint-plugin-react@7.37.5(eslint@9.39.1(jiti@1.21.7)): - dependencies: - array-includes: 3.1.9 - array.prototype.findlast: 1.2.5 - array.prototype.flatmap: 1.3.3 - array.prototype.tosorted: 1.1.4 - doctrine: 2.1.0 - es-iterator-helpers: 1.2.1 - eslint: 9.39.1(jiti@1.21.7) - estraverse: 5.3.0 - hasown: 2.0.2 - jsx-ast-utils: 3.3.5 - minimatch: 3.1.2 - object.entries: 1.1.9 - object.fromentries: 2.0.8 - object.values: 1.2.1 - prop-types: 15.8.1 - resolve: 2.0.0-next.5 - semver: 6.3.1 - string.prototype.matchall: 4.0.12 - string.prototype.repeat: 1.0.0 - eslint-plugin-react@7.37.5(eslint@9.39.1(jiti@2.6.1)): dependencies: array-includes: 3.1.9 @@ -7122,47 +7046,6 @@ snapshots: eslint-visitor-keys@4.2.1: {} - eslint@9.39.1(jiti@1.21.7): - dependencies: - '@eslint-community/eslint-utils': 4.9.0(eslint@9.39.1(jiti@1.21.7)) - '@eslint-community/regexpp': 4.12.2 - '@eslint/config-array': 0.21.1 - '@eslint/config-helpers': 0.4.2 - '@eslint/core': 0.17.0 - '@eslint/eslintrc': 3.3.1 - '@eslint/js': 9.39.1 - '@eslint/plugin-kit': 0.4.1 - '@humanfs/node': 0.16.7 - '@humanwhocodes/module-importer': 1.0.1 - '@humanwhocodes/retry': 0.4.3 - '@types/estree': 1.0.8 - ajv: 6.12.6 - chalk: 4.1.2 - cross-spawn: 7.0.6 - debug: 4.4.3 - escape-string-regexp: 4.0.0 - eslint-scope: 8.4.0 - eslint-visitor-keys: 4.2.1 - espree: 10.4.0 - esquery: 1.6.0 - esutils: 2.0.3 - fast-deep-equal: 3.1.3 - file-entry-cache: 8.0.0 - find-up: 5.0.0 - glob-parent: 6.0.2 - ignore: 5.3.2 - imurmurhash: 0.1.4 - is-glob: 4.0.3 - json-stable-stringify-without-jsonify: 1.0.1 - lodash.merge: 4.6.2 - minimatch: 3.1.2 - natural-compare: 1.4.0 - optionator: 0.9.4 - optionalDependencies: - jiti: 1.21.7 - transitivePeerDependencies: - - supports-color - eslint@9.39.1(jiti@2.6.1): dependencies: '@eslint-community/eslint-utils': 4.9.0(eslint@9.39.1(jiti@2.6.1)) @@ -8910,17 +8793,6 @@ snapshots: possible-typed-array-names: 1.1.0 reflect.getprototypeof: 1.0.10 - typescript-eslint@8.48.0(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3): - dependencies: - '@typescript-eslint/eslint-plugin': 8.48.0(@typescript-eslint/parser@8.48.0(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3))(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3) - '@typescript-eslint/parser': 8.48.0(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3) - '@typescript-eslint/typescript-estree': 8.48.0(typescript@5.9.3) - '@typescript-eslint/utils': 8.48.0(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3) - eslint: 9.39.1(jiti@1.21.7) - typescript: 5.9.3 - transitivePeerDependencies: - - supports-color - typescript-eslint@8.48.0(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3): dependencies: '@typescript-eslint/eslint-plugin': 8.48.0(@typescript-eslint/parser@8.48.0(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3) diff --git a/supabase/.gitignore b/supabase/.gitignore new file mode 100644 index 0000000..ad9264f --- /dev/null +++ b/supabase/.gitignore @@ -0,0 +1,8 @@ +# Supabase +.branches +.temp + +# dotenvx +.env.keys +.env.local +.env.*.local diff --git a/supabase/config.toml b/supabase/config.toml new file mode 100644 index 0000000..795d185 --- /dev/null +++ b/supabase/config.toml @@ -0,0 +1,347 @@ +# For detailed configuration reference documentation, visit: +# https://supabase.com/docs/guides/local-development/cli/config +# A string used to distinguish different Supabase projects on the same host. Defaults to the +# working directory name when running `supabase init`. +project_id = "tpmjs" + +[api] +enabled = true +# Port to use for the API URL. +port = 54321 +# Schemas to expose in your API. Tables, views and stored procedures in this schema will get API +# endpoints. `public` and `graphql_public` schemas are included by default. +schemas = ["public", "graphql_public"] +# Extra schemas to add to the search_path of every request. +extra_search_path = ["public", "extensions"] +# The maximum number of rows returns from a view, table, or stored procedure. Limits payload size +# for accidental or malicious requests. +max_rows = 1000 + +[api.tls] +# Enable HTTPS endpoints locally using a self-signed certificate. +enabled = false +# Paths to self-signed certificate pair. +# cert_path = "../certs/my-cert.pem" +# key_path = "../certs/my-key.pem" + +[db] +# Port to use for the local database URL. +port = 54322 +# Port used by db diff command to initialize the shadow database. +shadow_port = 54320 +# The database major version to use. This has to be the same as your remote database's. Run `SHOW +# server_version;` on the remote database to check. +major_version = 17 + +[db.pooler] +enabled = false +# Port to use for the local connection pooler. +port = 54329 +# Specifies when a server connection can be reused by other clients. +# Configure one of the supported pooler modes: `transaction`, `session`. +pool_mode = "transaction" +# How many server connections to allow per user/database pair. +default_pool_size = 20 +# Maximum number of client connections allowed. +max_client_conn = 100 + +# [db.vault] +# secret_key = "env(SECRET_VALUE)" + +[db.migrations] +# If disabled, migrations will be skipped during a db push or reset. +enabled = true +# Specifies an ordered list of schema files that describe your database. +# Supports glob patterns relative to supabase directory: "./schemas/*.sql" +schema_paths = [] + +[db.seed] +# If enabled, seeds the database after migrations during a db reset. +enabled = true +# Specifies an ordered list of seed files to load during db reset. +# Supports glob patterns relative to supabase directory: "./seeds/*.sql" +sql_paths = ["./seed.sql"] + +[db.network_restrictions] +# Enable management of network restrictions. +enabled = false +# List of IPv4 CIDR blocks allowed to connect to the database. +# Defaults to allow all IPv4 connections. Set empty array to block all IPs. +allowed_cidrs = ["0.0.0.0/0"] +# List of IPv6 CIDR blocks allowed to connect to the database. +# Defaults to allow all IPv6 connections. Set empty array to block all IPs. +allowed_cidrs_v6 = ["::/0"] + +[realtime] +enabled = true +# Bind realtime via either IPv4 or IPv6. (default: IPv4) +# ip_version = "IPv6" +# The maximum length in bytes of HTTP request headers. (default: 4096) +# max_header_length = 4096 + +[studio] +enabled = true +# Port to use for Supabase Studio. +port = 54323 +# External URL of the API server that frontend connects to. +api_url = "http://127.0.0.1" +# OpenAI API Key to use for Supabase AI in the Supabase Studio. +openai_api_key = "env(OPENAI_API_KEY)" + +# Email testing server. Emails sent with the local dev setup are not actually sent - rather, they +# are monitored, and you can view the emails that would have been sent from the web interface. +[inbucket] +enabled = true +# Port to use for the email testing server web interface. +port = 54324 +# Uncomment to expose additional ports for testing user applications that send emails. +# smtp_port = 54325 +# pop3_port = 54326 +# admin_email = "admin@email.com" +# sender_name = "Admin" + +[storage] +enabled = true +# The maximum file size allowed (e.g. "5MB", "500KB"). +file_size_limit = "50MiB" + +# Image transformation API is available to Supabase Pro plan. +# [storage.image_transformation] +# enabled = true + +# Uncomment to configure local storage buckets +# [storage.buckets.images] +# public = false +# file_size_limit = "50MiB" +# allowed_mime_types = ["image/png", "image/jpeg"] +# objects_path = "./images" + +[auth] +enabled = true +# The base URL of your website. Used as an allow-list for redirects and for constructing URLs used +# in emails. +site_url = "http://127.0.0.1:3000" +# A list of *exact* URLs that auth providers are permitted to redirect to post authentication. +additional_redirect_urls = ["https://127.0.0.1:3000"] +# How long tokens are valid for, in seconds. Defaults to 3600 (1 hour), maximum 604,800 (1 week). +jwt_expiry = 3600 +# Path to JWT signing key. DO NOT commit your signing keys file to git. +# signing_keys_path = "./signing_keys.json" +# If disabled, the refresh token will never expire. +enable_refresh_token_rotation = true +# Allows refresh tokens to be reused after expiry, up to the specified interval in seconds. +# Requires enable_refresh_token_rotation = true. +refresh_token_reuse_interval = 10 +# Allow/disallow new user signups to your project. +enable_signup = true +# Allow/disallow anonymous sign-ins to your project. +enable_anonymous_sign_ins = false +# Allow/disallow testing manual linking of accounts +enable_manual_linking = false +# Passwords shorter than this value will be rejected as weak. Minimum 6, recommended 8 or more. +minimum_password_length = 6 +# Passwords that do not meet the following requirements will be rejected as weak. Supported values +# are: `letters_digits`, `lower_upper_letters_digits`, `lower_upper_letters_digits_symbols` +password_requirements = "" + +[auth.rate_limit] +# Number of emails that can be sent per hour. Requires auth.email.smtp to be enabled. +email_sent = 2 +# Number of SMS messages that can be sent per hour. Requires auth.sms to be enabled. +sms_sent = 30 +# Number of anonymous sign-ins that can be made per hour per IP address. Requires enable_anonymous_sign_ins = true. +anonymous_users = 30 +# Number of sessions that can be refreshed in a 5 minute interval per IP address. +token_refresh = 150 +# Number of sign up and sign-in requests that can be made in a 5 minute interval per IP address (excludes anonymous users). +sign_in_sign_ups = 30 +# Number of OTP / Magic link verifications that can be made in a 5 minute interval per IP address. +token_verifications = 30 +# Number of Web3 logins that can be made in a 5 minute interval per IP address. +web3 = 30 + +# Configure one of the supported captcha providers: `hcaptcha`, `turnstile`. +# [auth.captcha] +# enabled = true +# provider = "hcaptcha" +# secret = "" + +[auth.email] +# Allow/disallow new user signups via email to your project. +enable_signup = true +# If enabled, a user will be required to confirm any email change on both the old, and new email +# addresses. If disabled, only the new email is required to confirm. +double_confirm_changes = true +# If enabled, users need to confirm their email address before signing in. +enable_confirmations = false +# If enabled, users will need to reauthenticate or have logged in recently to change their password. +secure_password_change = false +# Controls the minimum amount of time that must pass before sending another signup confirmation or password reset email. +max_frequency = "1s" +# Number of characters used in the email OTP. +otp_length = 6 +# Number of seconds before the email OTP expires (defaults to 1 hour). +otp_expiry = 3600 + +# Use a production-ready SMTP server +# [auth.email.smtp] +# enabled = true +# host = "smtp.sendgrid.net" +# port = 587 +# user = "apikey" +# pass = "env(SENDGRID_API_KEY)" +# admin_email = "admin@email.com" +# sender_name = "Admin" + +# Uncomment to customize email template +# [auth.email.template.invite] +# subject = "You have been invited" +# content_path = "./supabase/templates/invite.html" + +[auth.sms] +# Allow/disallow new user signups via SMS to your project. +enable_signup = false +# If enabled, users need to confirm their phone number before signing in. +enable_confirmations = false +# Template for sending OTP to users +template = "Your code is {{ .Code }}" +# Controls the minimum amount of time that must pass before sending another sms otp. +max_frequency = "5s" + +# Use pre-defined map of phone number to OTP for testing. +# [auth.sms.test_otp] +# 4152127777 = "123456" + +# Configure logged in session timeouts. +# [auth.sessions] +# Force log out after the specified duration. +# timebox = "24h" +# Force log out if the user has been inactive longer than the specified duration. +# inactivity_timeout = "8h" + +# This hook runs before a new user is created and allows developers to reject the request based on the incoming user object. +# [auth.hook.before_user_created] +# enabled = true +# uri = "pg-functions://postgres/auth/before-user-created-hook" + +# This hook runs before a token is issued and allows you to add additional claims based on the authentication method used. +# [auth.hook.custom_access_token] +# enabled = true +# uri = "pg-functions:////" + +# Configure one of the supported SMS providers: `twilio`, `twilio_verify`, `messagebird`, `textlocal`, `vonage`. +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +# DO NOT commit your Twilio auth token to git. Use environment variable substitution instead: +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +# Multi-factor-authentication is available to Supabase Pro plan. +[auth.mfa] +# Control how many MFA factors can be enrolled at once per user. +max_enrolled_factors = 10 + +# Control MFA via App Authenticator (TOTP) +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +# Configure MFA via Phone Messaging +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +# Configure MFA via WebAuthn +# [auth.mfa.web_authn] +# enroll_enabled = true +# verify_enabled = true + +# Use an external OAuth provider. The full list of providers are: `apple`, `azure`, `bitbucket`, +# `discord`, `facebook`, `github`, `gitlab`, `google`, `keycloak`, `linkedin_oidc`, `notion`, `twitch`, +# `twitter`, `slack`, `spotify`, `workos`, `zoom`. +[auth.external.apple] +enabled = false +client_id = "" +# DO NOT commit your OAuth provider secret to git. Use environment variable substitution instead: +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +# Overrides the default auth redirectUrl. +redirect_uri = "" +# Overrides the default auth provider URL. Used to support self-hosted gitlab, single-tenant Azure, +# or any other third-party OIDC providers. +url = "" +# If enabled, the nonce check will be skipped. Required for local sign in with Google auth. +skip_nonce_check = false + +# Allow Solana wallet holders to sign in to your project via the Sign in with Solana (SIWS, EIP-4361) standard. +# You can configure "web3" rate limit in the [auth.rate_limit] section and set up [auth.captcha] if self-hosting. +[auth.web3.solana] +enabled = false + +# Use Firebase Auth as a third-party provider alongside Supabase Auth. +[auth.third_party.firebase] +enabled = false +# project_id = "my-firebase-project" + +# Use Auth0 as a third-party provider alongside Supabase Auth. +[auth.third_party.auth0] +enabled = false +# tenant = "my-auth0-tenant" +# tenant_region = "us" + +# Use AWS Cognito (Amplify) as a third-party provider alongside Supabase Auth. +[auth.third_party.aws_cognito] +enabled = false +# user_pool_id = "my-user-pool-id" +# user_pool_region = "us-east-1" + +# Use Clerk as a third-party provider alongside Supabase Auth. +[auth.third_party.clerk] +enabled = false +# Obtain from https://clerk.com/setup/supabase +# domain = "example.clerk.accounts.dev" + +# OAuth server configuration +[auth.oauth_server] +# Enable OAuth server functionality +enabled = false +# Path for OAuth consent flow UI +authorization_url_path = "/oauth/consent" +# Allow dynamic client registration +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +# Supported request policies: `oneshot`, `per_worker`. +# `per_worker` (default) β€” enables hot reload during local development. +# `oneshot` β€” fallback mode if hot reload causes issues (e.g. in large repos or with symlinks). +policy = "per_worker" +# Port to attach the Chrome inspector for debugging edge functions. +inspector_port = 8083 +# The Deno major version to use. +deno_version = 2 + +# [edge_runtime.secrets] +# secret_key = "env(SECRET_VALUE)" + +[analytics] +enabled = true +port = 54327 +# Configure one of the supported backends: `postgres`, `bigquery`. +backend = "postgres" + +# Experimental features may be deprecated any time +[experimental] +# Configures Postgres storage engine to use OrioleDB (S3) +orioledb_version = "" +# Configures S3 bucket URL, eg. .s3-.amazonaws.com +s3_host = "env(S3_HOST)" +# Configures S3 bucket region, eg. us-east-1 +s3_region = "env(S3_REGION)" +# Configures AWS_ACCESS_KEY_ID for S3 bucket +s3_access_key = "env(S3_ACCESS_KEY)" +# Configures AWS_SECRET_ACCESS_KEY for S3 bucket +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/vercel.json b/vercel.json index f88823c..2eb65e2 100644 --- a/vercel.json +++ b/vercel.json @@ -8,5 +8,19 @@ "github": { "silent": false, "autoJobCancelation": true - } + }, + "crons": [ + { + "path": "/api/sync/changes", + "schedule": "*/2 * * * *" + }, + { + "path": "/api/sync/keyword", + "schedule": "*/15 * * * *" + }, + { + "path": "/api/sync/metrics", + "schedule": "0 * * * *" + } + ] }