From 1b3b47f8b255e9b6c80b9ac5908a8976c02c51f7 Mon Sep 17 00:00:00 2001 From: Ajax Davis Date: Fri, 28 Nov 2025 02:19:59 +1000 Subject: [PATCH] feat(npm-client): create NPM registry API client package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Implement changes feed client for real-time package discovery - Implement keyword search with auto-pagination support - Implement package metadata fetcher with tpmjs field extraction - Implement download statistics client for quality scoring - Implement rate limiter with configurable concurrency and delays - Implement retry logic with exponential backoff for 429 errors All API responses validated with Zod schemas for type safety. Rate limiter defaults: max 10 concurrent, 100ms minimum delay. Handles scoped packages (@scope/name) correctly. Graceful 404 handling returns null instead of throwing. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- packages/npm-client/package.json | 19 +++ packages/npm-client/src/changes.ts | 80 ++++++++++++ packages/npm-client/src/index.ts | 38 ++++++ packages/npm-client/src/package.ts | 163 ++++++++++++++++++++++++ packages/npm-client/src/rate-limiter.ts | 104 +++++++++++++++ packages/npm-client/src/search.ts | 147 +++++++++++++++++++++ packages/npm-client/src/stats.ts | 85 ++++++++++++ packages/npm-client/tsconfig.json | 11 ++ 8 files changed, 647 insertions(+) create mode 100644 packages/npm-client/package.json create mode 100644 packages/npm-client/src/changes.ts create mode 100644 packages/npm-client/src/index.ts create mode 100644 packages/npm-client/src/package.ts create mode 100644 packages/npm-client/src/rate-limiter.ts create mode 100644 packages/npm-client/src/search.ts create mode 100644 packages/npm-client/src/stats.ts create mode 100644 packages/npm-client/tsconfig.json diff --git a/packages/npm-client/package.json b/packages/npm-client/package.json new file mode 100644 index 0000000..1cea51a --- /dev/null +++ b/packages/npm-client/package.json @@ -0,0 +1,19 @@ +{ + "name": "@tpmjs/npm-client", + "version": "0.0.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "scripts": { + "type-check": "tsc --noEmit" + }, + "dependencies": { + "zod": "^3.24.1" + }, + "devDependencies": { + "@tpmjs/tsconfig": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.3" + } +} diff --git a/packages/npm-client/src/changes.ts b/packages/npm-client/src/changes.ts new file mode 100644 index 0000000..94084be --- /dev/null +++ b/packages/npm-client/src/changes.ts @@ -0,0 +1,80 @@ +/** + * NPM Changes Feed Client + * Polls the NPM registry changes feed for new package publications + */ + +import { z } from 'zod'; + +const NPM_CHANGES_URL = 'https://replicate.npmjs.com/registry/_changes'; + +/** + * Schema for a single change in the feed + */ +const ChangeSchema = z.object({ + id: z.string(), + seq: z.union([z.string(), z.number()]), + changes: z.array(z.object({ rev: z.string() })).optional(), + deleted: z.boolean().optional(), +}); + +/** + * Schema for the changes feed response + */ +const ChangesFeedSchema = z.object({ + results: z.array(ChangeSchema), + last_seq: z.union([z.string(), z.number()]), + pending: z.number().optional(), +}); + +export type Change = z.infer; +export type ChangesFeedResponse = z.infer; + +export interface FetchChangesOptions { + /** The sequence number to start from */ + since?: string | number; + /** Maximum number of changes to fetch */ + limit?: number; + /** Include document bodies (slower but more data) */ + includeDocs?: boolean; +} + +export interface FetchChangesResult { + results: Change[]; + lastSeq: string; + pending?: number; +} + +/** + * Fetches changes from the NPM registry + * @param options - Options for fetching changes + * @returns The changes and the last sequence number + */ +export async function fetchChanges(options: FetchChangesOptions = {}): Promise { + const { since = '0', limit = 100, includeDocs = false } = options; + + const url = new URL(NPM_CHANGES_URL); + url.searchParams.set('since', String(since)); + url.searchParams.set('limit', String(limit)); + if (includeDocs) { + url.searchParams.set('include_docs', 'true'); + } + + const response = await fetch(url.toString(), { + headers: { + Accept: 'application/json', + }, + }); + + if (!response.ok) { + throw new Error(`NPM changes feed error: ${response.status} ${response.statusText}`); + } + + const data = await response.json(); + const parsed = ChangesFeedSchema.parse(data); + + return { + results: parsed.results.filter((change) => !change.deleted), + lastSeq: String(parsed.last_seq), + pending: parsed.pending, + }; +} diff --git a/packages/npm-client/src/index.ts b/packages/npm-client/src/index.ts new file mode 100644 index 0000000..0e74a9c --- /dev/null +++ b/packages/npm-client/src/index.ts @@ -0,0 +1,38 @@ +/** + * @tpmjs/npm-client + * NPM Registry API client for TPMJS + */ + +// Changes feed +export { + fetchChanges, + type Change, + type ChangesFeedResponse, + type FetchChangesOptions, + type FetchChangesResult, +} from './changes'; + +// Keyword search +export { + searchByKeyword, + searchAllByKeyword, + type SearchPackage, + type SearchResult, + type SearchResponse, + type SearchByKeywordOptions, +} from './search'; + +// Package metadata +export { + fetchPackageMetadata, + fetchLatestPackageVersion, + fetchPackageTpmjsField, + type PackageVersion, + type PackageMetadata, +} from './package'; + +// Download statistics +export { fetchDownloadStats, fetchBulkDownloadStats, type DownloadsResponse } from './stats'; + +// Rate limiting +export { delay, RateLimiter, npmRateLimiter, retryWithBackoff } from './rate-limiter'; diff --git a/packages/npm-client/src/package.ts b/packages/npm-client/src/package.ts new file mode 100644 index 0000000..4086c5a --- /dev/null +++ b/packages/npm-client/src/package.ts @@ -0,0 +1,163 @@ +/** + * NPM Package Metadata Client + * Fetches detailed package information from NPM registry + */ + +import { z } from 'zod'; + +const NPM_REGISTRY_URL = 'https://registry.npmjs.org'; + +/** + * Schema for package version metadata + */ +const PackageVersionSchema = z.object({ + name: z.string(), + version: z.string(), + description: z.string().optional(), + keywords: z.array(z.string()).optional(), + homepage: z.string().optional(), + repository: z + .union([ + z.string(), + z.object({ + type: z.string(), + url: z.string(), + }), + ]) + .optional(), + license: z.string().optional(), + author: z + .union([ + z.string(), + z.object({ + name: z.string(), + email: z.string().optional(), + url: z.string().optional(), + }), + ]) + .optional(), + maintainers: z + .array( + z.object({ + name: z.string(), + email: z.string().optional(), + }) + ) + .optional(), + // The tpmjs field - this is what we're looking for! + tpmjs: z.unknown().optional(), + // Dist info + dist: z + .object({ + shasum: z.string(), + tarball: z.string(), + }) + .optional(), +}); + +/** + * Schema for full package metadata + */ +const PackageMetadataSchema = z.object({ + name: z.string(), + 'dist-tags': z.record(z.string()), + versions: z.record(PackageVersionSchema), + time: z.record(z.string()), + maintainers: z + .array( + z.object({ + name: z.string(), + email: z.string(), + }) + ) + .optional(), + description: z.string().optional(), + homepage: z.string().optional(), + keywords: z.array(z.string()).optional(), + repository: z + .union([ + z.string(), + z.object({ + type: z.string(), + url: z.string(), + }), + ]) + .optional(), + license: z.string().optional(), + readme: z.string().optional(), +}); + +export type PackageVersion = z.infer; +export type PackageMetadata = z.infer; + +/** + * Fetches package metadata from NPM registry + * @param packageName - The name of the package + * @returns The package metadata, or null if not found + */ +export async function fetchPackageMetadata(packageName: string): Promise { + // Encode the package name to handle scoped packages (@scope/name) + const encodedName = packageName.replace('/', '%2F'); + const url = `${NPM_REGISTRY_URL}/${encodedName}`; + + try { + const response = await fetch(url, { + headers: { + Accept: 'application/json', + }, + }); + + if (response.status === 404) { + return null; + } + + if (!response.ok) { + throw new Error(`NPM registry error: ${response.status} ${response.statusText}`); + } + + const data = await response.json(); + return PackageMetadataSchema.parse(data); + } catch (error) { + if (error instanceof Error && error.message.includes('404')) { + return null; + } + throw error; + } +} + +/** + * Gets the latest version of a package + * @param packageName - The name of the package + * @returns The latest package version metadata, or null if not found + */ +export async function fetchLatestPackageVersion( + packageName: string +): Promise { + const metadata = await fetchPackageMetadata(packageName); + + if (!metadata) { + return null; + } + + const latestTag = metadata['dist-tags'].latest; + if (!latestTag) { + return null; + } + + return metadata.versions[latestTag] || null; +} + +/** + * Checks if a package has a tpmjs field in its latest version + * @param packageName - The name of the package + * @returns The tpmjs field if present, null otherwise + */ +export async function fetchPackageTpmjsField(packageName: string): Promise { + const latestVersion = await fetchLatestPackageVersion(packageName); + + if (!latestVersion) { + return null; + } + + return latestVersion.tpmjs || null; +} diff --git a/packages/npm-client/src/rate-limiter.ts b/packages/npm-client/src/rate-limiter.ts new file mode 100644 index 0000000..1b64d3f --- /dev/null +++ b/packages/npm-client/src/rate-limiter.ts @@ -0,0 +1,104 @@ +/** + * Simple rate limiter utility + * Helps avoid overwhelming NPM API with too many requests + */ + +/** + * Delays execution for a specified number of milliseconds + */ +export function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Rate limiter class that enforces a maximum number of requests per time period + */ +export class RateLimiter { + private queue: Array<() => void> = []; + private running = 0; + + constructor( + private maxConcurrent = 10, + private minDelayMs = 100 + ) {} + + /** + * Executes a function with rate limiting + */ + async execute(fn: () => Promise): Promise { + // Wait if we're at max concurrent requests + while (this.running >= this.maxConcurrent) { + await new Promise((resolve) => this.queue.push(resolve)); + } + + this.running++; + + try { + const result = await fn(); + await delay(this.minDelayMs); // Enforce minimum delay between requests + return result; + } finally { + this.running--; + const next = this.queue.shift(); + if (next) { + next(); + } + } + } + + /** + * Executes multiple functions with rate limiting + */ + async executeAll(fns: Array<() => Promise>): Promise { + return Promise.all(fns.map((fn) => this.execute(fn))); + } +} + +/** + * Default rate limiter instance for NPM API calls + * - Max 10 concurrent requests + * - Min 100ms delay between requests + */ +export const npmRateLimiter = new RateLimiter(10, 100); + +/** + * Retry a function with exponential backoff + */ +export async function retryWithBackoff( + fn: () => Promise, + maxRetries = 3, + initialDelayMs = 1000 +): Promise { + let lastError: Error | unknown; + + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + return await fn(); + } catch (error) { + lastError = error; + + // Don't retry on last attempt + if (attempt === maxRetries) { + break; + } + + // Check if it's a 429 (rate limit) error + const is429 = + error instanceof Error && (error.message.includes('429') || error.message.includes('rate')); + + if (is429) { + // Exponential backoff: 1s, 2s, 4s, 8s, etc. + const delayMs = initialDelayMs * 2 ** attempt; + console.warn( + `Rate limited, retrying in ${delayMs}ms (attempt ${attempt + 1}/${maxRetries})` + ); + await delay(delayMs); + } else { + // For other errors, just use a short delay + await delay(500); + } + } + } + + throw lastError; +} diff --git a/packages/npm-client/src/search.ts b/packages/npm-client/src/search.ts new file mode 100644 index 0000000..7e035c2 --- /dev/null +++ b/packages/npm-client/src/search.ts @@ -0,0 +1,147 @@ +/** + * NPM Keyword Search Client + * Searches NPM registry by keyword to find packages + */ + +import { z } from 'zod'; + +const NPM_SEARCH_URL = 'https://registry.npmjs.org/-/v1/search'; + +/** + * Schema for search result package + */ +const SearchPackageSchema = z.object({ + name: z.string(), + version: z.string(), + description: z.string().optional(), + keywords: z.array(z.string()).optional(), + date: z.string().optional(), + links: z + .object({ + npm: z.string().optional(), + homepage: z.string().optional(), + repository: z.string().optional(), + bugs: z.string().optional(), + }) + .optional(), + author: z + .object({ + name: z.string(), + email: z.string().optional(), + }) + .optional(), + publisher: z + .object({ + username: z.string(), + email: z.string(), + }) + .optional(), + maintainers: z + .array( + z.object({ + username: z.string(), + email: z.string(), + }) + ) + .optional(), +}); + +/** + * Schema for search result object + */ +const SearchResultSchema = z.object({ + package: SearchPackageSchema, + score: z + .object({ + final: z.number(), + detail: z + .object({ + quality: z.number(), + popularity: z.number(), + maintenance: z.number(), + }) + .optional(), + }) + .optional(), + searchScore: z.number().optional(), +}); + +/** + * Schema for search response + */ +const SearchResponseSchema = z.object({ + objects: z.array(SearchResultSchema), + total: z.number(), + time: z.string().optional(), +}); + +export type SearchPackage = z.infer; +export type SearchResult = z.infer; +export type SearchResponse = z.infer; + +export interface SearchByKeywordOptions { + /** The keyword to search for */ + keyword: string; + /** Number of results to return (default: 250, max: 250) */ + size?: number; + /** Offset for pagination (default: 0) */ + from?: number; +} + +/** + * Searches NPM registry by keyword + * @param options - Search options + * @returns Array of matching packages + */ +export async function searchByKeyword(options: SearchByKeywordOptions): Promise { + const { keyword, size = 250, from = 0 } = options; + + const url = new URL(NPM_SEARCH_URL); + url.searchParams.set('text', `keywords:${keyword}`); + url.searchParams.set('size', String(Math.min(size, 250))); // NPM max is 250 + url.searchParams.set('from', String(from)); + + const response = await fetch(url.toString(), { + headers: { + Accept: 'application/json', + }, + }); + + if (!response.ok) { + throw new Error(`NPM search error: ${response.status} ${response.statusText}`); + } + + const data = await response.json(); + const parsed = SearchResponseSchema.parse(data); + + return parsed.objects; +} + +/** + * Searches for all packages with a specific keyword + * Handles pagination automatically to get all results + */ +export async function searchAllByKeyword(keyword: string): Promise { + const allResults: SearchResult[] = []; + let from = 0; + const size = 250; + + while (true) { + const results = await searchByKeyword({ keyword, size, from }); + + if (results.length === 0) { + break; + } + + allResults.push(...results); + + // If we got less than the page size, we're done + if (results.length < size) { + break; + } + + from += size; + } + + return allResults; +} diff --git a/packages/npm-client/src/stats.ts b/packages/npm-client/src/stats.ts new file mode 100644 index 0000000..470aa55 --- /dev/null +++ b/packages/npm-client/src/stats.ts @@ -0,0 +1,85 @@ +/** + * NPM Download Statistics Client + * Fetches download counts from NPM statistics API + */ + +import { z } from 'zod'; + +const NPM_DOWNLOADS_URL = 'https://api.npmjs.org/downloads'; + +/** + * Schema for downloads response + */ +const DownloadsResponseSchema = z.object({ + downloads: z.number(), + start: z.string(), + end: z.string(), + package: z.string(), +}); + +export type DownloadsResponse = z.infer; + +/** + * Fetches download statistics for a package + * @param packageName - The name of the package + * @param period - Time period (default: 'last-month') + * @returns The download count, or 0 if not found + */ +export async function fetchDownloadStats( + packageName: string, + period: 'last-day' | 'last-week' | 'last-month' | 'last-year' = 'last-month' +): Promise { + const url = `${NPM_DOWNLOADS_URL}/point/${period}/${packageName}`; + + try { + const response = await fetch(url, { + headers: { + Accept: 'application/json', + }, + }); + + if (response.status === 404) { + // Package not found or no download data + return 0; + } + + if (!response.ok) { + throw new Error(`NPM downloads API error: ${response.status} ${response.statusText}`); + } + + const data = await response.json(); + const parsed = DownloadsResponseSchema.parse(data); + + return parsed.downloads; + } catch (error) { + if (error instanceof Error && error.message.includes('404')) { + return 0; + } + throw error; + } +} + +/** + * Fetches download statistics for multiple packages in bulk + * Note: NPM API doesn't have a native bulk endpoint, so this makes individual requests + * Use with rate limiting to avoid overwhelming the API + */ +export async function fetchBulkDownloadStats( + packageNames: string[], + period: 'last-day' | 'last-week' | 'last-month' | 'last-year' = 'last-month' +): Promise> { + const results = new Map(); + + // Process in batches to avoid overwhelming the API + for (const packageName of packageNames) { + try { + const downloads = await fetchDownloadStats(packageName, period); + results.set(packageName, downloads); + } catch (error) { + console.error(`Failed to fetch stats for ${packageName}:`, error); + results.set(packageName, 0); + } + } + + return results; +} diff --git a/packages/npm-client/tsconfig.json b/packages/npm-client/tsconfig.json new file mode 100644 index 0000000..6521d56 --- /dev/null +++ b/packages/npm-client/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "@tpmjs/tsconfig/base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "incremental": false, + "composite": false + }, + "include": ["src"], + "exclude": ["node_modules", "dist"] +}