feat(executor): formalize Executor Protocol v1.0 with compliance testing
- Add EXECUTOR_SPECIFICATION.md with formal v1.0 protocol spec - Add executor-openapi.yaml (OpenAPI 3.0 specification) - Create @tpmjs/executor-test compliance test package (15 tests) - Update Railway executor to v1.0 compliance (15/15 tests pass) - Update Unsandbox executor to v1.0 compliance (15/15 tests pass) - Update Vercel executor to v1.0 compliance - Add /info endpoint with capability advertisement to all executors - Add structured error codes (PACKAGE_NOT_FOUND, TOOL_NOT_FOUND, etc.) - Add protocolVersion and implementationVersion to /health responses - Add X-TPMJS-Protocol-Version header support - Add EXECUTOR_COMPLIANCE.md with test results documentation
This commit is contained in:
Ajax Davis
parent
760cc4b77e
commit
32c6e097ed
24 changed files with 3084 additions and 258 deletions
7
packages/executor-test/bin/run.js
Normal file
7
packages/executor-test/bin/run.js
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
#!/usr/bin/env node
|
||||
import { run } from '../dist/index.js';
|
||||
|
||||
run(process.argv.slice(2)).catch((error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
58
packages/executor-test/package.json
Normal file
58
packages/executor-test/package.json
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
{
|
||||
"name": "@tpmjs/executor-test",
|
||||
"version": "0.1.0",
|
||||
"description": "TPMJS Executor Protocol compliance test suite",
|
||||
"author": "TPMJS",
|
||||
"license": "MIT",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/tpmjs/tpmjs.git",
|
||||
"directory": "packages/executor-test"
|
||||
},
|
||||
"homepage": "https://tpmjs.com",
|
||||
"keywords": [
|
||||
"tpmjs",
|
||||
"executor",
|
||||
"compliance",
|
||||
"testing",
|
||||
"mcp"
|
||||
],
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"executor-test": "./bin/run.js"
|
||||
},
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"bin"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "tsup",
|
||||
"dev": "tsup --watch",
|
||||
"test": "vitest",
|
||||
"type-check": "tsc --noEmit",
|
||||
"clean": "rm -rf dist .turbo"
|
||||
},
|
||||
"dependencies": {
|
||||
"picocolors": "^1.1.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tpmjs/test": "workspace:*",
|
||||
"@tpmjs/tsconfig": "workspace:*",
|
||||
"@types/node": "^22.15.29",
|
||||
"tsup": "^8.5.1",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^4.0.16"
|
||||
},
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
}
|
||||
4
packages/executor-test/src/index.ts
Normal file
4
packages/executor-test/src/index.ts
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
export { run } from './runner.js';
|
||||
export { runCoreTests } from './tests/core.js';
|
||||
export { runStandardTests } from './tests/standard.js';
|
||||
export type { ComplianceResult, TestResult, TestSuite } from './types.js';
|
||||
14
packages/executor-test/src/runner.test.ts
Normal file
14
packages/executor-test/src/runner.test.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
describe('executor-test', () => {
|
||||
it('should export run function', async () => {
|
||||
const { run } = await import('./runner.js');
|
||||
expect(typeof run).toBe('function');
|
||||
});
|
||||
|
||||
it('should export test suite functions', async () => {
|
||||
const { runCoreTests, runStandardTests } = await import('./index.js');
|
||||
expect(typeof runCoreTests).toBe('function');
|
||||
expect(typeof runStandardTests).toBe('function');
|
||||
});
|
||||
});
|
||||
201
packages/executor-test/src/runner.ts
Normal file
201
packages/executor-test/src/runner.ts
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
import pc from 'picocolors';
|
||||
import { runCoreTests } from './tests/core.js';
|
||||
import { runStandardTests } from './tests/standard.js';
|
||||
import type { ComplianceResult, TestSuite } from './types.js';
|
||||
|
||||
const VERSION = '0.1.0';
|
||||
const PROTOCOL_VERSION = '1.0';
|
||||
|
||||
function printUsage(): void {
|
||||
console.log(`
|
||||
${pc.bold('TPMJS Executor Compliance Test')} v${VERSION}
|
||||
|
||||
${pc.dim('Usage:')}
|
||||
npx @tpmjs/executor-test <executor-url> [options]
|
||||
|
||||
${pc.dim('Options:')}
|
||||
--api-key <key> API key for authentication (Bearer token)
|
||||
--json Output results as JSON
|
||||
--verbose Show detailed test output
|
||||
--help Show this help message
|
||||
|
||||
${pc.dim('Examples:')}
|
||||
npx @tpmjs/executor-test https://my-executor.example.com
|
||||
npx @tpmjs/executor-test https://my-executor.example.com --api-key sk-xxx
|
||||
npx @tpmjs/executor-test https://my-executor.example.com --json
|
||||
`);
|
||||
}
|
||||
|
||||
function printBanner(target: string): void {
|
||||
console.log();
|
||||
console.log(pc.bold(`TPMJS Executor Compliance Test v${VERSION}`));
|
||||
console.log(pc.dim(`Protocol Version: ${PROTOCOL_VERSION}`));
|
||||
console.log(pc.dim(`Target: ${target}`));
|
||||
console.log();
|
||||
}
|
||||
|
||||
function printSuite(suite: TestSuite): void {
|
||||
const levelLabel =
|
||||
suite.level === 'core'
|
||||
? pc.blue('Core')
|
||||
: suite.level === 'standard'
|
||||
? pc.yellow('Standard')
|
||||
: pc.magenta('Extended');
|
||||
|
||||
console.log(`${levelLabel} ${pc.bold(suite.name)}:`);
|
||||
|
||||
for (const result of suite.results) {
|
||||
const icon = result.passed ? pc.green('\u2713') : pc.red('\u2717');
|
||||
const name = result.passed ? result.name : pc.red(result.name);
|
||||
const duration = pc.dim(`(${result.durationMs}ms)`);
|
||||
|
||||
console.log(` ${icon} ${name} ${duration}`);
|
||||
|
||||
if (!result.passed && result.message) {
|
||||
console.log(` ${pc.dim(result.message)}`);
|
||||
}
|
||||
}
|
||||
console.log();
|
||||
}
|
||||
|
||||
function printSummary(result: ComplianceResult): void {
|
||||
const { summary } = result;
|
||||
|
||||
console.log(pc.bold('Summary:'));
|
||||
console.log(
|
||||
` Tests: ${pc.green(`${summary.passed} passed`)}, ${summary.failed > 0 ? pc.red(`${summary.failed} failed`) : `${summary.failed} failed`}, ${summary.totalTests} total`
|
||||
);
|
||||
console.log();
|
||||
|
||||
const coreStatus = summary.coreCompliant ? pc.green('PASS') : pc.red('FAIL');
|
||||
const standardStatus = summary.standardCompliant
|
||||
? pc.green('PASS')
|
||||
: summary.coreCompliant
|
||||
? pc.yellow('PARTIAL')
|
||||
: pc.red('FAIL');
|
||||
|
||||
console.log(` Core Compliance: ${coreStatus}`);
|
||||
console.log(` Standard Compliance: ${standardStatus}`);
|
||||
console.log();
|
||||
}
|
||||
|
||||
interface Options {
|
||||
apiKey?: string;
|
||||
json: boolean;
|
||||
verbose: boolean;
|
||||
}
|
||||
|
||||
function parseArgs(args: string[]): { url: string | null; options: Options } {
|
||||
const options: Options = {
|
||||
json: false,
|
||||
verbose: false,
|
||||
};
|
||||
|
||||
let url: string | null = null;
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i];
|
||||
|
||||
if (arg === '--help' || arg === '-h') {
|
||||
printUsage();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (arg === '--json') {
|
||||
options.json = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === '--verbose' || arg === '-v') {
|
||||
options.verbose = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === '--api-key') {
|
||||
i++;
|
||||
options.apiKey = args[i] ?? undefined;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!arg?.startsWith('-') && !url) {
|
||||
url = arg ?? null;
|
||||
}
|
||||
}
|
||||
|
||||
return { url, options };
|
||||
}
|
||||
|
||||
export async function run(args: string[]): Promise<void> {
|
||||
const { url, options } = parseArgs(args);
|
||||
|
||||
if (!url) {
|
||||
printUsage();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Normalize URL
|
||||
const target = url.replace(/\/$/, '');
|
||||
|
||||
if (!options.json) {
|
||||
printBanner(target);
|
||||
}
|
||||
|
||||
const suites: TestSuite[] = [];
|
||||
|
||||
// Run Core tests
|
||||
if (!options.json) {
|
||||
console.log(pc.dim('Running Core compliance tests...'));
|
||||
console.log();
|
||||
}
|
||||
|
||||
const coreSuite = await runCoreTests(target, options.apiKey);
|
||||
suites.push(coreSuite);
|
||||
|
||||
if (!options.json) {
|
||||
printSuite(coreSuite);
|
||||
}
|
||||
|
||||
// Run Standard tests
|
||||
if (!options.json) {
|
||||
console.log(pc.dim('Running Standard compliance tests...'));
|
||||
console.log();
|
||||
}
|
||||
|
||||
const standardSuite = await runStandardTests(target, options.apiKey);
|
||||
suites.push(standardSuite);
|
||||
|
||||
if (!options.json) {
|
||||
printSuite(standardSuite);
|
||||
}
|
||||
|
||||
// Calculate summary
|
||||
const totalTests = suites.reduce((sum, s) => sum + s.results.length, 0);
|
||||
const passed = suites.reduce((sum, s) => sum + s.results.filter((r) => r.passed).length, 0);
|
||||
const failed = totalTests - passed;
|
||||
|
||||
const coreCompliant = coreSuite.results.every((r) => r.passed);
|
||||
const standardCompliant = coreCompliant && standardSuite.results.every((r) => r.passed);
|
||||
|
||||
const result: ComplianceResult = {
|
||||
target,
|
||||
protocolVersion: PROTOCOL_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
suites,
|
||||
summary: {
|
||||
totalTests,
|
||||
passed,
|
||||
failed,
|
||||
coreCompliant,
|
||||
standardCompliant,
|
||||
},
|
||||
};
|
||||
|
||||
if (options.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
printSummary(result);
|
||||
}
|
||||
|
||||
// Exit with error code if not compliant
|
||||
process.exit(coreCompliant ? 0 : 1);
|
||||
}
|
||||
417
packages/executor-test/src/tests/core.ts
Normal file
417
packages/executor-test/src/tests/core.ts
Normal file
|
|
@ -0,0 +1,417 @@
|
|||
import type { ExecuteToolResponse, HealthResponse, TestResult, TestSuite } from '../types.js';
|
||||
|
||||
async function testHealthReturns200(baseUrl: string, _apiKey?: string): Promise<TestResult> {
|
||||
const start = Date.now();
|
||||
const name = 'GET /health returns 200';
|
||||
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/health`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'X-TPMJS-Protocol-Version': '1.0',
|
||||
},
|
||||
});
|
||||
|
||||
const durationMs = Date.now() - start;
|
||||
|
||||
if (response.status === 200) {
|
||||
return { name, passed: true, durationMs };
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Expected status 200, got ${response.status}`,
|
||||
durationMs,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Request failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function testHealthIncludesProtocolVersion(
|
||||
baseUrl: string,
|
||||
_apiKey?: string
|
||||
): Promise<TestResult> {
|
||||
const start = Date.now();
|
||||
const name = 'GET /health includes protocolVersion';
|
||||
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/health`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'X-TPMJS-Protocol-Version': '1.0',
|
||||
},
|
||||
});
|
||||
|
||||
const data = (await response.json()) as HealthResponse;
|
||||
const durationMs = Date.now() - start;
|
||||
|
||||
if (data.protocolVersion) {
|
||||
return { name, passed: true, durationMs };
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: 'Response missing protocolVersion field',
|
||||
durationMs,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Request failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function testHealthIncludesImplementationVersion(
|
||||
baseUrl: string,
|
||||
_apiKey?: string
|
||||
): Promise<TestResult> {
|
||||
const start = Date.now();
|
||||
const name = 'GET /health includes implementationVersion';
|
||||
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/health`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'X-TPMJS-Protocol-Version': '1.0',
|
||||
},
|
||||
});
|
||||
|
||||
const data = (await response.json()) as HealthResponse;
|
||||
const durationMs = Date.now() - start;
|
||||
|
||||
if (data.implementationVersion) {
|
||||
return { name, passed: true, durationMs };
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: 'Response missing implementationVersion field',
|
||||
durationMs,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Request failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function testExecuteToolAcceptsValidRequest(
|
||||
baseUrl: string,
|
||||
apiKey?: string
|
||||
): Promise<TestResult> {
|
||||
const start = Date.now();
|
||||
const name = 'POST /execute-tool accepts valid request';
|
||||
|
||||
try {
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
'X-TPMJS-Protocol-Version': '1.0',
|
||||
};
|
||||
|
||||
if (apiKey) {
|
||||
headers['Authorization'] = `Bearer ${apiKey}`;
|
||||
}
|
||||
|
||||
const response = await fetch(`${baseUrl}/execute-tool`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({
|
||||
packageName: '@anthropic-ai/sdk',
|
||||
name: 'default',
|
||||
params: {},
|
||||
}),
|
||||
});
|
||||
|
||||
const durationMs = Date.now() - start;
|
||||
|
||||
// We expect 200 even if the tool fails
|
||||
if (response.status === 200) {
|
||||
return { name, passed: true, durationMs };
|
||||
}
|
||||
|
||||
// 401 is acceptable if auth is required and not provided
|
||||
if (response.status === 401 && !apiKey) {
|
||||
return {
|
||||
name,
|
||||
passed: true,
|
||||
message: 'Authentication required (expected)',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Expected status 200, got ${response.status}`,
|
||||
durationMs,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Request failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function testExecuteToolReturnsStructuredResponse(
|
||||
baseUrl: string,
|
||||
apiKey?: string
|
||||
): Promise<TestResult> {
|
||||
const start = Date.now();
|
||||
const name = 'POST /execute-tool returns structured response';
|
||||
|
||||
try {
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
'X-TPMJS-Protocol-Version': '1.0',
|
||||
};
|
||||
|
||||
if (apiKey) {
|
||||
headers['Authorization'] = `Bearer ${apiKey}`;
|
||||
}
|
||||
|
||||
const response = await fetch(`${baseUrl}/execute-tool`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({
|
||||
packageName: '@anthropic-ai/sdk',
|
||||
name: 'default',
|
||||
params: {},
|
||||
}),
|
||||
});
|
||||
|
||||
const durationMs = Date.now() - start;
|
||||
|
||||
// Skip if auth required
|
||||
if (response.status === 401 && !apiKey) {
|
||||
return {
|
||||
name,
|
||||
passed: true,
|
||||
message: 'Skipped: Authentication required',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
const data = (await response.json()) as ExecuteToolResponse;
|
||||
|
||||
// Check required fields
|
||||
if (typeof data.success !== 'boolean') {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: 'Response missing "success" boolean field',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
if (typeof data.executionTimeMs !== 'number') {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: 'Response missing "executionTimeMs" number field',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
if (data.success && data.output === undefined) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: 'Successful response missing "output" field',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
if (!data.success && !data.error) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: 'Error response missing "error" field',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
return { name, passed: true, durationMs };
|
||||
} catch (error) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Request failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function testExecuteToolReturnsErrorForInvalidTool(
|
||||
baseUrl: string,
|
||||
apiKey?: string
|
||||
): Promise<TestResult> {
|
||||
const start = Date.now();
|
||||
const name = 'POST /execute-tool returns error for invalid tool';
|
||||
|
||||
try {
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
'X-TPMJS-Protocol-Version': '1.0',
|
||||
};
|
||||
|
||||
if (apiKey) {
|
||||
headers['Authorization'] = `Bearer ${apiKey}`;
|
||||
}
|
||||
|
||||
const response = await fetch(`${baseUrl}/execute-tool`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({
|
||||
packageName: '@tpmjs/nonexistent-package-12345',
|
||||
name: 'nonexistentTool',
|
||||
params: {},
|
||||
}),
|
||||
});
|
||||
|
||||
const durationMs = Date.now() - start;
|
||||
|
||||
// Skip if auth required
|
||||
if (response.status === 401 && !apiKey) {
|
||||
return {
|
||||
name,
|
||||
passed: true,
|
||||
message: 'Skipped: Authentication required',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
const data = (await response.json()) as ExecuteToolResponse;
|
||||
|
||||
if (data.success === false && data.error?.code) {
|
||||
return { name, passed: true, durationMs };
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: 'Expected error response with code for nonexistent package',
|
||||
durationMs,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Request failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function testCORSHeaders(baseUrl: string, _apiKey?: string): Promise<TestResult> {
|
||||
const start = Date.now();
|
||||
const name = 'CORS headers present';
|
||||
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/health`, {
|
||||
method: 'GET',
|
||||
});
|
||||
|
||||
const durationMs = Date.now() - start;
|
||||
|
||||
const allowOrigin = response.headers.get('access-control-allow-origin');
|
||||
|
||||
if (allowOrigin) {
|
||||
return { name, passed: true, durationMs };
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: 'Missing Access-Control-Allow-Origin header',
|
||||
durationMs,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Request failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function testOPTIONSPreflight(baseUrl: string, _apiKey?: string): Promise<TestResult> {
|
||||
const start = Date.now();
|
||||
const name = 'OPTIONS preflight works';
|
||||
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/execute-tool`, {
|
||||
method: 'OPTIONS',
|
||||
});
|
||||
|
||||
const durationMs = Date.now() - start;
|
||||
|
||||
if (response.status === 200 || response.status === 204) {
|
||||
const allowMethods = response.headers.get('access-control-allow-methods');
|
||||
const allowHeaders = response.headers.get('access-control-allow-headers');
|
||||
|
||||
if (allowMethods && allowHeaders) {
|
||||
return { name, passed: true, durationMs };
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: 'Missing CORS preflight headers',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Expected status 200 or 204, got ${response.status}`,
|
||||
durationMs,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Request failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function runCoreTests(baseUrl: string, apiKey?: string): Promise<TestSuite> {
|
||||
const results: TestResult[] = [];
|
||||
|
||||
// Run tests sequentially to avoid overwhelming the executor
|
||||
results.push(await testHealthReturns200(baseUrl, apiKey));
|
||||
results.push(await testHealthIncludesProtocolVersion(baseUrl, apiKey));
|
||||
results.push(await testHealthIncludesImplementationVersion(baseUrl, apiKey));
|
||||
results.push(await testExecuteToolAcceptsValidRequest(baseUrl, apiKey));
|
||||
results.push(await testExecuteToolReturnsStructuredResponse(baseUrl, apiKey));
|
||||
results.push(await testExecuteToolReturnsErrorForInvalidTool(baseUrl, apiKey));
|
||||
results.push(await testCORSHeaders(baseUrl, apiKey));
|
||||
results.push(await testOPTIONSPreflight(baseUrl, apiKey));
|
||||
|
||||
return {
|
||||
name: 'Core Requirements',
|
||||
level: 'core',
|
||||
results,
|
||||
};
|
||||
}
|
||||
434
packages/executor-test/src/tests/standard.ts
Normal file
434
packages/executor-test/src/tests/standard.ts
Normal file
|
|
@ -0,0 +1,434 @@
|
|||
import type { InfoResponse, TestResult, TestSuite } from '../types.js';
|
||||
|
||||
async function testInfoReturns200(baseUrl: string, _apiKey?: string): Promise<TestResult> {
|
||||
const start = Date.now();
|
||||
const name = 'GET /info returns 200';
|
||||
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/info`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'X-TPMJS-Protocol-Version': '1.0',
|
||||
},
|
||||
});
|
||||
|
||||
const durationMs = Date.now() - start;
|
||||
|
||||
if (response.status === 200) {
|
||||
return { name, passed: true, durationMs };
|
||||
}
|
||||
|
||||
if (response.status === 404) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: '/info endpoint not implemented (optional for Level 1)',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Expected status 200, got ${response.status}`,
|
||||
durationMs,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Request failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function testInfoIncludesCapabilities(
|
||||
baseUrl: string,
|
||||
_apiKey?: string
|
||||
): Promise<TestResult> {
|
||||
const start = Date.now();
|
||||
const name = 'GET /info includes capabilities';
|
||||
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/info`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'X-TPMJS-Protocol-Version': '1.0',
|
||||
},
|
||||
});
|
||||
|
||||
const durationMs = Date.now() - start;
|
||||
|
||||
if (response.status === 404) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: '/info endpoint not implemented',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
const data = (await response.json()) as InfoResponse;
|
||||
|
||||
if (!data.capabilities) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: 'Response missing "capabilities" field',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
const required = ['isolation', 'executionModes', 'maxExecutionTimeMs', 'maxRequestBodyBytes'];
|
||||
const missing = required.filter(
|
||||
(key) => data.capabilities[key as keyof typeof data.capabilities] === undefined
|
||||
);
|
||||
|
||||
if (missing.length > 0) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Missing capability fields: ${missing.join(', ')}`,
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
return { name, passed: true, durationMs };
|
||||
} catch (error) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Request failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function testInfoIncludesProtocolVersion(
|
||||
baseUrl: string,
|
||||
_apiKey?: string
|
||||
): Promise<TestResult> {
|
||||
const start = Date.now();
|
||||
const name = 'GET /info includes protocolVersion';
|
||||
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/info`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'X-TPMJS-Protocol-Version': '1.0',
|
||||
},
|
||||
});
|
||||
|
||||
const durationMs = Date.now() - start;
|
||||
|
||||
if (response.status === 404) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: '/info endpoint not implemented',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
const data = (await response.json()) as InfoResponse;
|
||||
|
||||
if (data.protocolVersion) {
|
||||
return { name, passed: true, durationMs };
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: 'Response missing "protocolVersion" field',
|
||||
durationMs,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Request failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function testInfoIsolationLevel(baseUrl: string, _apiKey?: string): Promise<TestResult> {
|
||||
const start = Date.now();
|
||||
const name = 'capabilities.isolation is valid';
|
||||
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/info`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'X-TPMJS-Protocol-Version': '1.0',
|
||||
},
|
||||
});
|
||||
|
||||
const durationMs = Date.now() - start;
|
||||
|
||||
if (response.status === 404) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: '/info endpoint not implemented',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
const data = (await response.json()) as InfoResponse;
|
||||
const validLevels = ['none', 'process', 'container', 'vm'];
|
||||
|
||||
if (validLevels.includes(data.capabilities?.isolation)) {
|
||||
return { name, passed: true, durationMs };
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Invalid isolation level: ${data.capabilities?.isolation}. Expected one of: ${validLevels.join(', ')}`,
|
||||
durationMs,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Request failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function testAuthenticationEnforced(baseUrl: string, apiKey?: string): Promise<TestResult> {
|
||||
const start = Date.now();
|
||||
const name = 'Authentication enforced when configured';
|
||||
|
||||
try {
|
||||
// First, make a request without auth to see if it's required
|
||||
const noAuthResponse = await fetch(`${baseUrl}/execute-tool`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-TPMJS-Protocol-Version': '1.0',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
packageName: '@anthropic-ai/sdk',
|
||||
name: 'default',
|
||||
params: {},
|
||||
}),
|
||||
});
|
||||
|
||||
const durationMs = Date.now() - start;
|
||||
|
||||
// If no auth is required, that's fine
|
||||
if (noAuthResponse.status !== 401) {
|
||||
return {
|
||||
name,
|
||||
passed: true,
|
||||
message: 'No authentication required',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
// If 401 without auth, verify it works with auth
|
||||
if (!apiKey) {
|
||||
return {
|
||||
name,
|
||||
passed: true,
|
||||
message: 'Authentication required (provide --api-key to fully test)',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
// Try with auth
|
||||
const authResponse = await fetch(`${baseUrl}/execute-tool`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-TPMJS-Protocol-Version': '1.0',
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
packageName: '@anthropic-ai/sdk',
|
||||
name: 'default',
|
||||
params: {},
|
||||
}),
|
||||
});
|
||||
|
||||
if (authResponse.status === 200) {
|
||||
return { name, passed: true, durationMs: Date.now() - start };
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Request with API key still failed: ${authResponse.status}`,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Request failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function testExecutionTimeoutEnforced(
|
||||
baseUrl: string,
|
||||
_apiKey?: string
|
||||
): Promise<TestResult> {
|
||||
const start = Date.now();
|
||||
const name = 'Execution timeout enforcement';
|
||||
|
||||
// This test just verifies the capability is advertised
|
||||
// Actual timeout testing would require a tool that hangs
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/info`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'X-TPMJS-Protocol-Version': '1.0',
|
||||
},
|
||||
});
|
||||
|
||||
const durationMs = Date.now() - start;
|
||||
|
||||
if (response.status === 404) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: '/info endpoint not implemented (cannot verify timeout)',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
const data = (await response.json()) as InfoResponse;
|
||||
|
||||
if (data.capabilities?.maxExecutionTimeMs && data.capabilities.maxExecutionTimeMs >= 60000) {
|
||||
return {
|
||||
name,
|
||||
passed: true,
|
||||
message: `maxExecutionTimeMs: ${data.capabilities.maxExecutionTimeMs}ms`,
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `maxExecutionTimeMs should be at least 60000ms, got: ${data.capabilities?.maxExecutionTimeMs}`,
|
||||
durationMs,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Request failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function testStructuredErrorCodes(baseUrl: string, apiKey?: string): Promise<TestResult> {
|
||||
const start = Date.now();
|
||||
const name = 'Structured error codes';
|
||||
|
||||
try {
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
'X-TPMJS-Protocol-Version': '1.0',
|
||||
};
|
||||
|
||||
if (apiKey) {
|
||||
headers['Authorization'] = `Bearer ${apiKey}`;
|
||||
}
|
||||
|
||||
const response = await fetch(`${baseUrl}/execute-tool`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({
|
||||
packageName: '@tpmjs/nonexistent-package-xyz-12345',
|
||||
name: 'nonexistentTool',
|
||||
params: {},
|
||||
}),
|
||||
});
|
||||
|
||||
const durationMs = Date.now() - start;
|
||||
|
||||
// Skip if auth required
|
||||
if (response.status === 401 && !apiKey) {
|
||||
return {
|
||||
name,
|
||||
passed: true,
|
||||
message: 'Skipped: Authentication required',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
const data = (await response.json()) as {
|
||||
success: boolean;
|
||||
error?: { code: string; message: string };
|
||||
};
|
||||
|
||||
if (!data.error?.code) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: 'Error response missing "code" field',
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
const validCodes = [
|
||||
'PACKAGE_NOT_FOUND',
|
||||
'TOOL_NOT_FOUND',
|
||||
'TOOL_INVALID',
|
||||
'TOOL_EXECUTION_ERROR',
|
||||
'EXECUTION_TIMEOUT',
|
||||
'INTERNAL_ERROR',
|
||||
];
|
||||
|
||||
if (validCodes.includes(data.error.code)) {
|
||||
return {
|
||||
name,
|
||||
passed: true,
|
||||
message: `Error code: ${data.error.code}`,
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Non-standard error code: ${data.error.code}`,
|
||||
durationMs,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
name,
|
||||
passed: false,
|
||||
message: `Request failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function runStandardTests(baseUrl: string, apiKey?: string): Promise<TestSuite> {
|
||||
const results: TestResult[] = [];
|
||||
|
||||
// Run tests sequentially
|
||||
results.push(await testInfoReturns200(baseUrl, apiKey));
|
||||
results.push(await testInfoIncludesCapabilities(baseUrl, apiKey));
|
||||
results.push(await testInfoIncludesProtocolVersion(baseUrl, apiKey));
|
||||
results.push(await testInfoIsolationLevel(baseUrl, apiKey));
|
||||
results.push(await testAuthenticationEnforced(baseUrl, apiKey));
|
||||
results.push(await testExecutionTimeoutEnforced(baseUrl, apiKey));
|
||||
results.push(await testStructuredErrorCodes(baseUrl, apiKey));
|
||||
|
||||
return {
|
||||
name: 'Standard Requirements',
|
||||
level: 'standard',
|
||||
results,
|
||||
};
|
||||
}
|
||||
72
packages/executor-test/src/types.ts
Normal file
72
packages/executor-test/src/types.ts
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
export interface TestResult {
|
||||
name: string;
|
||||
passed: boolean;
|
||||
message?: string;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
export interface TestSuite {
|
||||
name: string;
|
||||
level: 'core' | 'standard' | 'extended';
|
||||
results: TestResult[];
|
||||
}
|
||||
|
||||
export interface ComplianceResult {
|
||||
target: string;
|
||||
protocolVersion: string;
|
||||
timestamp: string;
|
||||
suites: TestSuite[];
|
||||
summary: {
|
||||
totalTests: number;
|
||||
passed: number;
|
||||
failed: number;
|
||||
coreCompliant: boolean;
|
||||
standardCompliant: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
export interface HealthResponse {
|
||||
status: string;
|
||||
protocolVersion: string;
|
||||
implementationVersion: string;
|
||||
runtime?: string;
|
||||
timestamp?: string;
|
||||
}
|
||||
|
||||
export interface ExecuteToolRequest {
|
||||
packageName: string;
|
||||
version?: string;
|
||||
name: string;
|
||||
params?: Record<string, unknown>;
|
||||
env?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface ExecuteToolResponse {
|
||||
success: boolean;
|
||||
output?: unknown;
|
||||
error?: {
|
||||
code: string;
|
||||
message: string;
|
||||
};
|
||||
executionTimeMs: number;
|
||||
}
|
||||
|
||||
export interface InfoResponse {
|
||||
name: string;
|
||||
version: string;
|
||||
protocolVersion: string;
|
||||
capabilities: {
|
||||
isolation: 'none' | 'process' | 'container' | 'vm';
|
||||
executionModes: string[];
|
||||
maxExecutionTimeMs: number;
|
||||
maxRequestBodyBytes: number;
|
||||
supportsStreaming?: boolean;
|
||||
supportsCallbacks?: boolean;
|
||||
supportsCaching?: boolean;
|
||||
};
|
||||
runtime?: {
|
||||
platform?: string;
|
||||
nodeVersion?: string;
|
||||
region?: string;
|
||||
};
|
||||
}
|
||||
11
packages/executor-test/tsconfig.json
Normal file
11
packages/executor-test/tsconfig.json
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"extends": "@tpmjs/tsconfig/base.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"incremental": false,
|
||||
"composite": false
|
||||
},
|
||||
"include": ["src"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
9
packages/executor-test/tsup.config.ts
Normal file
9
packages/executor-test/tsup.config.ts
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
import { defineConfig } from 'tsup';
|
||||
|
||||
export default defineConfig({
|
||||
entry: ['src/index.ts'],
|
||||
format: ['esm'],
|
||||
dts: true,
|
||||
clean: true,
|
||||
sourcemap: true,
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue