test: add comprehensive tests for CLI and scenarios
Add unit tests for: - CLI TpmClient (api-client.test.ts) - 26 tests covering authentication, tool execution, agent/collection/scenario management, error handling - Scenario evaluation (evaluate.test.ts) - 16 tests for regex assertions and verdict determination - Cosine similarity (similarity.test.ts) - 16 tests for embedding comparison Add integration tests for scenarios: - CRUD operations (scenarios-crud.integration.test.ts) - Run execution and quota management (scenarios-run.integration.test.ts) Configure vitest for CLI package with @tpmjs/test shared config.
This commit is contained in:
Ajax Davis
parent
9c731d582e
commit
a44f38eda9
8 changed files with 1491 additions and 1 deletions
222
apps/web/src/lib/scenarios/evaluate.test.ts
Normal file
222
apps/web/src/lib/scenarios/evaluate.test.ts
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
/**
|
||||
* Scenario Evaluation Unit Tests
|
||||
*
|
||||
* Tests for the evaluation logic including:
|
||||
* - Regex assertion matching
|
||||
* - Final verdict determination
|
||||
* - Assertion result handling
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { determineFinalVerdict, runAssertions } from './evaluate';
|
||||
|
||||
describe('runAssertions', () => {
|
||||
describe('regex assertions', () => {
|
||||
it('should pass when regex matches output', () => {
|
||||
const output = 'Hello, World!';
|
||||
const assertions = { regex: ['hello', 'world'] };
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toContain('regex:hello');
|
||||
expect(result.passed).toContain('regex:world');
|
||||
expect(result.failed).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should fail when regex does not match output', () => {
|
||||
const output = 'Goodbye, World!';
|
||||
const assertions = { regex: ['hello'] };
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.failed).toContain('regex:hello');
|
||||
expect(result.passed).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should handle multiple assertions with mixed results', () => {
|
||||
const output = 'The result is 42 and status is success';
|
||||
const assertions = { regex: ['result.*42', 'success', 'error', 'failure'] };
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toContain('regex:result.*42');
|
||||
expect(result.passed).toContain('regex:success');
|
||||
expect(result.failed).toContain('regex:error');
|
||||
expect(result.failed).toContain('regex:failure');
|
||||
});
|
||||
|
||||
it('should handle case-insensitive matching', () => {
|
||||
const output = 'SUCCESS';
|
||||
const assertions = { regex: ['success'] };
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toContain('regex:success');
|
||||
});
|
||||
|
||||
it('should handle invalid regex patterns gracefully', () => {
|
||||
const output = 'test';
|
||||
const assertions = { regex: ['[invalid', '(unclosed'] };
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.failed).toContain('regex:[invalid (invalid pattern)');
|
||||
expect(result.failed).toContain('regex:(unclosed (invalid pattern)');
|
||||
});
|
||||
|
||||
it('should handle empty regex array', () => {
|
||||
const output = 'test';
|
||||
const assertions = { regex: [] };
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toHaveLength(0);
|
||||
expect(result.failed).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should handle complex regex patterns', () => {
|
||||
const output = 'User john@example.com has ID 12345';
|
||||
const assertions = {
|
||||
regex: [
|
||||
'[a-z]+@[a-z]+\\.[a-z]+', // Email pattern
|
||||
'ID \\d+', // ID pattern
|
||||
'^User', // Starts with User
|
||||
],
|
||||
};
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toHaveLength(3);
|
||||
expect(result.failed).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('schema assertions', () => {
|
||||
it('should note when schema is provided', () => {
|
||||
const output = '{"name": "test"}';
|
||||
const assertions = { schema: { type: 'object' } };
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toContain('schema:provided (validation pending)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('combined assertions', () => {
|
||||
it('should handle both regex and schema assertions', () => {
|
||||
const output = '{"status": "success"}';
|
||||
const assertions = {
|
||||
regex: ['success'],
|
||||
schema: { type: 'object' },
|
||||
};
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toContain('regex:success');
|
||||
expect(result.passed).toContain('schema:provided (validation pending)');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('determineFinalVerdict', () => {
|
||||
describe('LLM evaluation only', () => {
|
||||
it('should return pass when LLM says pass', () => {
|
||||
const evaluation = {
|
||||
verdict: 'pass' as const,
|
||||
reason: 'Task completed successfully',
|
||||
confidence: 0.95,
|
||||
};
|
||||
|
||||
const result = determineFinalVerdict(evaluation);
|
||||
|
||||
expect(result).toBe('pass');
|
||||
});
|
||||
|
||||
it('should return fail when LLM says fail', () => {
|
||||
const evaluation = {
|
||||
verdict: 'fail' as const,
|
||||
reason: 'Task was not completed',
|
||||
confidence: 0.9,
|
||||
};
|
||||
|
||||
const result = determineFinalVerdict(evaluation);
|
||||
|
||||
expect(result).toBe('fail');
|
||||
});
|
||||
});
|
||||
|
||||
describe('with assertions', () => {
|
||||
it('should return pass when LLM passes and all assertions pass', () => {
|
||||
const evaluation = {
|
||||
verdict: 'pass' as const,
|
||||
reason: 'Task completed',
|
||||
confidence: 0.9,
|
||||
};
|
||||
const assertions = {
|
||||
passed: ['regex:success', 'regex:result'],
|
||||
failed: [],
|
||||
};
|
||||
|
||||
const result = determineFinalVerdict(evaluation, assertions);
|
||||
|
||||
expect(result).toBe('pass');
|
||||
});
|
||||
|
||||
it('should return fail when LLM passes but assertions fail', () => {
|
||||
const evaluation = {
|
||||
verdict: 'pass' as const,
|
||||
reason: 'Task completed',
|
||||
confidence: 0.9,
|
||||
};
|
||||
const assertions = {
|
||||
passed: ['regex:success'],
|
||||
failed: ['regex:expected_output'],
|
||||
};
|
||||
|
||||
const result = determineFinalVerdict(evaluation, assertions);
|
||||
|
||||
expect(result).toBe('fail');
|
||||
});
|
||||
|
||||
it('should return fail when LLM fails regardless of assertions', () => {
|
||||
const evaluation = {
|
||||
verdict: 'fail' as const,
|
||||
reason: 'Task failed',
|
||||
confidence: 0.95,
|
||||
};
|
||||
const assertions = {
|
||||
passed: ['regex:success'],
|
||||
failed: [],
|
||||
};
|
||||
|
||||
const result = determineFinalVerdict(evaluation, assertions);
|
||||
|
||||
expect(result).toBe('fail');
|
||||
});
|
||||
|
||||
it('should handle null assertions', () => {
|
||||
const evaluation = {
|
||||
verdict: 'pass' as const,
|
||||
reason: 'Task completed',
|
||||
confidence: 0.9,
|
||||
};
|
||||
|
||||
const result = determineFinalVerdict(evaluation, null);
|
||||
|
||||
expect(result).toBe('pass');
|
||||
});
|
||||
|
||||
it('should handle undefined assertions', () => {
|
||||
const evaluation = {
|
||||
verdict: 'pass' as const,
|
||||
reason: 'Task completed',
|
||||
confidence: 0.9,
|
||||
};
|
||||
|
||||
const result = determineFinalVerdict(evaluation, undefined);
|
||||
|
||||
expect(result).toBe('pass');
|
||||
});
|
||||
});
|
||||
});
|
||||
134
apps/web/src/lib/scenarios/similarity.test.ts
Normal file
134
apps/web/src/lib/scenarios/similarity.test.ts
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
/**
|
||||
* Scenario Similarity Unit Tests
|
||||
*
|
||||
* Tests for the similarity functions including:
|
||||
* - Cosine similarity calculation
|
||||
* - Edge cases and error handling
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { cosineSimilarity } from './similarity';
|
||||
|
||||
describe('cosineSimilarity', () => {
|
||||
it('should return 1 for identical vectors', () => {
|
||||
const vector = [1, 2, 3, 4, 5];
|
||||
const result = cosineSimilarity(vector, vector);
|
||||
expect(result).toBeCloseTo(1, 10);
|
||||
});
|
||||
|
||||
it('should return 0 for orthogonal vectors', () => {
|
||||
const a = [1, 0];
|
||||
const b = [0, 1];
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBeCloseTo(0, 10);
|
||||
});
|
||||
|
||||
it('should return -1 for opposite vectors', () => {
|
||||
const a = [1, 2, 3];
|
||||
const b = [-1, -2, -3];
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBeCloseTo(-1, 10);
|
||||
});
|
||||
|
||||
it('should handle normalized vectors correctly', () => {
|
||||
// Two unit vectors at 60 degrees have cosine similarity of 0.5
|
||||
const a = [1, 0];
|
||||
const b = [0.5, Math.sqrt(3) / 2]; // 60 degree rotation
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBeCloseTo(0.5, 5);
|
||||
});
|
||||
|
||||
it('should handle non-normalized vectors', () => {
|
||||
// Scaling a vector should not change the cosine similarity
|
||||
const a = [1, 2, 3];
|
||||
const b = [2, 4, 6]; // Same direction, scaled by 2
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBeCloseTo(1, 10);
|
||||
});
|
||||
|
||||
it('should handle vectors with different magnitudes', () => {
|
||||
const a = [3, 4]; // magnitude 5
|
||||
const b = [6, 8]; // magnitude 10, same direction
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBeCloseTo(1, 10);
|
||||
});
|
||||
|
||||
it('should throw error for vectors of different lengths', () => {
|
||||
const a = [1, 2, 3];
|
||||
const b = [1, 2];
|
||||
expect(() => cosineSimilarity(a, b)).toThrow('Vectors must have same length');
|
||||
});
|
||||
|
||||
it('should return 0 for zero vectors', () => {
|
||||
const a = [0, 0, 0];
|
||||
const b = [1, 2, 3];
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBe(0);
|
||||
});
|
||||
|
||||
it('should return 0 for two zero vectors', () => {
|
||||
const a = [0, 0, 0];
|
||||
const b = [0, 0, 0];
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBe(0);
|
||||
});
|
||||
|
||||
it('should handle high-dimensional vectors', () => {
|
||||
// Simulate embedding vectors (typically 1536 dimensions for text-embedding-3-small)
|
||||
const dim = 1536;
|
||||
const a = Array.from({ length: dim }, (_, i) => Math.sin(i));
|
||||
const b = Array.from({ length: dim }, (_, i) => Math.sin(i));
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBeCloseTo(1, 5);
|
||||
});
|
||||
|
||||
it('should handle high-dimensional dissimilar vectors', () => {
|
||||
const dim = 1536;
|
||||
const a = Array.from({ length: dim }, (_, i) => Math.sin(i));
|
||||
const b = Array.from({ length: dim }, (_, i) => Math.cos(i));
|
||||
const result = cosineSimilarity(a, b);
|
||||
// Should be between -1 and 1, but not 1
|
||||
expect(result).toBeLessThan(1);
|
||||
expect(result).toBeGreaterThanOrEqual(-1);
|
||||
});
|
||||
|
||||
it('should produce consistent results for partial similarity', () => {
|
||||
const a = [1, 0, 1, 0];
|
||||
const b = [1, 1, 0, 0];
|
||||
const result = cosineSimilarity(a, b);
|
||||
// Expected: (1*1 + 0*1 + 1*0 + 0*0) / (sqrt(2) * sqrt(2)) = 1/2 = 0.5
|
||||
expect(result).toBeCloseTo(0.5, 10);
|
||||
});
|
||||
|
||||
it('should handle negative values correctly', () => {
|
||||
const a = [-1, 2, -3];
|
||||
const b = [1, -2, 3];
|
||||
const result = cosineSimilarity(a, b);
|
||||
// Opposite directions
|
||||
expect(result).toBeCloseTo(-1, 10);
|
||||
});
|
||||
|
||||
it('should handle mixed positive and negative values', () => {
|
||||
const a = [1, -1, 1];
|
||||
const b = [1, 1, -1];
|
||||
const result = cosineSimilarity(a, b);
|
||||
// (1 - 1 - 1) / (sqrt(3) * sqrt(3)) = -1/3
|
||||
expect(result).toBeCloseTo(-1 / 3, 10);
|
||||
});
|
||||
|
||||
it('should handle very small values without underflow', () => {
|
||||
const a = [1e-10, 2e-10, 3e-10];
|
||||
const b = [1e-10, 2e-10, 3e-10];
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBeCloseTo(1, 5);
|
||||
});
|
||||
|
||||
it('should handle empty arrays', () => {
|
||||
const a: number[] = [];
|
||||
const b: number[] = [];
|
||||
const result = cosineSimilarity(a, b);
|
||||
// 0/0 case, returns 0 per implementation
|
||||
expect(result).toBe(0);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,233 @@
|
|||
/**
|
||||
* Scenarios CRUD Integration Tests
|
||||
*
|
||||
* Tests the scenario API endpoints including:
|
||||
* - Creating scenarios
|
||||
* - Listing scenarios
|
||||
* - Getting scenario details
|
||||
* - Updating scenarios
|
||||
* - Deleting scenarios
|
||||
*
|
||||
* NOTE: These are integration tests that require a running server.
|
||||
* Run `pnpm dev --filter=@tpmjs/web` first, then run tests.
|
||||
* Tests will be skipped if server is not available.
|
||||
*
|
||||
* To run integration tests manually:
|
||||
* INTEGRATION_TESTS=true pnpm --filter=@tpmjs/web test
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
|
||||
// Skip integration tests unless explicitly enabled
|
||||
const INTEGRATION_TESTS_ENABLED = process.env.INTEGRATION_TESTS === 'true';
|
||||
const BASE_URL = process.env.TEST_BASE_URL || 'http://localhost:3000';
|
||||
|
||||
// Check if server is available
|
||||
async function isServerAvailable(): Promise<boolean> {
|
||||
if (!INTEGRATION_TESTS_ENABLED) return false;
|
||||
try {
|
||||
const response = await fetch(`${BASE_URL}/api/health`, { signal: AbortSignal.timeout(2000) });
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
describe.skipIf(!INTEGRATION_TESTS_ENABLED)('Scenarios CRUD Integration', () => {
|
||||
let serverAvailable = false;
|
||||
let testCollection: { id: string; slug: string; userId: string } | null = null;
|
||||
const testScenarioId: string | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
serverAvailable = await isServerAvailable();
|
||||
if (!serverAvailable) {
|
||||
console.warn('⚠️ Server not available - skipping scenario integration tests');
|
||||
return;
|
||||
}
|
||||
|
||||
// Find a test collection to use
|
||||
testCollection = await prisma.collection.findFirst({
|
||||
where: { isPublic: true },
|
||||
select: { id: true, slug: true, userId: true },
|
||||
});
|
||||
|
||||
if (!testCollection) {
|
||||
console.warn('⚠️ No public collection found - skipping scenario tests');
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
try {
|
||||
// Clean up test scenario if created
|
||||
if (testScenarioId) {
|
||||
await prisma.scenario.delete({ where: { id: testScenarioId } }).catch(() => {});
|
||||
}
|
||||
await prisma.$disconnect();
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
});
|
||||
|
||||
describe('GET /api/scenarios', () => {
|
||||
it('should list public scenarios', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios?limit=10`);
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const result = await response.json();
|
||||
expect(result.success).toBe(true);
|
||||
expect(Array.isArray(result.data)).toBe(true);
|
||||
expect(result.pagination).toBeDefined();
|
||||
});
|
||||
|
||||
it('should support pagination', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios?limit=5&offset=0`);
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const result = await response.json();
|
||||
expect(result.pagination.limit).toBe(5);
|
||||
expect(result.pagination.offset).toBe(0);
|
||||
});
|
||||
|
||||
it('should filter by collection', async () => {
|
||||
if (!serverAvailable || !testCollection) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios?collectionId=${testCollection.id}`);
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const result = await response.json();
|
||||
expect(result.success).toBe(true);
|
||||
|
||||
// All returned scenarios should belong to the collection
|
||||
for (const scenario of result.data) {
|
||||
expect(scenario.collectionId).toBe(testCollection.id);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /api/scenarios/featured', () => {
|
||||
it('should return featured scenarios', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios/featured`);
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const result = await response.json();
|
||||
expect(result.success).toBe(true);
|
||||
expect(Array.isArray(result.data)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /api/collections/[id]/scenarios/generate', () => {
|
||||
it('should require authentication', async () => {
|
||||
if (!serverAvailable || !testCollection) return;
|
||||
|
||||
const response = await fetch(
|
||||
`${BASE_URL}/api/collections/${testCollection.id}/scenarios/generate`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ count: 1 }),
|
||||
}
|
||||
);
|
||||
|
||||
// Should return 401 without auth
|
||||
expect(response.status).toBe(401);
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /api/scenarios/[id]', () => {
|
||||
it('should return scenario details', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
// First find a scenario
|
||||
const listResponse = await fetch(`${BASE_URL}/api/scenarios?limit=1`);
|
||||
const listResult = await listResponse.json();
|
||||
|
||||
if (listResult.data.length === 0) {
|
||||
console.log('No scenarios found to test');
|
||||
return;
|
||||
}
|
||||
|
||||
const scenarioId = listResult.data[0].id;
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios/${scenarioId}`);
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const result = await response.json();
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data.id).toBe(scenarioId);
|
||||
expect(result.data.name).toBeDefined();
|
||||
expect(result.data.prompt).toBeDefined();
|
||||
});
|
||||
|
||||
it('should return 404 for non-existent scenario', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios/non-existent-scenario-id-12345`);
|
||||
|
||||
expect(response.status).toBe(404);
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /api/scenarios/[id]/runs', () => {
|
||||
it('should list scenario runs', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
// Find a scenario with runs
|
||||
const listResponse = await fetch(`${BASE_URL}/api/scenarios?limit=10`);
|
||||
const listResult = await listResponse.json();
|
||||
|
||||
if (listResult.data.length === 0) {
|
||||
console.log('No scenarios found');
|
||||
return;
|
||||
}
|
||||
|
||||
// Try each scenario until we find one (may or may not have runs)
|
||||
const scenarioId = listResult.data[0].id;
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios/${scenarioId}/runs`);
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const result = await response.json();
|
||||
expect(result.success).toBe(true);
|
||||
expect(Array.isArray(result.data)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /api/scenarios/check-similarity', () => {
|
||||
it('should check scenario similarity', async () => {
|
||||
if (!serverAvailable || !testCollection) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios/check-similarity`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
collectionId: testCollection.id,
|
||||
prompt: 'Test prompt for similarity checking',
|
||||
}),
|
||||
});
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const result = await response.json();
|
||||
expect(result.success).toBe(true);
|
||||
expect(typeof result.data.hasSimilar).toBe('boolean');
|
||||
});
|
||||
|
||||
it('should require collectionId and prompt', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios/check-similarity`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({}),
|
||||
});
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,193 @@
|
|||
/**
|
||||
* Scenario Execution Integration Tests
|
||||
*
|
||||
* Tests the scenario run API endpoint including:
|
||||
* - Running scenarios
|
||||
* - Quota management
|
||||
* - Result evaluation
|
||||
*
|
||||
* NOTE: These tests require authentication and may consume API quotas.
|
||||
* Run with caution in development environments only.
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
|
||||
// Skip integration tests unless explicitly enabled
|
||||
const INTEGRATION_TESTS_ENABLED = process.env.INTEGRATION_TESTS === 'true';
|
||||
const BASE_URL = process.env.TEST_BASE_URL || 'http://localhost:3000';
|
||||
|
||||
// Check if server is available
|
||||
async function isServerAvailable(): Promise<boolean> {
|
||||
if (!INTEGRATION_TESTS_ENABLED) return false;
|
||||
try {
|
||||
const response = await fetch(`${BASE_URL}/api/health`, { signal: AbortSignal.timeout(2000) });
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
describe.skipIf(!INTEGRATION_TESTS_ENABLED)('Scenario Execution Integration', () => {
|
||||
let serverAvailable = false;
|
||||
let testScenario: { id: string; name: string; collectionId: string } | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
serverAvailable = await isServerAvailable();
|
||||
if (!serverAvailable) {
|
||||
console.warn('⚠️ Server not available - skipping scenario execution tests');
|
||||
return;
|
||||
}
|
||||
|
||||
// Find a test scenario
|
||||
testScenario = await prisma.scenario.findFirst({
|
||||
where: {
|
||||
collection: { isPublic: true },
|
||||
},
|
||||
select: { id: true, name: true, collectionId: true },
|
||||
});
|
||||
|
||||
if (!testScenario) {
|
||||
console.warn('⚠️ No public scenario found - some tests will be skipped');
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (!serverAvailable) return;
|
||||
await prisma.$disconnect();
|
||||
});
|
||||
|
||||
describe('POST /api/scenarios/[id]/run', () => {
|
||||
it('should require authentication', async () => {
|
||||
if (!serverAvailable || !testScenario) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios/${testScenario.id}/run`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
|
||||
// Should return 401 without auth
|
||||
expect(response.status).toBe(401);
|
||||
});
|
||||
|
||||
it('should return 404 for non-existent scenario', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios/non-existent-id-12345/run`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
|
||||
// Should return 401 (auth required first) or 404
|
||||
expect([401, 404]).toContain(response.status);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Scenario metrics', () => {
|
||||
it('should have quality score between 0 and 1', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const scenarios = await prisma.scenario.findMany({
|
||||
take: 10,
|
||||
select: { id: true, qualityScore: true },
|
||||
});
|
||||
|
||||
for (const scenario of scenarios) {
|
||||
expect(scenario.qualityScore).toBeGreaterThanOrEqual(0);
|
||||
expect(scenario.qualityScore).toBeLessThanOrEqual(1);
|
||||
}
|
||||
});
|
||||
|
||||
it('should track consecutive passes and fails', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const scenarios = await prisma.scenario.findMany({
|
||||
take: 10,
|
||||
select: {
|
||||
id: true,
|
||||
consecutivePasses: true,
|
||||
consecutiveFails: true,
|
||||
totalRuns: true,
|
||||
},
|
||||
});
|
||||
|
||||
for (const scenario of scenarios) {
|
||||
expect(scenario.consecutivePasses).toBeGreaterThanOrEqual(0);
|
||||
expect(scenario.consecutiveFails).toBeGreaterThanOrEqual(0);
|
||||
expect(scenario.totalRuns).toBeGreaterThanOrEqual(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Scenario runs history', () => {
|
||||
it('should have valid run records', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const runs = await prisma.scenarioRun.findMany({
|
||||
take: 10,
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
scenarioId: true,
|
||||
userId: true,
|
||||
createdAt: true,
|
||||
},
|
||||
});
|
||||
|
||||
for (const run of runs) {
|
||||
expect(['pending', 'running', 'pass', 'fail', 'error']).toContain(run.status);
|
||||
expect(run.scenarioId).toBeDefined();
|
||||
expect(run.userId).toBeDefined();
|
||||
expect(run.createdAt).toBeInstanceOf(Date);
|
||||
}
|
||||
});
|
||||
|
||||
it('should track token usage for completed runs', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const completedRuns = await prisma.scenarioRun.findMany({
|
||||
where: { status: { in: ['pass', 'fail'] } },
|
||||
take: 10,
|
||||
select: {
|
||||
id: true,
|
||||
inputTokens: true,
|
||||
outputTokens: true,
|
||||
totalTokens: true,
|
||||
executionTimeMs: true,
|
||||
},
|
||||
});
|
||||
|
||||
for (const run of completedRuns) {
|
||||
// Completed runs should have token counts
|
||||
if (run.inputTokens !== null) {
|
||||
expect(run.inputTokens).toBeGreaterThanOrEqual(0);
|
||||
}
|
||||
if (run.outputTokens !== null) {
|
||||
expect(run.outputTokens).toBeGreaterThanOrEqual(0);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Quota management', () => {
|
||||
it('should track user quotas', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const quotas = await prisma.scenarioQuota.findMany({
|
||||
take: 5,
|
||||
select: {
|
||||
userId: true,
|
||||
dailyLimit: true,
|
||||
dailyUsed: true,
|
||||
lastResetAt: true,
|
||||
},
|
||||
});
|
||||
|
||||
for (const quota of quotas) {
|
||||
expect(quota.dailyLimit).toBeGreaterThan(0);
|
||||
expect(quota.dailyUsed).toBeGreaterThanOrEqual(0);
|
||||
expect(quota.dailyUsed).toBeLessThanOrEqual(quota.dailyLimit);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -73,6 +73,7 @@
|
|||
"scripts": {
|
||||
"build": "tsup && oclif manifest",
|
||||
"dev": "tsup --watch",
|
||||
"test": "vitest",
|
||||
"type-check": "tsc --noEmit",
|
||||
"clean": "rm -rf dist .turbo oclif.manifest.json",
|
||||
"postpack": "rm -f oclif.manifest.json",
|
||||
|
|
@ -91,11 +92,13 @@
|
|||
"cli-table3": "^0.6.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tpmjs/test": "workspace:*",
|
||||
"@tpmjs/tsconfig": "workspace:*",
|
||||
"@types/node": "^22.15.29",
|
||||
"oclif": "^4.17.35",
|
||||
"tsup": "^8.5.1",
|
||||
"typescript": "^5.9.3"
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^4.0.16"
|
||||
},
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
|
|
|
|||
575
packages/cli/src/lib/api-client.test.ts
Normal file
575
packages/cli/src/lib/api-client.test.ts
Normal file
|
|
@ -0,0 +1,575 @@
|
|||
/**
|
||||
* TpmClient Unit Tests
|
||||
*
|
||||
* Tests for the CLI API client including:
|
||||
* - Request formatting
|
||||
* - Error handling
|
||||
* - Authentication
|
||||
* - Response parsing
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { ApiError, TpmClient } from './api-client.js';
|
||||
|
||||
// Mock the config module
|
||||
vi.mock('./config.js', () => ({
|
||||
getApiKey: vi.fn(() => undefined),
|
||||
getApiUrl: vi.fn(() => 'https://default.api.com'),
|
||||
}));
|
||||
|
||||
// Mock fetch globally
|
||||
const mockFetch = vi.fn();
|
||||
global.fetch = mockFetch;
|
||||
|
||||
describe('TpmClient', () => {
|
||||
let client: TpmClient;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
client = new TpmClient({
|
||||
baseUrl: 'https://api.test.com',
|
||||
apiKey: 'test-api-key',
|
||||
timeout: 5000,
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('constructor', () => {
|
||||
it('should create client with custom options', () => {
|
||||
const customClient = new TpmClient({
|
||||
baseUrl: 'https://custom.api.com',
|
||||
apiKey: 'custom-key',
|
||||
timeout: 10000,
|
||||
});
|
||||
|
||||
expect(customClient.isAuthenticated()).toBe(true);
|
||||
});
|
||||
|
||||
it('should create client without API key', () => {
|
||||
// When no apiKey option is passed, and config returns undefined, client is not authenticated
|
||||
const unauthClient = new TpmClient({
|
||||
baseUrl: 'https://api.test.com',
|
||||
apiKey: undefined, // Explicitly set to undefined to override config default
|
||||
});
|
||||
|
||||
expect(unauthClient.isAuthenticated()).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('health', () => {
|
||||
it('should call health endpoint', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({ status: 'ok', timestamp: '2024-01-01T00:00:00Z' }),
|
||||
});
|
||||
|
||||
const result = await client.health();
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://api.test.com/health',
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: 'Bearer test-api-key',
|
||||
}),
|
||||
})
|
||||
);
|
||||
expect(result.status).toBe('ok');
|
||||
});
|
||||
});
|
||||
|
||||
describe('searchTools', () => {
|
||||
it('should search tools with query', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [{ id: '1', name: 'test-tool' }],
|
||||
pagination: { limit: 20, offset: 0, hasMore: false },
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await client.searchTools({ query: 'test', limit: 10 });
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://api.test.com/tools?q=test&limit=10',
|
||||
expect.any(Object)
|
||||
);
|
||||
expect(result.data).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('should search tools with category filter', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [],
|
||||
pagination: { limit: 20, offset: 0, hasMore: false },
|
||||
}),
|
||||
});
|
||||
|
||||
await client.searchTools({ category: 'sandbox' });
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://api.test.com/tools?category=sandbox',
|
||||
expect.any(Object)
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle empty search', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [],
|
||||
pagination: { limit: 20, offset: 0, hasMore: false },
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await client.searchTools({});
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith('https://api.test.com/tools', expect.any(Object));
|
||||
expect(result.data).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('listAgents', () => {
|
||||
it('should list agents with pagination', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [
|
||||
{ id: '1', uid: 'agent-1', name: 'Test Agent' },
|
||||
{ id: '2', uid: 'agent-2', name: 'Another Agent' },
|
||||
],
|
||||
pagination: { limit: 10, offset: 0, hasMore: true },
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await client.listAgents({ limit: 10, offset: 0 });
|
||||
|
||||
// Note: offset=0 is falsy so it won't be included in the URL
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://api.test.com/agents?limit=10',
|
||||
expect.any(Object)
|
||||
);
|
||||
expect(result.data).toHaveLength(2);
|
||||
expect(result.pagination.hasMore).toBe(true);
|
||||
});
|
||||
|
||||
it('should include offset when non-zero', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [],
|
||||
pagination: { limit: 10, offset: 10, hasMore: false },
|
||||
}),
|
||||
});
|
||||
|
||||
await client.listAgents({ limit: 10, offset: 10 });
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://api.test.com/agents?limit=10&offset=10',
|
||||
expect.any(Object)
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('createAgent', () => {
|
||||
it('should create an agent', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
success: true,
|
||||
data: {
|
||||
id: 'new-agent-id',
|
||||
uid: 'new-agent',
|
||||
name: 'New Agent',
|
||||
provider: 'openai',
|
||||
modelId: 'gpt-4',
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await client.createAgent({
|
||||
name: 'New Agent',
|
||||
uid: 'new-agent',
|
||||
provider: 'openai',
|
||||
modelId: 'gpt-4',
|
||||
});
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://api.test.com/agents',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
name: 'New Agent',
|
||||
uid: 'new-agent',
|
||||
provider: 'openai',
|
||||
modelId: 'gpt-4',
|
||||
}),
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data?.uid).toBe('new-agent');
|
||||
});
|
||||
});
|
||||
|
||||
describe('listCollections', () => {
|
||||
it('should list collections', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [
|
||||
{ id: '1', name: 'Collection 1', slug: 'collection-1' },
|
||||
{ id: '2', name: 'Collection 2', slug: 'collection-2' },
|
||||
],
|
||||
pagination: { limit: 20, offset: 0, hasMore: false },
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await client.listCollections();
|
||||
|
||||
expect(result.data).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('listScenarios', () => {
|
||||
it('should list scenarios with filters', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [{ id: '1', name: 'Scenario 1', qualityScore: 0.9 }],
|
||||
pagination: { limit: 20, offset: 0, hasMore: false },
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await client.listScenarios({
|
||||
collectionId: 'col-1',
|
||||
sortBy: 'qualityScore',
|
||||
});
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://api.test.com/scenarios?collectionId=col-1&sortBy=qualityScore',
|
||||
expect.any(Object)
|
||||
);
|
||||
expect(result.data).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('runScenario', () => {
|
||||
it('should run a scenario', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
success: true,
|
||||
data: {
|
||||
id: 'run-1',
|
||||
status: 'pass',
|
||||
success: true,
|
||||
evaluator: {
|
||||
model: 'gpt-4.1-mini',
|
||||
verdict: 'pass',
|
||||
reason: 'Task completed successfully',
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await client.runScenario('scenario-1');
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://api.test.com/scenarios/scenario-1/run',
|
||||
expect.objectContaining({ method: 'POST' })
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data?.status).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
describe('error handling', () => {
|
||||
it('should throw ApiError on HTTP error', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
status: 404,
|
||||
json: async () => ({ message: 'Not found' }),
|
||||
});
|
||||
|
||||
await expect(client.getAgent('non-existent')).rejects.toThrow(ApiError);
|
||||
});
|
||||
|
||||
it('should include status code in ApiError', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
status: 401,
|
||||
json: async () => ({ error: 'Unauthorized' }),
|
||||
});
|
||||
|
||||
try {
|
||||
await client.whoami();
|
||||
expect.fail('Should have thrown');
|
||||
} catch (error) {
|
||||
expect(error).toBeInstanceOf(ApiError);
|
||||
expect((error as ApiError).statusCode).toBe(401);
|
||||
}
|
||||
});
|
||||
|
||||
it('should handle network errors', async () => {
|
||||
mockFetch.mockRejectedValueOnce(new Error('Network error'));
|
||||
|
||||
await expect(client.health()).rejects.toThrow('Network error');
|
||||
});
|
||||
});
|
||||
|
||||
describe('authentication', () => {
|
||||
it('should include Authorization header when API key is set', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({ success: true, data: {} }),
|
||||
});
|
||||
|
||||
await client.whoami();
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
Authorization: 'Bearer test-api-key',
|
||||
}),
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should not include Authorization header when no API key', async () => {
|
||||
// Create client with explicitly undefined apiKey
|
||||
const unauthClient = new TpmClient({
|
||||
baseUrl: 'https://api.test.com',
|
||||
apiKey: undefined,
|
||||
});
|
||||
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({ status: 'ok' }),
|
||||
});
|
||||
|
||||
await unauthClient.health();
|
||||
|
||||
const callArgs = mockFetch.mock.calls[0];
|
||||
expect(callArgs).toBeDefined();
|
||||
const callHeaders = callArgs?.[1]?.headers as Record<string, string> | undefined;
|
||||
expect(callHeaders?.Authorization).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('tool execution', () => {
|
||||
it('should execute a tool with parameters', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({ result: 'executed successfully' }),
|
||||
});
|
||||
|
||||
const result = await client.executeTool('my-tool', { input: 'test' });
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://api.test.com/tools/my-tool/execute',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ input: 'test' }),
|
||||
})
|
||||
);
|
||||
expect(result).toEqual({ result: 'executed successfully' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('collection management', () => {
|
||||
it('should create a collection', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
success: true,
|
||||
data: { id: 'col-1', name: 'My Collection', isPublic: true },
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await client.createCollection({
|
||||
name: 'My Collection',
|
||||
isPublic: true,
|
||||
});
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://api.test.com/collections',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ name: 'My Collection', isPublic: true }),
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('should delete a collection', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({ success: true }),
|
||||
});
|
||||
|
||||
const result = await client.deleteCollection('col-1');
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://api.test.com/collections/col-1',
|
||||
expect.objectContaining({ method: 'DELETE' })
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('should update a collection', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
success: true,
|
||||
data: { id: 'col-1', name: 'Updated Name', isPublic: false },
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await client.updateCollection('col-1', {
|
||||
name: 'Updated Name',
|
||||
isPublic: false,
|
||||
});
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://api.test.com/collections/col-1',
|
||||
expect.objectContaining({
|
||||
method: 'PATCH',
|
||||
body: JSON.stringify({ name: 'Updated Name', isPublic: false }),
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('scenario management', () => {
|
||||
it('should get scenario runs', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [
|
||||
{ id: 'run-1', status: 'pass' },
|
||||
{ id: 'run-2', status: 'fail' },
|
||||
],
|
||||
pagination: { limit: 10, offset: 0, hasMore: false },
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await client.getScenarioRuns('scenario-1', { limit: 10 });
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://api.test.com/scenarios/scenario-1/runs?limit=10',
|
||||
expect.any(Object)
|
||||
);
|
||||
expect(result.data).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('should create a scenario', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
success: true,
|
||||
data: {
|
||||
id: 'scenario-1',
|
||||
prompt: 'Test the feature',
|
||||
name: 'Test Scenario',
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await client.createScenario({
|
||||
collectionId: 'col-1',
|
||||
prompt: 'Test the feature',
|
||||
name: 'Test Scenario',
|
||||
});
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://api.test.com/scenarios',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
collectionId: 'col-1',
|
||||
prompt: 'Test the feature',
|
||||
name: 'Test Scenario',
|
||||
}),
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('should generate scenarios for a collection', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
success: true,
|
||||
data: {
|
||||
scenarios: [
|
||||
{ scenario: { id: 's1', prompt: 'Generated 1' } },
|
||||
{ scenario: { id: 's2', prompt: 'Generated 2' } },
|
||||
],
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await client.generateScenarios('col-1', { count: 5 });
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://api.test.com/collections/col-1/scenarios/generate',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ count: 5 }),
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data?.scenarios).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('stats', () => {
|
||||
it('should get stats', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
success: true,
|
||||
data: {
|
||||
tools: { total: 100, official: 10, healthyImport: 95, healthyExecution: 90 },
|
||||
packages: { total: 50, official: 5 },
|
||||
categories: [{ name: 'sandbox', count: 20 }],
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await client.getStats();
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith('https://api.test.com/stats', expect.any(Object));
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data?.tools.total).toBe(100);
|
||||
});
|
||||
});
|
||||
|
||||
describe('user API keys', () => {
|
||||
it('should list API keys', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
success: true,
|
||||
data: [{ id: 'key-1', name: 'Test Key', keyPrefix: 'tpmjs_sk_...' }],
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await client.listApiKeys();
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://api.test.com/user/tpmjs-api-keys',
|
||||
expect.any(Object)
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
11
packages/cli/vitest.config.ts
Normal file
11
packages/cli/vitest.config.ts
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: true,
|
||||
coverage: {
|
||||
provider: 'v8',
|
||||
reporter: ['text', 'json', 'html'],
|
||||
},
|
||||
},
|
||||
});
|
||||
119
pnpm-lock.yaml
generated
119
pnpm-lock.yaml
generated
|
|
@ -230,6 +230,9 @@ importers:
|
|||
'@ai-sdk/anthropic':
|
||||
specifier: ^3.0.9
|
||||
version: 3.0.9(zod@4.3.5)
|
||||
'@ai-sdk/devtools':
|
||||
specifier: ^0.0.8
|
||||
version: 0.0.8
|
||||
'@ai-sdk/google':
|
||||
specifier: ^3.0.6
|
||||
version: 3.0.6(zod@4.3.5)
|
||||
|
|
@ -479,6 +482,9 @@ importers:
|
|||
specifier: ^1.1.1
|
||||
version: 1.1.1
|
||||
devDependencies:
|
||||
'@tpmjs/test':
|
||||
specifier: workspace:*
|
||||
version: link:../test
|
||||
'@tpmjs/tsconfig':
|
||||
specifier: workspace:*
|
||||
version: link:../config/tsconfig
|
||||
|
|
@ -494,6 +500,9 @@ importers:
|
|||
typescript:
|
||||
specifier: ^5.9.3
|
||||
version: 5.9.3
|
||||
vitest:
|
||||
specifier: ^4.0.16
|
||||
version: 4.0.16(@opentelemetry/api@1.9.0)(@types/node@22.19.5)(happy-dom@20.1.0)(jiti@2.6.1)(jsdom@27.4.0)(lightningcss@1.30.2)(msw@2.12.7(@types/node@22.19.5)(typescript@5.9.3))(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)
|
||||
|
||||
packages/config:
|
||||
dependencies:
|
||||
|
|
@ -4089,6 +4098,10 @@ packages:
|
|||
peerDependencies:
|
||||
zod: ^3.25.76 || ^4.1.8
|
||||
|
||||
'@ai-sdk/devtools@0.0.8':
|
||||
resolution: {integrity: sha512-oJXCdZ0svdQiC50FAw39uIp26qhl7zo56C9KHoN67LQZkCFNzgYyhS8ZxuUOWTQJGB4DKx4WpOyFGbzNT2VSyw==}
|
||||
hasBin: true
|
||||
|
||||
'@ai-sdk/gateway@2.0.0-beta.68':
|
||||
resolution: {integrity: sha512-eYv3hBfu/M+0XmxE1RoH4X7uhYplNbSDvRClzirKzTY+ZM5yQwP86L3DqLR7Y9/vH7yrQaSIwsBT3OMmcBnGaw==}
|
||||
engines: {node: '>=18'}
|
||||
|
|
@ -4197,6 +4210,10 @@ packages:
|
|||
resolution: {integrity: sha512-HrEmNt/BH/hkQ7zpi2o6N3k1ZR1QTb7z85WYhYygiTxOQuaml4CMtHCWRbric5WPU+RNsYI7r1EpyVQMKO1pYw==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
'@ai-sdk/provider@3.0.5':
|
||||
resolution: {integrity: sha512-2Xmoq6DBJqmSl80U6V9z5jJSJP7ehaJJQMy2iFUqTay06wdCqTnPVBBQbtEL8RCChenL+q5DC5H5WzU3vV3v8w==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
'@ai-sdk/react@3.0.23':
|
||||
resolution: {integrity: sha512-SlGHxj3IrY/t2zf/kKHac8DiwKOPWCSBiaxGejijjST1mlAS5Hjp9CchmQNswNh9NxZQ7mow2C3t2jC/PLfLbw==}
|
||||
engines: {node: '>=18'}
|
||||
|
|
@ -13278,6 +13295,12 @@ snapshots:
|
|||
'@ai-sdk/provider-utils': 4.0.4(zod@4.3.5)
|
||||
zod: 4.3.5
|
||||
|
||||
'@ai-sdk/devtools@0.0.8':
|
||||
dependencies:
|
||||
'@ai-sdk/provider': 3.0.5
|
||||
'@hono/node-server': 1.19.8(hono@4.10.6)
|
||||
hono: 4.10.6
|
||||
|
||||
'@ai-sdk/gateway@2.0.0-beta.68(effect@3.18.4)(zod@4.3.5)':
|
||||
dependencies:
|
||||
'@ai-sdk/provider': 3.0.0-beta.22
|
||||
|
|
@ -13412,6 +13435,10 @@ snapshots:
|
|||
dependencies:
|
||||
json-schema: 0.4.0
|
||||
|
||||
'@ai-sdk/provider@3.0.5':
|
||||
dependencies:
|
||||
json-schema: 0.4.0
|
||||
|
||||
'@ai-sdk/react@3.0.23(react@19.2.3)(zod@4.3.5)':
|
||||
dependencies:
|
||||
'@ai-sdk/provider-utils': 4.0.4(zod@4.3.5)
|
||||
|
|
@ -17232,6 +17259,15 @@ snapshots:
|
|||
chai: 6.2.2
|
||||
tinyrainbow: 3.0.3
|
||||
|
||||
'@vitest/mocker@4.0.16(msw@2.12.7(@types/node@22.19.5)(typescript@5.9.3))(vite@7.3.1(@types/node@22.19.5)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))':
|
||||
dependencies:
|
||||
'@vitest/spy': 4.0.16
|
||||
estree-walker: 3.0.3
|
||||
magic-string: 0.30.21
|
||||
optionalDependencies:
|
||||
msw: 2.12.7(@types/node@22.19.5)(typescript@5.9.3)
|
||||
vite: 7.3.1(@types/node@22.19.5)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)
|
||||
|
||||
'@vitest/mocker@4.0.16(msw@2.12.7(@types/node@25.0.3)(typescript@5.9.3))(vite@7.3.1(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))':
|
||||
dependencies:
|
||||
'@vitest/spy': 4.0.16
|
||||
|
|
@ -21168,6 +21204,32 @@ snapshots:
|
|||
|
||||
ms@2.1.3: {}
|
||||
|
||||
msw@2.12.7(@types/node@22.19.5)(typescript@5.9.3):
|
||||
dependencies:
|
||||
'@inquirer/confirm': 5.1.21(@types/node@22.19.5)
|
||||
'@mswjs/interceptors': 0.40.0
|
||||
'@open-draft/deferred-promise': 2.2.0
|
||||
'@types/statuses': 2.0.6
|
||||
cookie: 1.1.1
|
||||
graphql: 16.12.0
|
||||
headers-polyfill: 4.0.3
|
||||
is-node-process: 1.2.0
|
||||
outvariant: 1.4.3
|
||||
path-to-regexp: 6.3.0
|
||||
picocolors: 1.1.1
|
||||
rettime: 0.7.0
|
||||
statuses: 2.0.2
|
||||
strict-event-emitter: 0.5.1
|
||||
tough-cookie: 6.0.0
|
||||
type-fest: 5.3.1
|
||||
until-async: 3.0.2
|
||||
yargs: 17.7.2
|
||||
optionalDependencies:
|
||||
typescript: 5.9.3
|
||||
transitivePeerDependencies:
|
||||
- '@types/node'
|
||||
optional: true
|
||||
|
||||
msw@2.12.7(@types/node@25.0.3)(typescript@5.9.3):
|
||||
dependencies:
|
||||
'@inquirer/confirm': 5.1.21(@types/node@25.0.3)
|
||||
|
|
@ -23550,6 +23612,23 @@ snapshots:
|
|||
'@types/unist': 3.0.3
|
||||
vfile-message: 4.0.3
|
||||
|
||||
vite@7.3.1(@types/node@22.19.5)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2):
|
||||
dependencies:
|
||||
esbuild: 0.27.2
|
||||
fdir: 6.5.0(picomatch@4.0.3)
|
||||
picomatch: 4.0.3
|
||||
postcss: 8.5.6
|
||||
rollup: 4.55.1
|
||||
tinyglobby: 0.2.15
|
||||
optionalDependencies:
|
||||
'@types/node': 22.19.5
|
||||
fsevents: 2.3.3
|
||||
jiti: 2.6.1
|
||||
lightningcss: 1.30.2
|
||||
terser: 5.46.0
|
||||
tsx: 4.21.0
|
||||
yaml: 2.8.2
|
||||
|
||||
vite@7.3.1(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2):
|
||||
dependencies:
|
||||
esbuild: 0.27.2
|
||||
|
|
@ -23567,6 +23646,46 @@ snapshots:
|
|||
tsx: 4.21.0
|
||||
yaml: 2.8.2
|
||||
|
||||
vitest@4.0.16(@opentelemetry/api@1.9.0)(@types/node@22.19.5)(happy-dom@20.1.0)(jiti@2.6.1)(jsdom@27.4.0)(lightningcss@1.30.2)(msw@2.12.7(@types/node@22.19.5)(typescript@5.9.3))(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2):
|
||||
dependencies:
|
||||
'@vitest/expect': 4.0.16
|
||||
'@vitest/mocker': 4.0.16(msw@2.12.7(@types/node@22.19.5)(typescript@5.9.3))(vite@7.3.1(@types/node@22.19.5)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))
|
||||
'@vitest/pretty-format': 4.0.16
|
||||
'@vitest/runner': 4.0.16
|
||||
'@vitest/snapshot': 4.0.16
|
||||
'@vitest/spy': 4.0.16
|
||||
'@vitest/utils': 4.0.16
|
||||
es-module-lexer: 1.7.0
|
||||
expect-type: 1.3.0
|
||||
magic-string: 0.30.21
|
||||
obug: 2.1.1
|
||||
pathe: 2.0.3
|
||||
picomatch: 4.0.3
|
||||
std-env: 3.10.0
|
||||
tinybench: 2.9.0
|
||||
tinyexec: 1.0.2
|
||||
tinyglobby: 0.2.15
|
||||
tinyrainbow: 3.0.3
|
||||
vite: 7.3.1(@types/node@22.19.5)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)
|
||||
why-is-node-running: 2.3.0
|
||||
optionalDependencies:
|
||||
'@opentelemetry/api': 1.9.0
|
||||
'@types/node': 22.19.5
|
||||
happy-dom: 20.1.0
|
||||
jsdom: 27.4.0
|
||||
transitivePeerDependencies:
|
||||
- jiti
|
||||
- less
|
||||
- lightningcss
|
||||
- msw
|
||||
- sass
|
||||
- sass-embedded
|
||||
- stylus
|
||||
- sugarss
|
||||
- terser
|
||||
- tsx
|
||||
- yaml
|
||||
|
||||
vitest@4.0.16(@opentelemetry/api@1.9.0)(@types/node@25.0.3)(happy-dom@20.1.0)(jiti@2.6.1)(jsdom@27.4.0)(lightningcss@1.30.2)(msw@2.12.7(@types/node@25.0.3)(typescript@5.9.3))(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2):
|
||||
dependencies:
|
||||
'@vitest/expect': 4.0.16
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue