test: add comprehensive tests for CLI and scenarios
Add unit tests for: - CLI TpmClient (api-client.test.ts) - 26 tests covering authentication, tool execution, agent/collection/scenario management, error handling - Scenario evaluation (evaluate.test.ts) - 16 tests for regex assertions and verdict determination - Cosine similarity (similarity.test.ts) - 16 tests for embedding comparison Add integration tests for scenarios: - CRUD operations (scenarios-crud.integration.test.ts) - Run execution and quota management (scenarios-run.integration.test.ts) Configure vitest for CLI package with @tpmjs/test shared config.
This commit is contained in:
Ajax Davis
parent
9c731d582e
commit
a44f38eda9
8 changed files with 1491 additions and 1 deletions
222
apps/web/src/lib/scenarios/evaluate.test.ts
Normal file
222
apps/web/src/lib/scenarios/evaluate.test.ts
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
/**
|
||||
* Scenario Evaluation Unit Tests
|
||||
*
|
||||
* Tests for the evaluation logic including:
|
||||
* - Regex assertion matching
|
||||
* - Final verdict determination
|
||||
* - Assertion result handling
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { determineFinalVerdict, runAssertions } from './evaluate';
|
||||
|
||||
describe('runAssertions', () => {
|
||||
describe('regex assertions', () => {
|
||||
it('should pass when regex matches output', () => {
|
||||
const output = 'Hello, World!';
|
||||
const assertions = { regex: ['hello', 'world'] };
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toContain('regex:hello');
|
||||
expect(result.passed).toContain('regex:world');
|
||||
expect(result.failed).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should fail when regex does not match output', () => {
|
||||
const output = 'Goodbye, World!';
|
||||
const assertions = { regex: ['hello'] };
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.failed).toContain('regex:hello');
|
||||
expect(result.passed).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should handle multiple assertions with mixed results', () => {
|
||||
const output = 'The result is 42 and status is success';
|
||||
const assertions = { regex: ['result.*42', 'success', 'error', 'failure'] };
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toContain('regex:result.*42');
|
||||
expect(result.passed).toContain('regex:success');
|
||||
expect(result.failed).toContain('regex:error');
|
||||
expect(result.failed).toContain('regex:failure');
|
||||
});
|
||||
|
||||
it('should handle case-insensitive matching', () => {
|
||||
const output = 'SUCCESS';
|
||||
const assertions = { regex: ['success'] };
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toContain('regex:success');
|
||||
});
|
||||
|
||||
it('should handle invalid regex patterns gracefully', () => {
|
||||
const output = 'test';
|
||||
const assertions = { regex: ['[invalid', '(unclosed'] };
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.failed).toContain('regex:[invalid (invalid pattern)');
|
||||
expect(result.failed).toContain('regex:(unclosed (invalid pattern)');
|
||||
});
|
||||
|
||||
it('should handle empty regex array', () => {
|
||||
const output = 'test';
|
||||
const assertions = { regex: [] };
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toHaveLength(0);
|
||||
expect(result.failed).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should handle complex regex patterns', () => {
|
||||
const output = 'User john@example.com has ID 12345';
|
||||
const assertions = {
|
||||
regex: [
|
||||
'[a-z]+@[a-z]+\\.[a-z]+', // Email pattern
|
||||
'ID \\d+', // ID pattern
|
||||
'^User', // Starts with User
|
||||
],
|
||||
};
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toHaveLength(3);
|
||||
expect(result.failed).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('schema assertions', () => {
|
||||
it('should note when schema is provided', () => {
|
||||
const output = '{"name": "test"}';
|
||||
const assertions = { schema: { type: 'object' } };
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toContain('schema:provided (validation pending)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('combined assertions', () => {
|
||||
it('should handle both regex and schema assertions', () => {
|
||||
const output = '{"status": "success"}';
|
||||
const assertions = {
|
||||
regex: ['success'],
|
||||
schema: { type: 'object' },
|
||||
};
|
||||
|
||||
const result = runAssertions(output, assertions);
|
||||
|
||||
expect(result.passed).toContain('regex:success');
|
||||
expect(result.passed).toContain('schema:provided (validation pending)');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('determineFinalVerdict', () => {
|
||||
describe('LLM evaluation only', () => {
|
||||
it('should return pass when LLM says pass', () => {
|
||||
const evaluation = {
|
||||
verdict: 'pass' as const,
|
||||
reason: 'Task completed successfully',
|
||||
confidence: 0.95,
|
||||
};
|
||||
|
||||
const result = determineFinalVerdict(evaluation);
|
||||
|
||||
expect(result).toBe('pass');
|
||||
});
|
||||
|
||||
it('should return fail when LLM says fail', () => {
|
||||
const evaluation = {
|
||||
verdict: 'fail' as const,
|
||||
reason: 'Task was not completed',
|
||||
confidence: 0.9,
|
||||
};
|
||||
|
||||
const result = determineFinalVerdict(evaluation);
|
||||
|
||||
expect(result).toBe('fail');
|
||||
});
|
||||
});
|
||||
|
||||
describe('with assertions', () => {
|
||||
it('should return pass when LLM passes and all assertions pass', () => {
|
||||
const evaluation = {
|
||||
verdict: 'pass' as const,
|
||||
reason: 'Task completed',
|
||||
confidence: 0.9,
|
||||
};
|
||||
const assertions = {
|
||||
passed: ['regex:success', 'regex:result'],
|
||||
failed: [],
|
||||
};
|
||||
|
||||
const result = determineFinalVerdict(evaluation, assertions);
|
||||
|
||||
expect(result).toBe('pass');
|
||||
});
|
||||
|
||||
it('should return fail when LLM passes but assertions fail', () => {
|
||||
const evaluation = {
|
||||
verdict: 'pass' as const,
|
||||
reason: 'Task completed',
|
||||
confidence: 0.9,
|
||||
};
|
||||
const assertions = {
|
||||
passed: ['regex:success'],
|
||||
failed: ['regex:expected_output'],
|
||||
};
|
||||
|
||||
const result = determineFinalVerdict(evaluation, assertions);
|
||||
|
||||
expect(result).toBe('fail');
|
||||
});
|
||||
|
||||
it('should return fail when LLM fails regardless of assertions', () => {
|
||||
const evaluation = {
|
||||
verdict: 'fail' as const,
|
||||
reason: 'Task failed',
|
||||
confidence: 0.95,
|
||||
};
|
||||
const assertions = {
|
||||
passed: ['regex:success'],
|
||||
failed: [],
|
||||
};
|
||||
|
||||
const result = determineFinalVerdict(evaluation, assertions);
|
||||
|
||||
expect(result).toBe('fail');
|
||||
});
|
||||
|
||||
it('should handle null assertions', () => {
|
||||
const evaluation = {
|
||||
verdict: 'pass' as const,
|
||||
reason: 'Task completed',
|
||||
confidence: 0.9,
|
||||
};
|
||||
|
||||
const result = determineFinalVerdict(evaluation, null);
|
||||
|
||||
expect(result).toBe('pass');
|
||||
});
|
||||
|
||||
it('should handle undefined assertions', () => {
|
||||
const evaluation = {
|
||||
verdict: 'pass' as const,
|
||||
reason: 'Task completed',
|
||||
confidence: 0.9,
|
||||
};
|
||||
|
||||
const result = determineFinalVerdict(evaluation, undefined);
|
||||
|
||||
expect(result).toBe('pass');
|
||||
});
|
||||
});
|
||||
});
|
||||
134
apps/web/src/lib/scenarios/similarity.test.ts
Normal file
134
apps/web/src/lib/scenarios/similarity.test.ts
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
/**
|
||||
* Scenario Similarity Unit Tests
|
||||
*
|
||||
* Tests for the similarity functions including:
|
||||
* - Cosine similarity calculation
|
||||
* - Edge cases and error handling
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { cosineSimilarity } from './similarity';
|
||||
|
||||
describe('cosineSimilarity', () => {
|
||||
it('should return 1 for identical vectors', () => {
|
||||
const vector = [1, 2, 3, 4, 5];
|
||||
const result = cosineSimilarity(vector, vector);
|
||||
expect(result).toBeCloseTo(1, 10);
|
||||
});
|
||||
|
||||
it('should return 0 for orthogonal vectors', () => {
|
||||
const a = [1, 0];
|
||||
const b = [0, 1];
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBeCloseTo(0, 10);
|
||||
});
|
||||
|
||||
it('should return -1 for opposite vectors', () => {
|
||||
const a = [1, 2, 3];
|
||||
const b = [-1, -2, -3];
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBeCloseTo(-1, 10);
|
||||
});
|
||||
|
||||
it('should handle normalized vectors correctly', () => {
|
||||
// Two unit vectors at 60 degrees have cosine similarity of 0.5
|
||||
const a = [1, 0];
|
||||
const b = [0.5, Math.sqrt(3) / 2]; // 60 degree rotation
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBeCloseTo(0.5, 5);
|
||||
});
|
||||
|
||||
it('should handle non-normalized vectors', () => {
|
||||
// Scaling a vector should not change the cosine similarity
|
||||
const a = [1, 2, 3];
|
||||
const b = [2, 4, 6]; // Same direction, scaled by 2
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBeCloseTo(1, 10);
|
||||
});
|
||||
|
||||
it('should handle vectors with different magnitudes', () => {
|
||||
const a = [3, 4]; // magnitude 5
|
||||
const b = [6, 8]; // magnitude 10, same direction
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBeCloseTo(1, 10);
|
||||
});
|
||||
|
||||
it('should throw error for vectors of different lengths', () => {
|
||||
const a = [1, 2, 3];
|
||||
const b = [1, 2];
|
||||
expect(() => cosineSimilarity(a, b)).toThrow('Vectors must have same length');
|
||||
});
|
||||
|
||||
it('should return 0 for zero vectors', () => {
|
||||
const a = [0, 0, 0];
|
||||
const b = [1, 2, 3];
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBe(0);
|
||||
});
|
||||
|
||||
it('should return 0 for two zero vectors', () => {
|
||||
const a = [0, 0, 0];
|
||||
const b = [0, 0, 0];
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBe(0);
|
||||
});
|
||||
|
||||
it('should handle high-dimensional vectors', () => {
|
||||
// Simulate embedding vectors (typically 1536 dimensions for text-embedding-3-small)
|
||||
const dim = 1536;
|
||||
const a = Array.from({ length: dim }, (_, i) => Math.sin(i));
|
||||
const b = Array.from({ length: dim }, (_, i) => Math.sin(i));
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBeCloseTo(1, 5);
|
||||
});
|
||||
|
||||
it('should handle high-dimensional dissimilar vectors', () => {
|
||||
const dim = 1536;
|
||||
const a = Array.from({ length: dim }, (_, i) => Math.sin(i));
|
||||
const b = Array.from({ length: dim }, (_, i) => Math.cos(i));
|
||||
const result = cosineSimilarity(a, b);
|
||||
// Should be between -1 and 1, but not 1
|
||||
expect(result).toBeLessThan(1);
|
||||
expect(result).toBeGreaterThanOrEqual(-1);
|
||||
});
|
||||
|
||||
it('should produce consistent results for partial similarity', () => {
|
||||
const a = [1, 0, 1, 0];
|
||||
const b = [1, 1, 0, 0];
|
||||
const result = cosineSimilarity(a, b);
|
||||
// Expected: (1*1 + 0*1 + 1*0 + 0*0) / (sqrt(2) * sqrt(2)) = 1/2 = 0.5
|
||||
expect(result).toBeCloseTo(0.5, 10);
|
||||
});
|
||||
|
||||
it('should handle negative values correctly', () => {
|
||||
const a = [-1, 2, -3];
|
||||
const b = [1, -2, 3];
|
||||
const result = cosineSimilarity(a, b);
|
||||
// Opposite directions
|
||||
expect(result).toBeCloseTo(-1, 10);
|
||||
});
|
||||
|
||||
it('should handle mixed positive and negative values', () => {
|
||||
const a = [1, -1, 1];
|
||||
const b = [1, 1, -1];
|
||||
const result = cosineSimilarity(a, b);
|
||||
// (1 - 1 - 1) / (sqrt(3) * sqrt(3)) = -1/3
|
||||
expect(result).toBeCloseTo(-1 / 3, 10);
|
||||
});
|
||||
|
||||
it('should handle very small values without underflow', () => {
|
||||
const a = [1e-10, 2e-10, 3e-10];
|
||||
const b = [1e-10, 2e-10, 3e-10];
|
||||
const result = cosineSimilarity(a, b);
|
||||
expect(result).toBeCloseTo(1, 5);
|
||||
});
|
||||
|
||||
it('should handle empty arrays', () => {
|
||||
const a: number[] = [];
|
||||
const b: number[] = [];
|
||||
const result = cosineSimilarity(a, b);
|
||||
// 0/0 case, returns 0 per implementation
|
||||
expect(result).toBe(0);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,233 @@
|
|||
/**
|
||||
* Scenarios CRUD Integration Tests
|
||||
*
|
||||
* Tests the scenario API endpoints including:
|
||||
* - Creating scenarios
|
||||
* - Listing scenarios
|
||||
* - Getting scenario details
|
||||
* - Updating scenarios
|
||||
* - Deleting scenarios
|
||||
*
|
||||
* NOTE: These are integration tests that require a running server.
|
||||
* Run `pnpm dev --filter=@tpmjs/web` first, then run tests.
|
||||
* Tests will be skipped if server is not available.
|
||||
*
|
||||
* To run integration tests manually:
|
||||
* INTEGRATION_TESTS=true pnpm --filter=@tpmjs/web test
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
|
||||
// Skip integration tests unless explicitly enabled
|
||||
const INTEGRATION_TESTS_ENABLED = process.env.INTEGRATION_TESTS === 'true';
|
||||
const BASE_URL = process.env.TEST_BASE_URL || 'http://localhost:3000';
|
||||
|
||||
// Check if server is available
|
||||
async function isServerAvailable(): Promise<boolean> {
|
||||
if (!INTEGRATION_TESTS_ENABLED) return false;
|
||||
try {
|
||||
const response = await fetch(`${BASE_URL}/api/health`, { signal: AbortSignal.timeout(2000) });
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
describe.skipIf(!INTEGRATION_TESTS_ENABLED)('Scenarios CRUD Integration', () => {
|
||||
let serverAvailable = false;
|
||||
let testCollection: { id: string; slug: string; userId: string } | null = null;
|
||||
const testScenarioId: string | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
serverAvailable = await isServerAvailable();
|
||||
if (!serverAvailable) {
|
||||
console.warn('⚠️ Server not available - skipping scenario integration tests');
|
||||
return;
|
||||
}
|
||||
|
||||
// Find a test collection to use
|
||||
testCollection = await prisma.collection.findFirst({
|
||||
where: { isPublic: true },
|
||||
select: { id: true, slug: true, userId: true },
|
||||
});
|
||||
|
||||
if (!testCollection) {
|
||||
console.warn('⚠️ No public collection found - skipping scenario tests');
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
try {
|
||||
// Clean up test scenario if created
|
||||
if (testScenarioId) {
|
||||
await prisma.scenario.delete({ where: { id: testScenarioId } }).catch(() => {});
|
||||
}
|
||||
await prisma.$disconnect();
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
});
|
||||
|
||||
describe('GET /api/scenarios', () => {
|
||||
it('should list public scenarios', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios?limit=10`);
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const result = await response.json();
|
||||
expect(result.success).toBe(true);
|
||||
expect(Array.isArray(result.data)).toBe(true);
|
||||
expect(result.pagination).toBeDefined();
|
||||
});
|
||||
|
||||
it('should support pagination', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios?limit=5&offset=0`);
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const result = await response.json();
|
||||
expect(result.pagination.limit).toBe(5);
|
||||
expect(result.pagination.offset).toBe(0);
|
||||
});
|
||||
|
||||
it('should filter by collection', async () => {
|
||||
if (!serverAvailable || !testCollection) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios?collectionId=${testCollection.id}`);
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const result = await response.json();
|
||||
expect(result.success).toBe(true);
|
||||
|
||||
// All returned scenarios should belong to the collection
|
||||
for (const scenario of result.data) {
|
||||
expect(scenario.collectionId).toBe(testCollection.id);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /api/scenarios/featured', () => {
|
||||
it('should return featured scenarios', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios/featured`);
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const result = await response.json();
|
||||
expect(result.success).toBe(true);
|
||||
expect(Array.isArray(result.data)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /api/collections/[id]/scenarios/generate', () => {
|
||||
it('should require authentication', async () => {
|
||||
if (!serverAvailable || !testCollection) return;
|
||||
|
||||
const response = await fetch(
|
||||
`${BASE_URL}/api/collections/${testCollection.id}/scenarios/generate`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ count: 1 }),
|
||||
}
|
||||
);
|
||||
|
||||
// Should return 401 without auth
|
||||
expect(response.status).toBe(401);
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /api/scenarios/[id]', () => {
|
||||
it('should return scenario details', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
// First find a scenario
|
||||
const listResponse = await fetch(`${BASE_URL}/api/scenarios?limit=1`);
|
||||
const listResult = await listResponse.json();
|
||||
|
||||
if (listResult.data.length === 0) {
|
||||
console.log('No scenarios found to test');
|
||||
return;
|
||||
}
|
||||
|
||||
const scenarioId = listResult.data[0].id;
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios/${scenarioId}`);
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const result = await response.json();
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data.id).toBe(scenarioId);
|
||||
expect(result.data.name).toBeDefined();
|
||||
expect(result.data.prompt).toBeDefined();
|
||||
});
|
||||
|
||||
it('should return 404 for non-existent scenario', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios/non-existent-scenario-id-12345`);
|
||||
|
||||
expect(response.status).toBe(404);
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /api/scenarios/[id]/runs', () => {
|
||||
it('should list scenario runs', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
// Find a scenario with runs
|
||||
const listResponse = await fetch(`${BASE_URL}/api/scenarios?limit=10`);
|
||||
const listResult = await listResponse.json();
|
||||
|
||||
if (listResult.data.length === 0) {
|
||||
console.log('No scenarios found');
|
||||
return;
|
||||
}
|
||||
|
||||
// Try each scenario until we find one (may or may not have runs)
|
||||
const scenarioId = listResult.data[0].id;
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios/${scenarioId}/runs`);
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const result = await response.json();
|
||||
expect(result.success).toBe(true);
|
||||
expect(Array.isArray(result.data)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /api/scenarios/check-similarity', () => {
|
||||
it('should check scenario similarity', async () => {
|
||||
if (!serverAvailable || !testCollection) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios/check-similarity`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
collectionId: testCollection.id,
|
||||
prompt: 'Test prompt for similarity checking',
|
||||
}),
|
||||
});
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const result = await response.json();
|
||||
expect(result.success).toBe(true);
|
||||
expect(typeof result.data.hasSimilar).toBe('boolean');
|
||||
});
|
||||
|
||||
it('should require collectionId and prompt', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios/check-similarity`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({}),
|
||||
});
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,193 @@
|
|||
/**
|
||||
* Scenario Execution Integration Tests
|
||||
*
|
||||
* Tests the scenario run API endpoint including:
|
||||
* - Running scenarios
|
||||
* - Quota management
|
||||
* - Result evaluation
|
||||
*
|
||||
* NOTE: These tests require authentication and may consume API quotas.
|
||||
* Run with caution in development environments only.
|
||||
*/
|
||||
|
||||
import { prisma } from '@tpmjs/db';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
|
||||
// Skip integration tests unless explicitly enabled
|
||||
const INTEGRATION_TESTS_ENABLED = process.env.INTEGRATION_TESTS === 'true';
|
||||
const BASE_URL = process.env.TEST_BASE_URL || 'http://localhost:3000';
|
||||
|
||||
// Check if server is available
|
||||
async function isServerAvailable(): Promise<boolean> {
|
||||
if (!INTEGRATION_TESTS_ENABLED) return false;
|
||||
try {
|
||||
const response = await fetch(`${BASE_URL}/api/health`, { signal: AbortSignal.timeout(2000) });
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
describe.skipIf(!INTEGRATION_TESTS_ENABLED)('Scenario Execution Integration', () => {
|
||||
let serverAvailable = false;
|
||||
let testScenario: { id: string; name: string; collectionId: string } | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
serverAvailable = await isServerAvailable();
|
||||
if (!serverAvailable) {
|
||||
console.warn('⚠️ Server not available - skipping scenario execution tests');
|
||||
return;
|
||||
}
|
||||
|
||||
// Find a test scenario
|
||||
testScenario = await prisma.scenario.findFirst({
|
||||
where: {
|
||||
collection: { isPublic: true },
|
||||
},
|
||||
select: { id: true, name: true, collectionId: true },
|
||||
});
|
||||
|
||||
if (!testScenario) {
|
||||
console.warn('⚠️ No public scenario found - some tests will be skipped');
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (!serverAvailable) return;
|
||||
await prisma.$disconnect();
|
||||
});
|
||||
|
||||
describe('POST /api/scenarios/[id]/run', () => {
|
||||
it('should require authentication', async () => {
|
||||
if (!serverAvailable || !testScenario) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios/${testScenario.id}/run`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
|
||||
// Should return 401 without auth
|
||||
expect(response.status).toBe(401);
|
||||
});
|
||||
|
||||
it('should return 404 for non-existent scenario', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const response = await fetch(`${BASE_URL}/api/scenarios/non-existent-id-12345/run`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
|
||||
// Should return 401 (auth required first) or 404
|
||||
expect([401, 404]).toContain(response.status);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Scenario metrics', () => {
|
||||
it('should have quality score between 0 and 1', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const scenarios = await prisma.scenario.findMany({
|
||||
take: 10,
|
||||
select: { id: true, qualityScore: true },
|
||||
});
|
||||
|
||||
for (const scenario of scenarios) {
|
||||
expect(scenario.qualityScore).toBeGreaterThanOrEqual(0);
|
||||
expect(scenario.qualityScore).toBeLessThanOrEqual(1);
|
||||
}
|
||||
});
|
||||
|
||||
it('should track consecutive passes and fails', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const scenarios = await prisma.scenario.findMany({
|
||||
take: 10,
|
||||
select: {
|
||||
id: true,
|
||||
consecutivePasses: true,
|
||||
consecutiveFails: true,
|
||||
totalRuns: true,
|
||||
},
|
||||
});
|
||||
|
||||
for (const scenario of scenarios) {
|
||||
expect(scenario.consecutivePasses).toBeGreaterThanOrEqual(0);
|
||||
expect(scenario.consecutiveFails).toBeGreaterThanOrEqual(0);
|
||||
expect(scenario.totalRuns).toBeGreaterThanOrEqual(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Scenario runs history', () => {
|
||||
it('should have valid run records', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const runs = await prisma.scenarioRun.findMany({
|
||||
take: 10,
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
scenarioId: true,
|
||||
userId: true,
|
||||
createdAt: true,
|
||||
},
|
||||
});
|
||||
|
||||
for (const run of runs) {
|
||||
expect(['pending', 'running', 'pass', 'fail', 'error']).toContain(run.status);
|
||||
expect(run.scenarioId).toBeDefined();
|
||||
expect(run.userId).toBeDefined();
|
||||
expect(run.createdAt).toBeInstanceOf(Date);
|
||||
}
|
||||
});
|
||||
|
||||
it('should track token usage for completed runs', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const completedRuns = await prisma.scenarioRun.findMany({
|
||||
where: { status: { in: ['pass', 'fail'] } },
|
||||
take: 10,
|
||||
select: {
|
||||
id: true,
|
||||
inputTokens: true,
|
||||
outputTokens: true,
|
||||
totalTokens: true,
|
||||
executionTimeMs: true,
|
||||
},
|
||||
});
|
||||
|
||||
for (const run of completedRuns) {
|
||||
// Completed runs should have token counts
|
||||
if (run.inputTokens !== null) {
|
||||
expect(run.inputTokens).toBeGreaterThanOrEqual(0);
|
||||
}
|
||||
if (run.outputTokens !== null) {
|
||||
expect(run.outputTokens).toBeGreaterThanOrEqual(0);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Quota management', () => {
|
||||
it('should track user quotas', async () => {
|
||||
if (!serverAvailable) return;
|
||||
|
||||
const quotas = await prisma.scenarioQuota.findMany({
|
||||
take: 5,
|
||||
select: {
|
||||
userId: true,
|
||||
dailyLimit: true,
|
||||
dailyUsed: true,
|
||||
lastResetAt: true,
|
||||
},
|
||||
});
|
||||
|
||||
for (const quota of quotas) {
|
||||
expect(quota.dailyLimit).toBeGreaterThan(0);
|
||||
expect(quota.dailyUsed).toBeGreaterThanOrEqual(0);
|
||||
expect(quota.dailyUsed).toBeLessThanOrEqual(quota.dailyLimit);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue