engineering/tpmjs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2.1k

fix: switch scenario evaluator to gpt-4.1-mini

Browse source 
Changed default evaluator from claude-3-5-haiku-latest to gpt-4.1-mini
since OPENAI_API_KEY is configured in production but ANTHROPIC_API_KEY
is not.
This commit is contained in:
Ajax Davis 2026-01-19 05:20:05 +10:00
parent c52e6eaf24
commit a1f45fa33a
2 changed files with 7 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

9
apps/web/src/lib/scenarios/evaluate.ts
View file

@ -24,7 +24,8 @@ export type EvaluatorModelId =
| 'claude-3-5-sonnet-latest' | 'claude-3-5-sonnet-latest'
| 'claude-3-5-haiku-latest' | 'claude-3-5-haiku-latest'
| 'gpt-4o' | 'gpt-4o'
| 'gpt-4o-mini'; | 'gpt-4o-mini'
| 'gpt-4.1-mini';
/** /**
* Get the model instance for an evaluator model ID * Get the model instance for an evaluator model ID
@ -39,12 +40,14 @@ function getEvaluatorModel(modelId: EvaluatorModelId) {
return openai('gpt-4o'); return openai('gpt-4o');
case 'gpt-4o-mini': case 'gpt-4o-mini':
return openai('gpt-4o-mini'); return openai('gpt-4o-mini');
case 'gpt-4.1-mini':
return openai('gpt-4.1-mini');
default: default:
return anthropic('claude-3-5-haiku-latest'); return openai('gpt-4.1-mini');
} }
} }
const DEFAULT_EVALUATOR: EvaluatorModelId = 'claude-3-5-haiku-latest'; const DEFAULT_EVALUATOR: EvaluatorModelId = 'gpt-4.1-mini';
/** /**
* Evaluate if a scenario execution was successful * Evaluate if a scenario execution was successful

2
apps/web/src/lib/scenarios/execute.ts
View file

@ -14,7 +14,7 @@ import {
runAssertions, runAssertions,
} from './evaluate'; } from './evaluate';
const DEFAULT_EVALUATOR: EvaluatorModelId = 'claude-3-5-haiku-latest'; const DEFAULT_EVALUATOR: EvaluatorModelId = 'gpt-4.1-mini';
const MAX_RETRIES = 1; const MAX_RETRIES = 1;
interface ExecutionOptions { interface ExecutionOptions {