engineering/tpmjs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2.1k

feat(judge): add AI conversation quality evaluation tool

Browse source 
- Evaluates conversations across 10 metrics:
  taskCompletion, accuracy, relevance, clarity, efficiency,
  userIntentAlignment, actionability, progress, errorHandling, completeness
- Returns weighted overall score (0-10)
- Provides verdict (pass/retry/fail) with reasons
- Detects conversation loops and stuck states
- Lists must-dos, suggestions, and next steps
- Designed for frequent use in agentic loops
This commit is contained in:
Ajax Davis 2026-01-17 10:49:40 +10:00
parent aad8b58031
commit 1234b383e7
7 changed files with 883 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

65
packages/tools/official/blocks.yml
View file

@ -7539,6 +7539,71 @@ blocks:
extra_prompt: "Include a playful, cozy message about taking a little rest. Examples: '😴 zzz... had a lovely {duration} second nap!', '💤 *yawn* refreshed after snoozing for {duration}s', '🛏️ ahh, that was a nice little rest...'"
measures: [working_implementation, valid_output_structure, proper_error_handling, ai_sdk_compliance]
# ---------------------------------------------------------------------------
# Agent Quality Tools
# ---------------------------------------------------------------------------
agent.judgeConversation:
type: utility
description: "Evaluate an AI conversation across 10 quality metrics to ensure the AI is making real progress and completing user intent. Use this tool FREQUENTLY in agentic loops to verify progress, catch stuck states, and get actionable improvements."
path: "judge"
domain_rules:
- id: metric_evaluation
description: "Must evaluate all 10 metrics: taskCompletion, accuracy, relevance, clarity, efficiency, userIntentAlignment, actionability, progress, errorHandling, completeness"
- id: loop_detection
description: "Must detect conversation loops, repetition patterns, and oscillation"
- id: tool_analysis
description: "Must analyze tool call patterns and success/failure rates"
- id: verdict_generation
description: "Must return a verdict (pass/retry/fail) with actionable must-dos"
inputs:
- name: messages
type: array
description: "Array of AI SDK messages with role, content, toolCalls, and toolResults"
- name: originalUserRequest
type: string
optional: true
description: "The original user request if different from first message"
- name: context
type: string
optional: true
description: "Additional context about what the conversation should accomplish"
- name: strictMode
type: boolean
optional: true
description: "If true, requires higher scores to pass (default: false)"
outputs:
- name: overallScore
type: number
description: "Weighted average score across all metrics (0-10)"
- name: verdict
type: string
description: "pass, retry, or fail"
- name: verdictReason
type: string
description: "Explanation for the verdict"
- name: metrics
type: array
description: "Individual metric evaluations with scores, reasons, mustDos, and suggestions"
- name: criticalIssues
type: array
description: "List of blocking issues that must be addressed"
- name: mustDos
type: array
description: "Critical actions that MUST be taken"
- name: suggestions
type: array
description: "Helpful improvements"
- name: nextSteps
type: array
description: "Recommended next actions"
- name: loopDetected
type: boolean
description: "Whether a conversation loop was detected"
- name: conversationSummary
type: string
description: "Brief summary of the conversation"
measures: [working_implementation, valid_output_structure, proper_error_handling, ai_sdk_compliance]
# =============================================================================
# VALIDATORS - Which validators to run against each block
# =============================================================================

15
packages/tools/official/judge/block.ts Normal file
View file

@ -0,0 +1,15 @@
/**
* Block metadata for judge tool
* This file provides metadata for the blocks validator
*/
import { judgeConversation } from './src/index.js';
export const block = {
name: 'judge',
description: 'AI conversation quality judge. Evaluates AI SDK messages across 10 metrics with scores, reasoning, suggestions, and actionable improvements.',
tools: {
judgeConversation,
},
};
export default block;

6
packages/tools/official/judge/index.ts Normal file
View file

@ -0,0 +1,6 @@
/**
* AI Conversation Judge Tool for TPMJS
* Re-export all tools from src/index.ts
*/
export * from './src/index.js';
export { default } from './src/index.js';

61
packages/tools/official/judge/package.json Normal file
View file

@ -0,0 +1,61 @@
{
"name": "@tpmjs/tools-judge",
"version": "0.1.0",
"description": "AI conversation quality judge. Evaluates AI SDK messages across 10 metrics with scores, reasoning, suggestions, and actionable improvements.",
"type": "module",
"keywords": [
"tpmjs",
"judge",
"ai",
"evaluation",
"quality",
"metrics",
"agent",
"conversation"
],
"exports": {
".": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
}
},
"files": [
"dist"
],
"scripts": {
"build": "tsup",
"dev": "tsup --watch",
"type-check": "tsc --noEmit",
"clean": "rm -rf dist .turbo"
},
"devDependencies": {
"@tpmjs/tsconfig": "workspace:*",
"tsup": "^8.5.1",
"typescript": "^5.9.3"
},
"publishConfig": {
"access": "public"
},
"repository": {
"type": "git",
"url": "https://github.com/tpmjs/tpmjs.git",
"directory": "packages/tools/official/judge"
},
"homepage": "https://tpmjs.com",
"license": "MIT",
"tpmjs": {
"category": "agent",
"frameworks": [
"vercel-ai"
],
"tools": [
{
"name": "judgeConversation",
"description": "Evaluate an AI conversation across 10 quality metrics. Use this tool frequently in agentic loops to verify the AI is making progress, staying on track, and actually completing what the user intended. Returns scores, reasoning, must-dos, and improvement suggestions for each metric."
}
]
},
"dependencies": {
"ai": "6.0.23"
}
}

715
packages/tools/official/judge/src/index.ts Normal file
View file

@ -0,0 +1,715 @@
/**
* AI Conversation Judge Tool for TPMJS
*
* Evaluates AI SDK conversations across 10 quality metrics to ensure
* agentic loops are making real progress and completing user intent.
*
* Use this tool frequently during agentic workflows to:
* - Verify the AI is on track
* - Catch loops or stuck states early
* - Ensure task completion before moving on
* - Get actionable improvement suggestions
*/
import { jsonSchema, tool } from 'ai';
/**
* The 10 metrics used to evaluate AI conversations
*/
export const EVALUATION_METRICS = [
'taskCompletion',
'accuracy',
'relevance',
'clarity',
'efficiency',
'userIntentAlignment',
'actionability',
'progress',
'errorHandling',
'completeness',
] as const;
export type EvaluationMetric = (typeof EVALUATION_METRICS)[number];
/**
* Metric descriptions for context
*/
export const METRIC_DESCRIPTIONS: Record<EvaluationMetric, string> = {
taskCompletion: 'Did the AI make meaningful progress toward completing the user\'s request?',
accuracy: 'Are the responses factually correct and free of hallucinations or errors?',
relevance: 'Are responses directly addressing what the user asked for?',
clarity: 'Are responses clear, well-structured, and easy to understand?',
efficiency: 'Is the AI being concise without unnecessary verbosity or token waste?',
userIntentAlignment: 'Does the AI correctly understand what the user actually wants?',
actionability: 'Are the outputs actionable and immediately usable by the user?',
progress: 'Is the conversation moving forward or stuck in loops/repetition?',
errorHandling: 'Does the AI handle errors gracefully and recover appropriately?',
completeness: 'Has the AI fully addressed all aspects of the request?',
};
/**
* Metric weights for overall score calculation
*/
export const METRIC_WEIGHTS: Record<EvaluationMetric, number> = {
taskCompletion: 2.0, // Most important - did we do what was asked?
userIntentAlignment: 1.5, // Critical - understanding the user
completeness: 1.5, // Important - full coverage
accuracy: 1.5, // Important - correctness
progress: 1.2, // Catches loops
actionability: 1.0, // Usability
relevance: 1.0, // On-topic
errorHandling: 1.0, // Resilience
clarity: 0.8, // Nice to have
efficiency: 0.5, // Least critical
};
/**
* AI SDK message structure
*/
interface Message {
role: 'user' | 'assistant' | 'system' | 'tool';
content: string;
toolCalls?: Array<{
toolName: string;
args: Record<string, unknown>;
}>;
toolResults?: Array<{
toolName: string;
result: unknown;
}>;
}
/**
* Individual metric evaluation result
*/
interface MetricEvaluation {
metric: EvaluationMetric;
score: number;
reason: string;
mustDos: string[];
suggestions: string[];
observations: string[];
}
/**
* Verdict on whether to continue, retry, or fail
*/
type Verdict = 'pass' | 'retry' | 'fail';
/**
* Complete evaluation result
*/
interface JudgeResult {
overallScore: number;
verdict: Verdict;
verdictReason: string;
metrics: MetricEvaluation[];
criticalIssues: string[];
mustDos: string[];
suggestions: string[];
nextSteps: string[];
loopDetected: boolean;
conversationSummary: string;
}
/**
* Input for the judge tool
*/
interface JudgeInput {
messages: Message[];
originalUserRequest?: string;
context?: string;
strictMode?: boolean;
}
/**
* Analyze messages for loop detection
*/
function detectLoops(messages: Message[]): { detected: boolean; details: string[] } {
const details: string[] = [];
const assistantMessages = messages.filter(m => m.role === 'assistant');
if (assistantMessages.length < 2) {
return { detected: false, details: [] };
}
// Check for repeated content
const contentSet = new Set<string>();
let duplicateCount = 0;
for (const msg of assistantMessages) {
const normalized = msg.content.toLowerCase().trim().slice(0, 200);
if (contentSet.has(normalized)) {
duplicateCount++;
}
contentSet.add(normalized);
}
if (duplicateCount > 1) {
details.push(`Detected ${duplicateCount} duplicate or near-duplicate responses`);
}
// Check for repeated tool calls
const toolCallPatterns = new Map<string, number>();
for (const msg of assistantMessages) {
if (msg.toolCalls) {
for (const call of msg.toolCalls) {
const pattern = `${call.toolName}:${JSON.stringify(call.args).slice(0, 100)}`;
toolCallPatterns.set(pattern, (toolCallPatterns.get(pattern) || 0) + 1);
}
}
}
for (const [pattern, count] of toolCallPatterns) {
if (count > 2) {
details.push(`Tool call pattern repeated ${count} times: ${pattern.slice(0, 50)}...`);
}
}
// Check for oscillation patterns
if (assistantMessages.length >= 4) {
const last4 = assistantMessages.slice(-4).map(m => m.content.slice(0, 100));
if (last4[0] === last4[2] && last4[1] === last4[3]) {
details.push('Detected oscillation pattern in last 4 responses');
}
}
return {
detected: details.length > 0,
details,
};
}
/**
* Extract the original user request from messages
*/
function extractUserRequest(messages: Message[], provided?: string): string {
if (provided) return provided;
const userMessages = messages.filter(m => m.role === 'user');
if (userMessages.length === 0) return 'Unknown request';
return userMessages[0]!.content;
}
/**
* Count tool usage statistics
*/
function analyzeToolUsage(messages: Message[]): {
totalCalls: number;
successfulCalls: number;
failedCalls: number;
toolsUsed: string[];
} {
let totalCalls = 0;
let successfulCalls = 0;
let failedCalls = 0;
const toolsUsed = new Set<string>();
for (const msg of messages) {
if (msg.toolCalls) {
for (const call of msg.toolCalls) {
totalCalls++;
toolsUsed.add(call.toolName);
}
}
if (msg.toolResults) {
for (const result of msg.toolResults) {
const resultStr = JSON.stringify(result.result).toLowerCase();
if (resultStr.includes('error') || resultStr.includes('failed') || resultStr.includes('exception')) {
failedCalls++;
} else {
successfulCalls++;
}
}
}
}
return {
totalCalls,
successfulCalls,
failedCalls,
toolsUsed: Array.from(toolsUsed),
};
}
/**
* Evaluate a single metric
*/
function evaluateMetric(
metric: EvaluationMetric,
messages: Message[],
userRequest: string,
loopInfo: { detected: boolean; details: string[] },
toolStats: ReturnType<typeof analyzeToolUsage>
): MetricEvaluation {
const mustDos: string[] = [];
const suggestions: string[] = [];
const observations: string[] = [];
let score = 5;
let reason = '';
const lastAssistant = [...messages].reverse().find(m => m.role === 'assistant');
const assistantCount = messages.filter(m => m.role === 'assistant').length;
const userCount = messages.filter(m => m.role === 'user').length;
switch (metric) {
case 'taskCompletion': {
// Check if there's evidence of task completion
const lastContent = lastAssistant?.content.toLowerCase() || '';
const completionIndicators = ['done', 'completed', 'finished', 'created', 'updated', 'fixed', 'resolved', 'here is', 'here\'s'];
const hasCompletion = completionIndicators.some(ind => lastContent.includes(ind));
if (hasCompletion && toolStats.successfulCalls > 0) {
score = 8;
reason = 'Evidence of task completion with successful tool usage';
suggestions.push('Verify the output matches user expectations');
} else if (hasCompletion) {
score = 6;
reason = 'Claims of completion but limited tool usage to verify';
mustDos.push('Verify claims with actual tool calls or evidence');
} else if (toolStats.totalCalls > 0) {
score = 4;
reason = 'Tools were used but no clear completion signal';
mustDos.push('Explicitly confirm task completion to the user');
} else {
score = 2;
reason = 'No evidence of task completion';
mustDos.push('Actually complete the task using appropriate tools');
}
break;
}
case 'accuracy': {
if (toolStats.failedCalls > toolStats.successfulCalls) {
score = 3;
reason = 'More failed tool calls than successful ones indicates potential accuracy issues';
mustDos.push('Investigate and fix the failing tool calls');
observations.push(`${toolStats.failedCalls} failed vs ${toolStats.successfulCalls} successful calls`);
} else if (toolStats.successfulCalls > 0) {
score = 7;
reason = 'Successful tool calls suggest accurate execution';
suggestions.push('Consider validating outputs against expected results');
} else {
score = 5;
reason = 'No tool calls to verify accuracy';
observations.push('Unable to assess accuracy without tool results');
}
break;
}
case 'relevance': {
// Simple heuristic: check if user request terms appear in responses
const requestTerms = userRequest.toLowerCase().split(/\s+/).filter(t => t.length > 3);
const responseText = messages.filter(m => m.role === 'assistant').map(m => m.content.toLowerCase()).join(' ');
const matchedTerms = requestTerms.filter(t => responseText.includes(t));
const relevanceRatio = requestTerms.length > 0 ? matchedTerms.length / requestTerms.length : 0.5;
score = Math.round(relevanceRatio * 10);
if (score >= 7) {
reason = 'Responses are highly relevant to the user request';
} else if (score >= 4) {
reason = 'Responses are partially relevant but may have drifted off-topic';
suggestions.push('Refocus on the core user request');
} else {
reason = 'Responses appear to have drifted from the original request';
mustDos.push('Re-read the original request and address it directly');
}
observations.push(`${matchedTerms.length}/${requestTerms.length} key terms addressed`);
break;
}
case 'clarity': {
const totalWords = messages.filter(m => m.role === 'assistant')
.reduce((sum, m) => sum + m.content.split(/\s+/).length, 0);
const avgWordsPerResponse = assistantCount > 0 ? totalWords / assistantCount : 0;
if (avgWordsPerResponse > 50 && avgWordsPerResponse < 500) {
score = 8;
reason = 'Responses are well-balanced in length for clarity';
} else if (avgWordsPerResponse <= 50) {
score = 6;
reason = 'Responses may be too brief for complex explanations';
suggestions.push('Provide more detailed explanations where helpful');
} else {
score = 5;
reason = 'Responses may be too verbose';
suggestions.push('Be more concise while maintaining clarity');
}
observations.push(`Average ${Math.round(avgWordsPerResponse)} words per response`);
break;
}
case 'efficiency': {
const messageRatio = assistantCount / Math.max(userCount, 1);
if (messageRatio <= 3 && toolStats.totalCalls > 0) {
score = 8;
reason = 'Efficient conversation with purposeful tool usage';
} else if (messageRatio > 5) {
score = 4;
reason = 'High number of assistant messages may indicate inefficiency';
suggestions.push('Try to accomplish more per response');
} else {
score = 6;
reason = 'Reasonable efficiency';
}
observations.push(`${assistantCount} assistant messages for ${userCount} user messages`);
break;
}
case 'userIntentAlignment': {
// Check if the conversation seems to understand the request
const hasQuestions = messages.some(m => m.role === 'assistant' && m.content.includes('?'));
const hasConfirmation = messages.some(m => m.role === 'assistant' &&
(m.content.toLowerCase().includes('i understand') ||
m.content.toLowerCase().includes('let me') ||
m.content.toLowerCase().includes('i\'ll')));
if (hasConfirmation && !hasQuestions) {
score = 8;
reason = 'AI appears to understand and is executing on user intent';
} else if (hasQuestions && hasConfirmation) {
score = 7;
reason = 'AI is clarifying while also executing';
observations.push('Asking clarifying questions shows engagement');
} else if (hasQuestions) {
score = 5;
reason = 'AI is asking questions but may not be executing';
suggestions.push('Balance clarification with action');
} else {
score = 4;
reason = 'Unclear if AI fully understands user intent';
mustDos.push('Confirm understanding of the user request');
}
break;
}
case 'actionability': {
const lastContent = lastAssistant?.content || '';
const hasCode = lastContent.includes('```') || lastContent.includes('function') || lastContent.includes('const ');
const hasSteps = /\d+\.\s/.test(lastContent) || lastContent.includes('- ');
const hasFiles = toolStats.toolsUsed.some(t => t.toLowerCase().includes('write') || t.toLowerCase().includes('create'));
if (hasFiles) {
score = 9;
reason = 'Concrete files/artifacts were created';
observations.push('Created tangible outputs');
} else if (hasCode && hasSteps) {
score = 7;
reason = 'Provided code and clear steps';
} else if (hasCode || hasSteps) {
score = 6;
reason = 'Some actionable content provided';
suggestions.push('Add more concrete next steps or examples');
} else {
score = 4;
reason = 'Response lacks actionable content';
mustDos.push('Provide concrete actions, code, or steps the user can take');
}
break;
}
case 'progress': {
if (loopInfo.detected) {
score = 2;
reason = 'Loop or repetition detected - conversation is stuck';
mustDos.push('Break out of the current approach and try something different');
observations.push(...loopInfo.details);
} else if (assistantCount > 10) {
score = 5;
reason = 'Long conversation may indicate slow progress';
suggestions.push('Consider a more direct approach');
} else if (toolStats.successfulCalls > 0) {
score = 8;
reason = 'Making forward progress with successful tool usage';
} else {
score = 6;
reason = 'Conversation appears to be progressing';
}
break;
}
case 'errorHandling': {
if (toolStats.failedCalls === 0) {
score = 8;
reason = 'No errors encountered or all handled gracefully';
} else if (toolStats.failedCalls > 0 && toolStats.successfulCalls > toolStats.failedCalls) {
score = 6;
reason = 'Some errors occurred but conversation continued';
suggestions.push('Review failed operations for potential fixes');
} else if (toolStats.failedCalls > toolStats.successfulCalls) {
score = 3;
reason = 'More failures than successes indicates poor error handling';
mustDos.push('Investigate root cause of failures before retrying');
} else {
score = 5;
reason = 'Error handling status unclear';
}
observations.push(`${toolStats.failedCalls} failed, ${toolStats.successfulCalls} successful calls`);
break;
}
case 'completeness': {
// Check for common incompleteness indicators
const lastContent = lastAssistant?.content.toLowerCase() || '';
const incompleteIndicators = ['todo', 'later', 'next time', 'will add', 'placeholder', 'not yet', 'tbd', '...'];
const hasIncomplete = incompleteIndicators.some(ind => lastContent.includes(ind));
const completeIndicators = ['all done', 'complete', 'finished', 'everything', 'all set'];
const hasComplete = completeIndicators.some(ind => lastContent.includes(ind));
if (hasComplete && !hasIncomplete) {
score = 9;
reason = 'Explicit completion signals without incompleteness markers';
} else if (hasIncomplete) {
score = 4;
reason = 'Found incompleteness indicators in response';
mustDos.push('Complete all TODO items and placeholders before finishing');
observations.push('Response contains placeholder or incomplete markers');
} else if (toolStats.successfulCalls > 2) {
score = 7;
reason = 'Multiple successful tool calls suggest thorough work';
} else {
score = 5;
reason = 'Completeness is unclear';
suggestions.push('Explicitly list what has been completed');
}
break;
}
}
return {
metric,
score: Math.max(0, Math.min(10, score)),
reason,
mustDos,
suggestions,
observations,
};
}
/**
* Calculate overall score with weights
*/
function calculateOverallScore(metrics: MetricEvaluation[]): number {
let totalWeight = 0;
let weightedSum = 0;
for (const evaluation of metrics) {
const weight = METRIC_WEIGHTS[evaluation.metric];
weightedSum += evaluation.score * weight;
totalWeight += weight;
}
return Math.round((weightedSum / totalWeight) * 10) / 10;
}
/**
* Determine verdict based on metrics
*/
function determineVerdict(
metrics: MetricEvaluation[],
overallScore: number,
loopDetected: boolean,
strictMode: boolean
): { verdict: Verdict; reason: string } {
const threshold = strictMode ? 7 : 5;
// Critical failures that always result in retry/fail
const taskCompletion = metrics.find(m => m.metric === 'taskCompletion')?.score || 0;
const progress = metrics.find(m => m.metric === 'progress')?.score || 0;
if (loopDetected) {
return { verdict: 'retry', reason: 'Loop detected - must try a different approach' };
}
if (taskCompletion <= 2) {
return { verdict: 'fail', reason: 'No meaningful task completion detected' };
}
if (progress <= 2) {
return { verdict: 'retry', reason: 'Conversation is stuck and not making progress' };
}
if (overallScore >= 8) {
return { verdict: 'pass', reason: 'High quality execution across all metrics' };
}
if (overallScore >= threshold) {
return { verdict: 'pass', reason: 'Acceptable quality with room for improvement' };
}
if (overallScore >= 4) {
return { verdict: 'retry', reason: 'Below threshold - retry with improvements' };
}
return { verdict: 'fail', reason: 'Quality too low across multiple metrics' };
}
/**
* Generate conversation summary
*/
function generateSummary(messages: Message[], toolStats: ReturnType<typeof analyzeToolUsage>): string {
const userCount = messages.filter(m => m.role === 'user').length;
const assistantCount = messages.filter(m => m.role === 'assistant').length;
return `Conversation with ${userCount} user message(s), ${assistantCount} assistant response(s), and ${toolStats.totalCalls} tool call(s) (${toolStats.successfulCalls} successful, ${toolStats.failedCalls} failed). Tools used: ${toolStats.toolsUsed.join(', ') || 'none'}.`;
}
/**
* The main judge tool
*/
export const judgeConversation = tool({
description: `Evaluate an AI conversation across 10 quality metrics to ensure the AI is making real progress and completing user intent.
USE THIS TOOL FREQUENTLY in agentic loops to:
- Verify the AI is on track before moving to the next step
- Catch loops, stuck states, or regressions early
- Ensure task completion before declaring success
- Get actionable must-dos and improvement suggestions
The 10 metrics evaluated:
1. Task Completion - Did the AI complete what was asked?
2. Accuracy - Are responses correct and error-free?
3. Relevance - Are responses on-topic?
4. Clarity - Are responses clear and understandable?
5. Efficiency - Is the AI being concise?
6. User Intent Alignment - Does the AI understand the user?
7. Actionability - Are outputs usable?
8. Progress - Is the conversation moving forward?
9. Error Handling - Are errors handled gracefully?
10. Completeness - Are all aspects addressed?
Returns a verdict (pass/retry/fail) with specific must-dos for any issues.`,
inputSchema: jsonSchema<JudgeInput>({
type: 'object',
properties: {
messages: {
type: 'array',
description: 'Array of AI SDK messages to evaluate. Each message should have role and content.',
items: {
type: 'object',
properties: {
role: {
type: 'string',
enum: ['user', 'assistant', 'system', 'tool'],
description: 'Message role',
},
content: {
type: 'string',
description: 'Message content',
},
toolCalls: {
type: 'array',
description: 'Optional tool calls made by assistant',
items: {
type: 'object',
properties: {
toolName: { type: 'string' },
args: { type: 'object' },
},
},
},
toolResults: {
type: 'array',
description: 'Optional tool results',
items: {
type: 'object',
properties: {
toolName: { type: 'string' },
result: {},
},
},
},
},
required: ['role', 'content'],
},
},
originalUserRequest: {
type: 'string',
description: 'Optional: The original user request if different from first message',
},
context: {
type: 'string',
description: 'Optional: Additional context about what the conversation should accomplish',
},
strictMode: {
type: 'boolean',
description: 'Optional: If true, requires higher scores to pass (default: false)',
},
},
required: ['messages'],
additionalProperties: false,
}),
async execute(input: JudgeInput): Promise<JudgeResult> {
const { messages, originalUserRequest, context: _context, strictMode = false } = input;
if (!messages || messages.length === 0) {
return {
overallScore: 0,
verdict: 'fail',
verdictReason: 'No messages provided to evaluate',
metrics: [],
criticalIssues: ['No messages to evaluate'],
mustDos: ['Provide messages to evaluate'],
suggestions: [],
nextSteps: ['Call this tool with actual conversation messages'],
loopDetected: false,
conversationSummary: 'Empty conversation',
};
}
const userRequest = extractUserRequest(messages, originalUserRequest);
const loopInfo = detectLoops(messages);
const toolStats = analyzeToolUsage(messages);
// Evaluate all metrics
const metrics = EVALUATION_METRICS.map(metric =>
evaluateMetric(metric, messages, userRequest, loopInfo, toolStats)
);
const overallScore = calculateOverallScore(metrics);
const { verdict, reason: verdictReason } = determineVerdict(metrics, overallScore, loopInfo.detected, strictMode);
// Aggregate must-dos and suggestions
const allMustDos = metrics.flatMap(m => m.mustDos);
const allSuggestions = metrics.flatMap(m => m.suggestions);
// Identify critical issues (metrics with score <= 3)
const criticalIssues = metrics
.filter(m => m.score <= 3)
.map(m => `${m.metric}: ${m.reason}`);
// Generate next steps based on verdict
const nextSteps: string[] = [];
if (verdict === 'pass') {
nextSteps.push('Proceed to the next task or confirm completion with the user');
} else if (verdict === 'retry') {
nextSteps.push('Address the must-dos listed above');
nextSteps.push('Try a different approach if stuck');
if (loopInfo.detected) {
nextSteps.push('Break out of the current pattern - do something different');
}
} else {
nextSteps.push('Re-read the original user request');
nextSteps.push('Start fresh with a clear plan');
nextSteps.push('Focus on actually completing the task, not just responding');
}
return {
overallScore,
verdict,
verdictReason,
metrics,
criticalIssues,
mustDos: [...new Set(allMustDos)], // Deduplicate
suggestions: [...new Set(allSuggestions)],
nextSteps,
loopDetected: loopInfo.detected,
conversationSummary: generateSummary(messages, toolStats),
};
},
});
export default judgeConversation;

11
packages/tools/official/judge/tsconfig.json Normal file
View file

@ -0,0 +1,11 @@
{
"extends": "@tpmjs/tsconfig/base.json",
"compilerOptions": {
"outDir": "dist",
"rootDir": "src",
"incremental": false,
"composite": false
},
"include": ["src"],
"exclude": ["node_modules", "dist"]
}

10
packages/tools/official/judge/tsup.config.ts Normal file
View file

@ -0,0 +1,10 @@
import { defineConfig } from 'tsup';
export default defineConfig({
entry: ['src/index.ts'],
format: ['esm'],
dts: true,
clean: true,
target: 'es2022',
outDir: 'dist',
});