workflow_judge.ts

/**
 * LLM Judge for evaluating conversation quality
 * Uses structured output (JSON schema) for robust parsing
 */

// eslint-disable-next-line import/extensions
import type { ResponseFormatJSONSchema } from 'openai/resources/shared';

import type { WorkflowTestCase } from '../shared/types.js';
import { JUDGE_PROMPT_TEMPLATE, MODELS } from './config.js';
import type { LlmClient } from './llm_client.js';
import type { ConversationHistory } from './types.js';

/**
 * Judge evaluation result
 */
export type JudgeResult = {
    /** PASS or FAIL verdict */
    verdict: 'PASS' | 'FAIL';
    /** Explanation from judge */
    reason: string;
    /** Raw response from judge (for debugging) */
    rawResponse: string;
}

/**
 * JSON schema for structured judge output
 * Guarantees the LLM returns valid JSON matching this schema
 */
const JUDGE_RESPONSE_SCHEMA: ResponseFormatJSONSchema = {
    type: 'json_schema',
    json_schema: {
        name: 'judge_evaluation',
        strict: true,
        schema: {
            type: 'object',
            properties: {
                verdict: {
                    type: 'string',
                    enum: ['PASS', 'FAIL'],
                    description: 'Whether the agent passed or failed the evaluation',
                },
                reason: {
                    type: 'string',
                    description: 'Brief explanation in 1-2 sentences explaining why the agent passed or failed',
                },
            },
            required: ['verdict', 'reason'],
            additionalProperties: false,
        },
    },
};

/**
 * Format conversation for judge evaluation
 * Judge sees: tool calls + arguments + final responses (NOT tool results)
 */
function formatConversationForJudge(conversation: ConversationHistory): string {
    const lines: string[] = [];

    // User prompt
    lines.push(`USER: ${conversation.userPrompt}`);
    lines.push('');

    // Each turn
    for (const turn of conversation.turns) {
        // Show tool calls (if any)
        if (turn.toolCalls.length > 0) {
            for (const toolCall of turn.toolCalls) {
                lines.push(`AGENT: [Called tool: ${toolCall.name} with args: ${JSON.stringify(toolCall.arguments)}]`);
            }
        }

        // Show final response (if present)
        if (turn.finalResponse) {
            lines.push(`AGENT: ${turn.finalResponse}`);
        }

        lines.push('');
    }

    return lines.join('\n').trim();
}

/**
 * Parse structured JSON response from judge
 */
function parseJudgeResponse(response: string): { verdict: 'PASS' | 'FAIL'; reason: string } {
    try {
        const parsed = JSON.parse(response) as { verdict: 'PASS' | 'FAIL'; reason: string };

        // Validate the structure (should be guaranteed by schema, but double-check)
        if (!parsed.verdict || (parsed.verdict !== 'PASS' && parsed.verdict !== 'FAIL')) {
            throw new Error(`Invalid verdict: ${parsed.verdict}`);
        }

        if (!parsed.reason || typeof parsed.reason !== 'string') {
            throw new Error(`Invalid reason: ${parsed.reason}`);
        }

        return parsed;
    } catch (error) {
        throw new Error(
            `Failed to parse judge JSON response: ${error instanceof Error ? error.message : String(error)}\n`
            + `Raw response: ${response}`,
        );
    }
}

/**
 * Evaluate a conversation using the judge LLM
 */
export async function evaluateConversation(
    testCase: WorkflowTestCase,
    conversation: ConversationHistory,
    llmClient: LlmClient,
    judgeModel: string = MODELS.judge,
): Promise<JudgeResult> {
    // Format conversation for judge
    const formattedConversation = formatConversationForJudge(conversation);

    // Create judge prompt using reference field
    const judgePrompt = JUDGE_PROMPT_TEMPLATE
        .replace('{{reference}}', testCase.reference || '')
        .replace('{{conversation}}', formattedConversation);

    // Call judge LLM with structured output schema
    const response = await llmClient.callLlm(
        [{ role: 'user', content: judgePrompt }],
        judgeModel,
        undefined, // No tools
        JUDGE_RESPONSE_SCHEMA, // Use structured output
    );

    const rawResponse = response.content || '';

    // Parse response
    try {
        const { verdict, reason } = parseJudgeResponse(rawResponse);
        return {
            verdict,
            reason,
            rawResponse,
        };
    } catch (error) {
        throw new Error(
            `Failed to parse judge response: ${error instanceof Error ? error.message : String(error)}\n`
            + `Raw response: ${rawResponse}`,
        );
    }
}