369 lines
14 KiB
TypeScript
369 lines
14 KiB
TypeScript
/**
|
|
* Judge wiring tests (Sprint 9 Task 2).
|
|
*
|
|
* Brief: PM-Waggle-OS/briefs/2026-04-20-cc-sprint-9-tasks.md Task 2 §Acceptance
|
|
* Scope: judge-client retry semantics + judge-runner payload assembly +
|
|
* ensemble aggregation. All mocked — zero real LLM calls.
|
|
*/
|
|
|
|
import fs from 'node:fs';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
|
import { createJudgeLlmClient, type JudgeClientCostEntry } from '../src/judge-client.js';
|
|
import { runJudge, type JudgeConfig } from '../src/judge-runner.js';
|
|
import { runOne } from '../src/runner.js';
|
|
import type { LlmClient } from '../src/judge-types.js';
|
|
import type { DatasetSpec, JsonlRecord, ModelSpec } from '../src/types.js';
|
|
|
|
// ── Fixtures ─────────────────────────────────────────────────────────────
|
|
|
|
const SYNTHETIC_DATASET: DatasetSpec = {
|
|
id: 'synthetic',
|
|
displayName: 'Synthetic',
|
|
dataPath: 'synthetic/placeholder.jsonl',
|
|
source: 'synthetic',
|
|
};
|
|
|
|
const QWEN_MODEL: ModelSpec = {
|
|
id: 'qwen3.6-35b-a3b',
|
|
displayName: 'Qwen3.6-35B-A3B',
|
|
provider: 'alibaba',
|
|
litellmModel: 'dashscope/qwen3.6-35b-a3b',
|
|
pricePerMillionInput: 0.2,
|
|
pricePerMillionOutput: 0.8,
|
|
contextWindow: 262144,
|
|
};
|
|
|
|
/** Scripted LlmClient — enqueues responses or errors and returns them
|
|
* in order. Used for judge module unit tests. */
|
|
class ScriptedLlmClient implements LlmClient {
|
|
readonly calls: string[] = [];
|
|
private queue: Array<string | Error>;
|
|
constructor(responses: Array<string | Error>) {
|
|
this.queue = [...responses];
|
|
}
|
|
async complete(prompt: string): Promise<string> {
|
|
this.calls.push(prompt);
|
|
if (this.queue.length === 0) throw new Error('ScriptedLlmClient out of responses');
|
|
const next = this.queue.shift()!;
|
|
if (next instanceof Error) throw next;
|
|
return next;
|
|
}
|
|
}
|
|
|
|
function readJsonl(file: string): JsonlRecord[] {
|
|
if (!fs.existsSync(file)) return [];
|
|
return fs.readFileSync(file, 'utf-8')
|
|
.split('\n')
|
|
.filter(l => l.trim().length > 0)
|
|
.map(l => JSON.parse(l) as JsonlRecord);
|
|
}
|
|
|
|
// ── Judge client — retry semantics ───────────────────────────────────────
|
|
|
|
describe('createJudgeLlmClient — transport retry semantics (brief §Failure-handling)', () => {
|
|
it('succeeds on first attempt without retry', async () => {
|
|
const fetchCalls: Array<{ url: string; body: unknown }> = [];
|
|
const fakeFetch: typeof fetch = async (url, init) => {
|
|
fetchCalls.push({ url: String(url), body: init?.body });
|
|
return new Response(
|
|
JSON.stringify({
|
|
choices: [{ message: { content: '{"verdict":"correct","failure_mode":null,"rationale":"ok"}' } }],
|
|
usage: { prompt_tokens: 100, completion_tokens: 20 },
|
|
}),
|
|
{ status: 200, headers: { 'Content-Type': 'application/json' } },
|
|
);
|
|
};
|
|
const costs: JudgeClientCostEntry[] = [];
|
|
const client = createJudgeLlmClient({
|
|
litellmUrl: 'http://test',
|
|
litellmApiKey: 'sk-test',
|
|
model: 'claude-sonnet-4-6',
|
|
fetchImpl: fakeFetch,
|
|
backoffMs: [1, 1],
|
|
onCall: e => costs.push(e),
|
|
});
|
|
const text = await client.complete('hello judge');
|
|
expect(text).toContain('verdict');
|
|
expect(fetchCalls).toHaveLength(1);
|
|
expect(costs).toHaveLength(1);
|
|
expect(costs[0].ok).toBe(true);
|
|
expect(costs[0].promptTokens).toBe(100);
|
|
expect(costs[0].completionTokens).toBe(20);
|
|
expect(costs[0].usd).toBeGreaterThan(0);
|
|
});
|
|
|
|
it('retries twice on HTTP 500 then succeeds on the third attempt', async () => {
|
|
let calls = 0;
|
|
const fakeFetch: typeof fetch = async () => {
|
|
calls++;
|
|
if (calls <= 2) {
|
|
return new Response('upstream is down', { status: 500 });
|
|
}
|
|
return new Response(
|
|
JSON.stringify({
|
|
choices: [{ message: { content: '{"verdict":"incorrect","failure_mode":"F3","rationale":"wrong date"}' } }],
|
|
usage: { prompt_tokens: 100, completion_tokens: 22 },
|
|
}),
|
|
{ status: 200, headers: { 'Content-Type': 'application/json' } },
|
|
);
|
|
};
|
|
const costs: JudgeClientCostEntry[] = [];
|
|
const client = createJudgeLlmClient({
|
|
litellmUrl: 'http://test',
|
|
litellmApiKey: 'sk-test',
|
|
model: 'claude-sonnet-4-6',
|
|
fetchImpl: fakeFetch,
|
|
backoffMs: [1, 1], // collapse backoff for tests
|
|
onCall: e => costs.push(e),
|
|
});
|
|
const text = await client.complete('hello');
|
|
expect(text).toContain('F3');
|
|
expect(calls).toBe(3);
|
|
// Single success entry — failures are absorbed by the retry loop
|
|
// and don't emit cost entries until the final outcome.
|
|
expect(costs).toHaveLength(1);
|
|
expect(costs[0].ok).toBe(true);
|
|
});
|
|
|
|
it('emits a final failed cost entry and throws when all retries are exhausted', async () => {
|
|
let calls = 0;
|
|
const fakeFetch: typeof fetch = async () => {
|
|
calls++;
|
|
return new Response('persistent 503', { status: 503 });
|
|
};
|
|
const costs: JudgeClientCostEntry[] = [];
|
|
const client = createJudgeLlmClient({
|
|
litellmUrl: 'http://test',
|
|
litellmApiKey: 'sk-test',
|
|
model: 'claude-sonnet-4-6',
|
|
fetchImpl: fakeFetch,
|
|
backoffMs: [1, 1],
|
|
onCall: e => costs.push(e),
|
|
});
|
|
await expect(client.complete('hello')).rejects.toThrow(/HTTP 503/);
|
|
// 1 initial + 2 retries = 3 attempts, all failing.
|
|
expect(calls).toBe(3);
|
|
expect(costs).toHaveLength(1);
|
|
expect(costs[0].ok).toBe(false);
|
|
});
|
|
|
|
it('falls back to reasoning_content when content is empty (thinking-mode provider)', async () => {
|
|
const fakeFetch: typeof fetch = async () =>
|
|
new Response(
|
|
JSON.stringify({
|
|
choices: [{
|
|
message: {
|
|
content: '',
|
|
reasoning_content: '{"verdict":"correct","failure_mode":null,"rationale":"parsed from reasoning"}',
|
|
},
|
|
}],
|
|
usage: { prompt_tokens: 80, completion_tokens: 200 },
|
|
}),
|
|
{ status: 200 },
|
|
);
|
|
const client = createJudgeLlmClient({
|
|
litellmUrl: 'http://test',
|
|
litellmApiKey: 'sk-test',
|
|
model: 'qwen3.6-35b-a3b-via-openrouter',
|
|
fetchImpl: fakeFetch,
|
|
backoffMs: [1, 1],
|
|
});
|
|
const text = await client.complete('hello');
|
|
expect(text).toContain('parsed from reasoning');
|
|
});
|
|
});
|
|
|
|
// ── runJudge — single-judge path ─────────────────────────────────────────
|
|
|
|
describe('runJudge — single judge produces a populated payload', () => {
|
|
it('maps judgeAnswer output onto the JudgePayload shape', async () => {
|
|
const client = new ScriptedLlmClient([
|
|
JSON.stringify({ verdict: 'correct', failure_mode: null, rationale: 'All facts match.' }),
|
|
]);
|
|
const payload = await runJudge(
|
|
{
|
|
question: 'Who painted the Mona Lisa?',
|
|
groundTruth: 'Leonardo da Vinci',
|
|
contextExcerpt: 'Leonardo da Vinci painted the Mona Lisa…',
|
|
modelAnswer: 'Leonardo da Vinci',
|
|
},
|
|
{ kind: 'single', model: 'claude-sonnet-4-6', client },
|
|
);
|
|
expect(payload.judge_verdict).toBe('correct');
|
|
expect(payload.judge_failure_mode).toBeNull();
|
|
expect(payload.judge_rationale).toBe('All facts match.');
|
|
expect(payload.judge_model).toBe('claude-sonnet-4-6');
|
|
expect(payload.judge_timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T/);
|
|
expect(payload.model_answer).toBe('Leonardo da Vinci');
|
|
expect(payload.judge_error).toBeUndefined();
|
|
});
|
|
|
|
it('survives a JudgeParseError without aborting the batch', async () => {
|
|
// Judge returns garbage twice → module throws JudgeParseError.
|
|
const client = new ScriptedLlmClient(['not json', 'still not json']);
|
|
const payload = await runJudge(
|
|
{
|
|
question: 'q',
|
|
groundTruth: 'gt',
|
|
contextExcerpt: 'ctx',
|
|
modelAnswer: 'ma',
|
|
},
|
|
{ kind: 'single', model: 'gpt-5', client },
|
|
);
|
|
expect(payload.judge_verdict).toBeUndefined();
|
|
expect(payload.judge_failure_mode).toBeUndefined();
|
|
expect(payload.judge_error).toMatch(/^parse:/);
|
|
// model_answer still propagated — runner will store the raw answer
|
|
// even when judging failed, so re-judging later is possible.
|
|
expect(payload.model_answer).toBe('ma');
|
|
});
|
|
});
|
|
|
|
// ── runJudge — ensemble path + majority + tie-break ──────────────────────
|
|
|
|
describe('runJudge — ensemble aggregates per-judge verdicts + majority', () => {
|
|
const models = ['claude-sonnet-4-6', 'claude-haiku-4-5', 'gpt-5'];
|
|
const mkClient = (verdict: 'correct' | 'incorrect', mode: 'F3' | null = null, rationale = 'r'): LlmClient =>
|
|
new ScriptedLlmClient([JSON.stringify({ verdict, failure_mode: mode, rationale })]);
|
|
|
|
it('3-0 unanimous majority populates ensemble entries and picks the shared verdict', async () => {
|
|
const clients = new Map<string, LlmClient>([
|
|
['claude-sonnet-4-6', mkClient('correct', null, 'sonnet')],
|
|
['claude-haiku-4-5', mkClient('correct', null, 'haiku')],
|
|
['gpt-5', mkClient('correct', null, 'gpt-5')],
|
|
]);
|
|
const payload = await runJudge(
|
|
{ question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma' },
|
|
{ kind: 'ensemble', models, clients },
|
|
);
|
|
expect(payload.judge_verdict).toBe('correct');
|
|
expect(payload.judge_ensemble).toHaveLength(3);
|
|
// Ensemble entries carry the per-judge model id + its individual
|
|
// verdict so the aggregator can compute inter-judge agreement
|
|
// without re-reading per-judge calls.
|
|
expect(payload.judge_ensemble?.map(e => e.model).sort()).toEqual([...models].sort());
|
|
});
|
|
|
|
it('2-1 majority takes the majority verdict; minority preserved in ensemble', async () => {
|
|
const clients = new Map<string, LlmClient>([
|
|
['claude-sonnet-4-6', mkClient('incorrect', 'F3', 'wrong date')],
|
|
['claude-haiku-4-5', mkClient('incorrect', 'F3', 'wrong date')],
|
|
['gpt-5', mkClient('correct', null, 'actually looks fine')],
|
|
]);
|
|
const payload = await runJudge(
|
|
{ question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma' },
|
|
{ kind: 'ensemble', models, clients },
|
|
);
|
|
expect(payload.judge_verdict).toBe('incorrect');
|
|
expect(payload.judge_failure_mode).toBe('F3');
|
|
// Minority verdict surfaced in the ensemble entries.
|
|
const gptEntry = payload.judge_ensemble?.find(e => e.model === 'gpt-5');
|
|
expect(gptEntry?.verdict).toBe('correct');
|
|
});
|
|
});
|
|
|
|
// ── Integration — runOne propagates judge fields into the JSONL ─────────
|
|
|
|
describe('runOne integration — judge fields land on every record when judgeConfig is set', () => {
|
|
let tmpDir: string;
|
|
beforeEach(() => {
|
|
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-judge-wire-'));
|
|
});
|
|
afterEach(() => {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it('single-judge mode — every row has judge_verdict + judge_failure_mode + judge_model', async () => {
|
|
const outputPath = path.join(tmpDir, 'judge.jsonl');
|
|
const queue = [
|
|
JSON.stringify({ verdict: 'correct', failure_mode: null, rationale: 'match' }),
|
|
JSON.stringify({ verdict: 'incorrect', failure_mode: 'F3', rationale: 'wrong date' }),
|
|
JSON.stringify({ verdict: 'correct', failure_mode: null, rationale: 'match' }),
|
|
];
|
|
const scripted = new ScriptedLlmClient(queue);
|
|
|
|
await runOne({
|
|
run: { kind: 'cell', name: 'raw' },
|
|
dataset: SYNTHETIC_DATASET,
|
|
model: QWEN_MODEL,
|
|
limit: 3,
|
|
seed: 42,
|
|
budgetUsd: Number.POSITIVE_INFINITY,
|
|
outputPath,
|
|
dryRun: true,
|
|
litellmUrl: 'http://unused',
|
|
litellmApiKey: 'unused',
|
|
emitPreregistrationEvent: false,
|
|
judgeConfig: { kind: 'single', model: 'claude-sonnet-4-6', client: scripted },
|
|
});
|
|
|
|
const records = readJsonl(outputPath);
|
|
expect(records).toHaveLength(3);
|
|
for (const r of records) {
|
|
expect(r.judge_model).toBe('claude-sonnet-4-6');
|
|
expect(r.judge_verdict).toBeDefined();
|
|
expect(r.judge_timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T/);
|
|
expect(r.model_answer).toBeDefined();
|
|
}
|
|
// Two correct + one incorrect, in the scripted order.
|
|
expect(records[0].judge_verdict).toBe('correct');
|
|
expect(records[1].judge_verdict).toBe('incorrect');
|
|
expect(records[1].judge_failure_mode).toBe('F3');
|
|
expect(records[2].judge_verdict).toBe('correct');
|
|
});
|
|
|
|
it('skips judging when judgeConfig is absent (existing behavior preserved)', async () => {
|
|
const outputPath = path.join(tmpDir, 'no-judge.jsonl');
|
|
await runOne({
|
|
run: { kind: 'cell', name: 'raw' },
|
|
dataset: SYNTHETIC_DATASET,
|
|
model: QWEN_MODEL,
|
|
limit: 2,
|
|
seed: 42,
|
|
budgetUsd: Number.POSITIVE_INFINITY,
|
|
outputPath,
|
|
dryRun: true,
|
|
litellmUrl: 'http://unused',
|
|
litellmApiKey: 'unused',
|
|
emitPreregistrationEvent: false,
|
|
});
|
|
const records = readJsonl(outputPath);
|
|
expect(records).toHaveLength(2);
|
|
for (const r of records) {
|
|
expect(r.judge_verdict).toBeUndefined();
|
|
expect(r.judge_model).toBeUndefined();
|
|
expect(r.model_answer).toBeUndefined();
|
|
}
|
|
});
|
|
|
|
it('unjudgeable rows still populate model_answer + leave verdict undefined (no crash)', async () => {
|
|
const outputPath = path.join(tmpDir, 'unjudged.jsonl');
|
|
// Both attempts produce unparseable output → JudgeParseError → row
|
|
// keeps model_answer, judge_verdict stays undefined.
|
|
const scripted = new ScriptedLlmClient(['garbage', 'still garbage', 'garbage', 'still garbage']);
|
|
await runOne({
|
|
run: { kind: 'cell', name: 'raw' },
|
|
dataset: SYNTHETIC_DATASET,
|
|
model: QWEN_MODEL,
|
|
limit: 2,
|
|
seed: 42,
|
|
budgetUsd: Number.POSITIVE_INFINITY,
|
|
outputPath,
|
|
dryRun: true,
|
|
litellmUrl: 'http://unused',
|
|
litellmApiKey: 'unused',
|
|
emitPreregistrationEvent: false,
|
|
judgeConfig: { kind: 'single', model: 'gpt-5', client: scripted },
|
|
});
|
|
const records = readJsonl(outputPath);
|
|
expect(records).toHaveLength(2);
|
|
for (const r of records) {
|
|
expect(r.judge_verdict).toBeUndefined();
|
|
expect(r.judge_failure_mode).toBeUndefined();
|
|
expect(r.model_answer).toBeDefined();
|
|
}
|
|
});
|
|
});
|