Self-learning is excluded from the community edition (ADR-0004); the project owner reworks it separately. Cost tracking stays. - agent: delete behaviour evolution (evolve-schema, iterative GEPA, compose-evolution, evolution gates, orchestrator and deploy), improvement and correction detection, AgentLearning, the eval-dataset builder, optimization-capture and the agent_insights tool. Secret detection moves from eval-dataset to secret-patterns. The LLM wiring keeps only the Haiku adapter and judge call used by skill audit and hygiene, so @ax-llm/ax is now declared by agent instead of server. - server: delete /api/evolution/*, the evolution and optimizer services, GEPA expansion of the first chat message, the prompt_optimization and monthly_assessment jobs, the idle-session reviewer and its session-reviewer persona, prompt injection of past corrections and thumbs-down recorded as correction signals. Existing job rows of the removed types are ignored, not deleted. - core: delete OptimizationLogStore; stop re-exporting the improvement signal and evolution run stores. - web: delete the Evolution screen and the two job types in the automation editor. - delete packages/optimizer. Also logs UF-027 (self-learning loops read data nothing collects) and UF-028 (the Settings prompt-shape choice is ignored by chat).
342 lines
10 KiB
TypeScript
342 lines
10 KiB
TypeScript
/**
|
|
* LLM-as-Judge — rubric-based scorer used by skill audit, skill hygiene and
|
|
* loop jobs.
|
|
*
|
|
* Scores a candidate response against an expected ground truth along
|
|
* three weighted dimensions:
|
|
*
|
|
* correctness 0.5 — does the answer match the expected output?
|
|
* procedureFollowing 0.3 — did the agent follow the instructions / format?
|
|
* conciseness 0.2 — is it on-point without padding?
|
|
*
|
|
* Then applies a length penalty (soft cap) multiplied into the final score.
|
|
*
|
|
* The textual feedback must explain *what* went wrong (or right) so the
|
|
* caller can act on it.
|
|
*
|
|
* Model-agnostic — caller provides the llmCall function. Defaults assume
|
|
* a Haiku-class judge via LiteLLM, but any OpenAI-compatible backend works.
|
|
*/
|
|
|
|
export type JudgeLLMCall = (prompt: string) => Promise<string>;
|
|
|
|
export interface JudgeInput {
|
|
/** The original user instruction. */
|
|
input: string;
|
|
/** Ground-truth / reference output. */
|
|
expected: string;
|
|
/** The candidate to be scored. */
|
|
actual: string;
|
|
/** Optional additional context (e.g., task_shape, persona). */
|
|
context?: string;
|
|
}
|
|
|
|
export interface JudgeScore {
|
|
/** Weighted overall score 0..1 after length penalty. */
|
|
overall: number;
|
|
/** Raw weighted score before length penalty. */
|
|
weighted: number;
|
|
/** Component scores 0..1. */
|
|
correctness: number;
|
|
procedureFollowing: number;
|
|
conciseness: number;
|
|
/** Multiplier applied to the weighted score (0..1). */
|
|
lengthPenalty: number;
|
|
/** Free-form feedback explaining the score. */
|
|
feedback: string;
|
|
/** True when the judge successfully parsed a structured response. */
|
|
parsed: boolean;
|
|
}
|
|
|
|
export interface JudgeOptions {
|
|
weights?: {
|
|
correctness?: number;
|
|
procedure?: number;
|
|
conciseness?: number;
|
|
};
|
|
/**
|
|
* Target output length in characters. If `actual` is within
|
|
* `[0, lengthTarget * tolerance]` the penalty is 1.0, falling off
|
|
* linearly beyond. Defaults to a generous 2,000 chars.
|
|
*/
|
|
lengthTarget?: number;
|
|
/** Allowed overshoot fraction before penalty kicks in. Default 0.5 (50%). */
|
|
lengthTolerance?: number;
|
|
/**
|
|
* Floor for the length penalty — scores never fall below this multiplier
|
|
* due to length alone. Default 0.5.
|
|
*/
|
|
lengthFloor?: number;
|
|
/** Override the default rubric prompt entirely. */
|
|
rubricOverride?: string;
|
|
}
|
|
|
|
export const DEFAULT_WEIGHTS = { correctness: 0.5, procedure: 0.3, conciseness: 0.2 };
|
|
|
|
export const DEFAULT_RUBRIC = `You are a strict, fair evaluator scoring an AI assistant's response.
|
|
|
|
You will be given:
|
|
- The user's INSTRUCTION
|
|
- The EXPECTED output (ground truth)
|
|
- The ACTUAL output from the AI assistant
|
|
|
|
Score the ACTUAL response on three dimensions, each on a 0-10 integer scale:
|
|
|
|
1. CORRECTNESS (0-10)
|
|
- 10: semantically equivalent to expected, all facts / steps correct
|
|
- 7-9: mostly correct, minor missing / wrong detail
|
|
- 4-6: partially correct, material gap or error
|
|
- 1-3: mostly wrong
|
|
- 0: unrelated or empty
|
|
|
|
2. PROCEDURE_FOLLOWING (0-10)
|
|
- Did the response follow the requested format / structure / constraints
|
|
implied by the instruction?
|
|
- 10: format matches exactly
|
|
- 5: format roughly matches
|
|
- 0: ignored the procedural requirements
|
|
|
|
3. CONCISENESS (0-10)
|
|
- 10: tight, no padding, appropriate length
|
|
- 5: somewhat verbose or somewhat terse
|
|
- 0: extreme padding or trivially short
|
|
|
|
Then write a short, ACTIONABLE FEEDBACK (max 3 sentences) explaining
|
|
the biggest improvement opportunity. Focus on what to change in the
|
|
prompt / approach, not what the user should do differently.
|
|
|
|
Return ONLY a JSON object on a single line, no markdown:
|
|
{"correctness": <0-10>, "procedure": <0-10>, "conciseness": <0-10>, "feedback": "<string>"}`;
|
|
|
|
// ── Public API ─────────────────────────────────────────────────
|
|
|
|
export class LLMJudge {
|
|
private llmCall: JudgeLLMCall;
|
|
private weights: { correctness: number; procedure: number; conciseness: number };
|
|
private lengthTarget: number;
|
|
private lengthTolerance: number;
|
|
private lengthFloor: number;
|
|
private rubric: string;
|
|
|
|
constructor(llmCall: JudgeLLMCall, options: JudgeOptions = {}) {
|
|
this.llmCall = llmCall;
|
|
this.weights = {
|
|
correctness: options.weights?.correctness ?? DEFAULT_WEIGHTS.correctness,
|
|
procedure: options.weights?.procedure ?? DEFAULT_WEIGHTS.procedure,
|
|
conciseness: options.weights?.conciseness ?? DEFAULT_WEIGHTS.conciseness,
|
|
};
|
|
validateWeights(this.weights);
|
|
this.lengthTarget = options.lengthTarget ?? 2_000;
|
|
this.lengthTolerance = options.lengthTolerance ?? 0.5;
|
|
this.lengthFloor = options.lengthFloor ?? 0.5;
|
|
this.rubric = options.rubricOverride ?? DEFAULT_RUBRIC;
|
|
}
|
|
|
|
/** Score a single example. */
|
|
async score(input: JudgeInput): Promise<JudgeScore> {
|
|
const prompt = buildPrompt(this.rubric, input);
|
|
let rawResponse = '';
|
|
try {
|
|
rawResponse = await this.llmCall(prompt);
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
return errorScore(`Judge LLM call failed: ${message}`);
|
|
}
|
|
|
|
const parsed = parseJudgeResponse(rawResponse);
|
|
if (!parsed) {
|
|
return errorScore(
|
|
`Judge response could not be parsed. Raw: ${rawResponse.slice(0, 200)}`,
|
|
);
|
|
}
|
|
|
|
const correctness = clampUnit(parsed.correctness / 10);
|
|
const procedureFollowing = clampUnit(parsed.procedure / 10);
|
|
const conciseness = clampUnit(parsed.conciseness / 10);
|
|
|
|
const weighted =
|
|
correctness * this.weights.correctness +
|
|
procedureFollowing * this.weights.procedure +
|
|
conciseness * this.weights.conciseness;
|
|
|
|
const lengthPenalty = computeLengthPenalty(
|
|
input.actual.length,
|
|
this.lengthTarget,
|
|
this.lengthTolerance,
|
|
this.lengthFloor,
|
|
);
|
|
|
|
return {
|
|
overall: clampUnit(weighted * lengthPenalty),
|
|
weighted,
|
|
correctness,
|
|
procedureFollowing,
|
|
conciseness,
|
|
lengthPenalty,
|
|
feedback: parsed.feedback || 'No feedback provided.',
|
|
parsed: true,
|
|
};
|
|
}
|
|
|
|
/** Score many examples. Returns in input order. */
|
|
async scoreBatch(inputs: JudgeInput[]): Promise<JudgeScore[]> {
|
|
const scores: JudgeScore[] = [];
|
|
for (const input of inputs) {
|
|
scores.push(await this.score(input));
|
|
}
|
|
return scores;
|
|
}
|
|
}
|
|
|
|
// ── Helpers (exported for tests) ───────────────────────────────
|
|
|
|
export function buildPrompt(rubric: string, input: JudgeInput): string {
|
|
const ctx = input.context ? `\n\nCONTEXT: ${input.context}` : '';
|
|
return `${rubric}${ctx}
|
|
|
|
INSTRUCTION:
|
|
${input.input}
|
|
|
|
EXPECTED:
|
|
${input.expected}
|
|
|
|
ACTUAL:
|
|
${input.actual}
|
|
|
|
Return the JSON now.`;
|
|
}
|
|
|
|
export interface ParsedJudgeResponse {
|
|
correctness: number;
|
|
procedure: number;
|
|
conciseness: number;
|
|
feedback: string;
|
|
}
|
|
|
|
/**
|
|
* Extract the judge JSON from the LLM response. Tolerates markdown fences
|
|
* and extra prose before / after the JSON.
|
|
*/
|
|
export function parseJudgeResponse(raw: string): ParsedJudgeResponse | null {
|
|
if (!raw) return null;
|
|
|
|
// Strip markdown code fences if present.
|
|
const cleaned = raw
|
|
.replace(/```json\s*/gi, '')
|
|
.replace(/```\s*/g, '')
|
|
.trim();
|
|
|
|
// Find the first balanced { ... } block and try to parse.
|
|
const candidates = extractJsonCandidates(cleaned);
|
|
for (const candidate of candidates) {
|
|
try {
|
|
const obj = JSON.parse(candidate) as Record<string, unknown>;
|
|
if (
|
|
typeof obj.correctness === 'number' &&
|
|
typeof obj.procedure === 'number' &&
|
|
typeof obj.conciseness === 'number'
|
|
) {
|
|
const feedback = typeof obj.feedback === 'string' ? obj.feedback : '';
|
|
return {
|
|
correctness: obj.correctness,
|
|
procedure: obj.procedure,
|
|
conciseness: obj.conciseness,
|
|
feedback,
|
|
};
|
|
}
|
|
} catch {
|
|
// try next candidate
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Enumerate substrings that might be a JSON object, starting at each `{`.
|
|
* Returns them in order (longest-first preference via outer `{` starts).
|
|
*/
|
|
export function extractJsonCandidates(text: string): string[] {
|
|
const out: string[] = [];
|
|
const starts: number[] = [];
|
|
for (let i = 0; i < text.length; i++) {
|
|
if (text[i] === '{') starts.push(i);
|
|
}
|
|
for (const start of starts) {
|
|
let depth = 0;
|
|
let inString = false;
|
|
let escape = false;
|
|
for (let i = start; i < text.length; i++) {
|
|
const ch = text[i];
|
|
if (inString) {
|
|
if (escape) escape = false;
|
|
else if (ch === '\\') escape = true;
|
|
else if (ch === '"') inString = false;
|
|
} else if (ch === '"') {
|
|
inString = true;
|
|
} else if (ch === '{') {
|
|
depth++;
|
|
} else if (ch === '}') {
|
|
depth--;
|
|
if (depth === 0) {
|
|
out.push(text.slice(start, i + 1));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Prefer the largest candidate first — most likely to be the whole object.
|
|
return out.sort((a, b) => b.length - a.length);
|
|
}
|
|
|
|
/**
|
|
* Length penalty: 1.0 while actual.length ≤ target * (1 + tolerance),
|
|
* falling linearly toward `floor` as length grows to 3x the target.
|
|
*/
|
|
export function computeLengthPenalty(
|
|
actualLength: number,
|
|
target: number,
|
|
tolerance: number,
|
|
floor: number,
|
|
): number {
|
|
if (actualLength <= 0 || target <= 0) return 1;
|
|
const limit = target * (1 + tolerance);
|
|
if (actualLength <= limit) return 1;
|
|
|
|
// Linear falloff from `limit` to `3 * target` where we hit `floor`.
|
|
const farLimit = target * 3;
|
|
if (actualLength >= farLimit) return floor;
|
|
const range = farLimit - limit;
|
|
const over = actualLength - limit;
|
|
return Math.max(floor, 1 - (1 - floor) * (over / range));
|
|
}
|
|
|
|
function clampUnit(x: number): number {
|
|
if (!Number.isFinite(x)) return 0;
|
|
if (x < 0) return 0;
|
|
if (x > 1) return 1;
|
|
return x;
|
|
}
|
|
|
|
function validateWeights(w: { correctness: number; procedure: number; conciseness: number }): void {
|
|
if ([w.correctness, w.procedure, w.conciseness].some(x => x < 0)) {
|
|
throw new Error('Judge weights must be non-negative');
|
|
}
|
|
const sum = w.correctness + w.procedure + w.conciseness;
|
|
if (Math.abs(sum - 1.0) > 1e-6) {
|
|
throw new Error(`Judge weights must sum to 1.0, got ${sum}`);
|
|
}
|
|
}
|
|
|
|
function errorScore(feedback: string): JudgeScore {
|
|
return {
|
|
overall: 0,
|
|
weighted: 0,
|
|
correctness: 0,
|
|
procedureFollowing: 0,
|
|
conciseness: 0,
|
|
lengthPenalty: 1,
|
|
feedback,
|
|
parsed: false,
|
|
};
|
|
}
|