moving

packages/agent/tests/evolution-orchestrator.test.ts

@@ -0,0 +1,447 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { MindDB, ExecutionTraceStore, EvolutionRunStore } from '@waggle/core';
+import type { ParsedExecutionTrace, EvolutionRun } from '@waggle/core';
+import {
+  EvolutionOrchestrator,
+  eligibleForEvolution,
+  summarizeRuns,
+} from '../src/evolution-orchestrator.js';
+import type { JudgeScore } from '../src/judge.js';
+import type { Schema, SchemaExecuteFn } from '../src/evolve-schema.js';
+import type { MutateFn } from '../src/iterative-optimizer.js';
+
+// ── Fixtures ───────────────────────────────────────────────────
+
+function makeSchema(fields: string[]): Schema {
+  return {
+    name: 'test',
+    fields: fields.map(n => ({
+      name: n, type: 'string' as const,
+      description: `${n} field`,
+      required: true, constraints: [],
+    })),
+    version: 1,
+  };
+}
+
+function makeJudgeScore(overall: number, feedback = 'ok'): JudgeScore {
+  return {
+    overall, weighted: overall,
+    correctness: overall, procedureFollowing: overall, conciseness: overall,
+    lengthPenalty: 1, feedback, parsed: true,
+  };
+}
+
+// Executor that produces longer output for bigger schemas.
+function makeExec(): SchemaExecuteFn {
+  return async ({ schema }) => ({
+    actual: schema.fields.map(f => f.name).join(','),
+    parsed: true,
+  });
+}
+
+// Judge that prefers candidates containing the "plus" token so the
+// mutateAppend helper ("... plus") wins reliably. Creates a measurable
+// delta between baseline and winner regardless of absolute lengths.
+function makeJudge(): { score: (args: { input: string; expected: string; actual: string }) => Promise<JudgeScore> } {
+  return {
+    async score(args) {
+      const base = Math.min(1, args.actual.length / 100) * 0.6;
+      const bonus = args.actual.includes('plus') ? 0.3 : 0;
+      return makeJudgeScore(Math.min(1, base + bonus), 'the response is too terse');
+    },
+  };
+}
+
+// Mutate appends ~10 chars so growth ratio stays below the default +20%.
+const mutateAppend: MutateFn = async ({ parent }) => `${parent.prompt} plus`;
+
+// A baseline that's long enough that +20% growth cap isn't a problem for short appends.
+const BASELINE =
+  'Return a concise answer that directly addresses the question. ' +
+  'Provide a brief explanation and the final answer. Be clear and accurate.';
+
+function seedSuccessfulTraces(store: ExecutionTraceStore, n: number, personaId = 'researcher') {
+  for (let i = 0; i < n; i++) {
+    const id = store.start({
+      personaId,
+      input: `test question ${i} with enough length to pass filters`,
+      workspaceId: 'ws-1',
+      taskShape: 'qa',
+    });
+    store.finalize(id, {
+      outcome: 'success',
+      output: `a full answer for question ${i} that is long enough for use`,
+    });
+  }
+}
+
+function baseComposeOptions() {
+  return {
+    schema: {
+      examples: [] as never[],
+      execute: makeExec(),
+      judge: makeJudge(),
+      populationSize: 2, generations: 1,
+      evalSize: 3, anchorEvalSize: 3,
+    },
+    instructions: {
+      examples: [] as never[],
+      judge: makeJudge(),
+      mutate: mutateAppend,
+      allowBareJudge: true,
+      populationSize: 2, generations: 1,
+      microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 3,
+    },
+  };
+}
+
+// ── eligibleForEvolution ──
+
+describe('eligibleForEvolution', () => {
+  it('keeps finalized traces with non-empty input and output', () => {
+    const eligible = eligibleForEvolution([
+      {
+        id: 1, session_id: 's', persona_id: null, workspace_id: null,
+        model: null, task_shape: null, outcome: 'success',
+        cost_usd: 0, duration_ms: 0, created_at: '', finalized_at: '',
+        payload: {
+          input: 'q', output: 'a', reasoning: [], toolCalls: [], artifacts: [],
+          tokens: { input: 0, output: 0 }, tags: [],
+        },
+      } as unknown as ParsedExecutionTrace,
+    ]);
+    expect(eligible).toHaveLength(1);
+  });
+
+  it('drops pending traces', () => {
+    const eligible = eligibleForEvolution([
+      {
+        id: 1, outcome: 'pending',
+        payload: { input: 'q', output: 'a', reasoning: [], toolCalls: [], artifacts: [], tokens: { input: 0, output: 0 }, tags: [] },
+      } as unknown as ParsedExecutionTrace,
+    ]);
+    expect(eligible).toHaveLength(0);
+  });
+
+  it('drops traces with empty input', () => {
+    const eligible = eligibleForEvolution([
+      {
+        id: 1, outcome: 'success',
+        payload: { input: '', output: 'a', reasoning: [], toolCalls: [], artifacts: [], tokens: { input: 0, output: 0 }, tags: [] },
+      } as unknown as ParsedExecutionTrace,
+    ]);
+    expect(eligible).toHaveLength(0);
+  });
+
+  it('keeps corrected traces even with empty output as long as correctionFeedback is present', () => {
+    const eligible = eligibleForEvolution([
+      {
+        id: 1, outcome: 'corrected',
+        payload: {
+          input: 'q', output: '', reasoning: [], toolCalls: [], artifacts: [],
+          tokens: { input: 0, output: 0 }, tags: [],
+          correctionFeedback: 'use bullets',
+        },
+      } as unknown as ParsedExecutionTrace,
+    ]);
+    expect(eligible).toHaveLength(1);
+  });
+});
+
+// ── summarizeRuns ──
+
+describe('summarizeRuns', () => {
+  it('returns zero-aggregates on empty input', () => {
+    const s = summarizeRuns([]);
+    expect(s.total).toBe(0);
+    expect(s.byStatus.proposed).toBe(0);
+    expect(s.bestDelta).toBe(0);
+  });
+
+  it('counts by status and target kind, tracks best delta', () => {
+    const runs = [
+      { status: 'proposed', target_kind: 'persona-system-prompt', delta_accuracy: 0.05 },
+      { status: 'accepted', target_kind: 'persona-system-prompt', delta_accuracy: 0.12 },
+      { status: 'rejected', target_kind: 'tool-description', delta_accuracy: 0.02 },
+      { status: 'deployed', target_kind: 'persona-system-prompt', delta_accuracy: 0.09 },
+    ] as unknown as EvolutionRun[];
+    const s = summarizeRuns(runs);
+    expect(s.total).toBe(4);
+    expect(s.byStatus.proposed).toBe(1);
+    expect(s.byStatus.accepted).toBe(1);
+    expect(s.byStatus.rejected).toBe(1);
+    expect(s.byStatus.deployed).toBe(1);
+    expect(s.byTargetKind['persona-system-prompt']).toBe(3);
+    expect(s.byTargetKind['tool-description']).toBe(1);
+    expect(s.bestDelta).toBeCloseTo(0.12, 5);
+  });
+});
+
+// ── EvolutionOrchestrator end-to-end ───────────────────────────
+
+describe('EvolutionOrchestrator', () => {
+  let db: MindDB;
+  let traceStore: ExecutionTraceStore;
+  let runStore: EvolutionRunStore;
+  let orchestrator: EvolutionOrchestrator;
+
+  beforeEach(() => {
+    db = new MindDB(':memory:');
+    traceStore = new ExecutionTraceStore(db);
+    runStore = new EvolutionRunStore(db);
+    orchestrator = new EvolutionOrchestrator({ traceStore, runStore });
+  });
+
+  afterEach(() => {
+    db.close();
+  });
+
+  // ── runOnce ──
+
+  describe('runOnce', () => {
+    it('produces a proposed run when gates pass and delta exceeds minimum', async () => {
+      seedSuccessfulTraces(traceStore, 10);
+      const result = await orchestrator.runOnce({
+        targetKind: 'persona-system-prompt',
+        targetName: 'researcher',
+        baseline: BASELINE,
+        schemaBaseline: makeSchema(['answer']),
+        compose: baseComposeOptions(),
+      });
+
+      expect(result.outcome).toBe('proposed');
+      expect(result.run?.status).toBe('proposed');
+      expect(result.run?.delta_accuracy).toBeGreaterThan(0);
+      expect(result.compose).toBeDefined();
+      expect(result.gateResults).toBeDefined();
+    });
+
+    it('persists compose artifacts (schema JSON, gates, delta)', async () => {
+      seedSuccessfulTraces(traceStore, 8);
+      const result = await orchestrator.runOnce({
+        targetKind: 'persona-system-prompt',
+        targetName: 'researcher',
+        baseline: BASELINE,
+        schemaBaseline: makeSchema(['answer']),
+        compose: baseComposeOptions(),
+      });
+
+      const run = result.run!;
+      expect(run.winner_schema_json).toBeTruthy();
+      const schema = JSON.parse(run.winner_schema_json!);
+      expect(schema.name).toBe('test');
+      expect(JSON.parse(run.gate_reasons_json).length).toBeGreaterThan(0);
+    });
+
+    it('skips with skipped-delta when improvement is below threshold', async () => {
+      seedSuccessfulTraces(traceStore, 8);
+      const result = await orchestrator.runOnce({
+        targetKind: 'persona-system-prompt',
+        baseline: 'a reasonably detailed baseline prompt that already scores well',
+        schemaBaseline: makeSchema(['answer']),
+        compose: {
+          ...baseComposeOptions(),
+          instructions: {
+            ...baseComposeOptions().instructions,
+            // Mutate returns the same text → zero delta
+            mutate: async ({ parent }) => parent.prompt,
+          },
+        },
+        minDelta: 0.5, // artificially high
+      });
+
+      expect(result.outcome).toBe('skipped-delta');
+      expect(result.run).toBeUndefined();
+    });
+
+    it('auto-trigger: skips when too few traces', async () => {
+      seedSuccessfulTraces(traceStore, 2);
+      const result = await orchestrator.runOnce({
+        targetKind: 'persona-system-prompt',
+        baseline: 'baseline',
+        schemaBaseline: makeSchema(['answer']),
+        compose: baseComposeOptions(),
+        autoTrigger: { minTraces: 50 },
+      });
+
+      expect(result.outcome).toBe('skipped-trigger');
+      expect(result.run).toBeUndefined();
+    });
+
+    it('auto-trigger: proceeds when threshold is met', async () => {
+      seedSuccessfulTraces(traceStore, 10);
+      const result = await orchestrator.runOnce({
+        targetKind: 'persona-system-prompt',
+        baseline: BASELINE,
+        schemaBaseline: makeSchema(['answer']),
+        compose: baseComposeOptions(),
+        autoTrigger: { minTraces: 5 },
+      });
+
+      expect(result.outcome).toBe('proposed');
+    });
+
+    it('immediately rejects gate-failing runs (still persisted for audit)', async () => {
+      seedSuccessfulTraces(traceStore, 8);
+      // Judge that prefers the mutated "y" text — ensures the GEPA winner
+      // is the oversized candidate, not the baseline.
+      const yPreferringJudge = {
+        async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
+          return makeJudgeScore(args.actual.includes('y') ? 0.9 : 0.3, 'ok');
+        },
+      };
+      const result = await orchestrator.runOnce({
+        targetKind: 'tool-description',
+        baseline: 'x'.repeat(30),
+        schemaBaseline: makeSchema(['answer']),
+        compose: {
+          ...baseComposeOptions(),
+          instructions: {
+            ...baseComposeOptions().instructions,
+            judge: yPreferringJudge,
+            // Mutate produces candidate that blows past tool-description cap (500)
+            mutate: async () => 'y'.repeat(2000),
+          },
+        },
+        minDelta: 0, // allow any delta so we reach the gate stage
+      });
+
+      expect(result.outcome).toBe('skipped-gates');
+      expect(result.run?.status).toBe('rejected');
+      expect(result.run?.gate_verdict).toBe('fail');
+      expect(result.run?.user_note).toMatch(/gate failure/);
+    });
+
+    it('aborts cleanly when signal fires before start', async () => {
+      seedSuccessfulTraces(traceStore, 5);
+      const ctrl = new AbortController();
+      ctrl.abort();
+      const result = await orchestrator.runOnce({
+        targetKind: 'generic',
+        baseline: 'baseline',
+        schemaBaseline: makeSchema(['answer']),
+        compose: baseComposeOptions(),
+        signal: ctrl.signal,
+      });
+      expect(result.outcome).toBe('aborted');
+    });
+
+    it('emits progress events', async () => {
+      seedSuccessfulTraces(traceStore, 5);
+      const phases: string[] = [];
+      await orchestrator.runOnce({
+        targetKind: 'generic',
+        baseline: BASELINE,
+        schemaBaseline: makeSchema(['answer']),
+        compose: baseComposeOptions(),
+        onProgress: (e) => phases.push(e.phase),
+      });
+      expect(phases).toContain('compose');
+      expect(phases).toContain('done');
+    });
+  });
+
+  // ── accept ──
+
+  describe('accept', () => {
+    it('invokes deploy callback and marks run deployed on success', async () => {
+      const deploy = vi.fn(async () => { /* success */ });
+      const deployedOrch = new EvolutionOrchestrator({ traceStore, runStore, deploy });
+      seedSuccessfulTraces(traceStore, 8);
+
+      const run = await deployedOrch.runOnce({
+        targetKind: 'persona-system-prompt',
+        baseline: BASELINE,
+        schemaBaseline: makeSchema(['answer']),
+        compose: baseComposeOptions(),
+      });
+      expect(run.outcome).toBe('proposed');
+
+      const accepted = await deployedOrch.accept(run.run!.run_uuid, 'good mutation');
+      expect(accepted?.status).toBe('deployed');
+      expect(deploy).toHaveBeenCalledTimes(1);
+    });
+
+    it('marks run failed when deploy callback throws', async () => {
+      const deploy = vi.fn(async () => { throw new Error('persona write failed'); });
+      const orch = new EvolutionOrchestrator({ traceStore, runStore, deploy });
+      seedSuccessfulTraces(traceStore, 8);
+
+      const run = await orch.runOnce({
+        targetKind: 'persona-system-prompt',
+        baseline: BASELINE,
+        schemaBaseline: makeSchema(['answer']),
+        compose: baseComposeOptions(),
+      });
+
+      const result = await orch.accept(run.run!.run_uuid);
+      expect(result?.status).toBe('failed');
+      expect(result?.failure_reason).toContain('persona write failed');
+    });
+
+    it('leaves run as accepted when no deploy hook configured', async () => {
+      seedSuccessfulTraces(traceStore, 8);
+      const run = await orchestrator.runOnce({
+        targetKind: 'generic',
+        baseline: BASELINE,
+        schemaBaseline: makeSchema(['answer']),
+        compose: baseComposeOptions(),
+      });
+      const accepted = await orchestrator.accept(run.run!.run_uuid);
+      expect(accepted?.status).toBe('accepted');
+    });
+
+    it('returns undefined for unknown uuid', async () => {
+      expect(await orchestrator.accept('does-not-exist')).toBeUndefined();
+    });
+  });
+
+  // ── reject ──
+
+  describe('reject', () => {
+    it('marks proposed run as rejected', async () => {
+      seedSuccessfulTraces(traceStore, 8);
+      const run = await orchestrator.runOnce({
+        targetKind: 'generic',
+        baseline: BASELINE,
+        schemaBaseline: makeSchema(['answer']),
+        compose: baseComposeOptions(),
+      });
+      const rejected = orchestrator.reject(run.run!.run_uuid, 'regressed on custom cases');
+      expect(rejected?.status).toBe('rejected');
+      expect(rejected?.user_note).toBe('regressed on custom cases');
+    });
+  });
+
+  // ── list / get ──
+
+  describe('list / get', () => {
+    it('returns runs in reverse-chronological order', async () => {
+      seedSuccessfulTraces(traceStore, 8);
+      await orchestrator.runOnce({
+        targetKind: 'generic', baseline: 'a',
+        schemaBaseline: makeSchema(['answer']),
+        compose: baseComposeOptions(),
+      });
+      await orchestrator.runOnce({
+        targetKind: 'generic', baseline: 'b',
+        schemaBaseline: makeSchema(['answer']),
+        compose: baseComposeOptions(),
+      });
+      const list = orchestrator.list();
+      expect(list.length).toBeGreaterThanOrEqual(2);
+    });
+
+    it('get returns a specific run by uuid', async () => {
+      seedSuccessfulTraces(traceStore, 8);
+      const created = await orchestrator.runOnce({
+        targetKind: 'generic', baseline: 'short',
+        schemaBaseline: makeSchema(['answer']),
+        compose: baseComposeOptions(),
+      });
+      const fetched = orchestrator.get(created.run!.run_uuid);
+      expect(fetched?.id).toBe(created.run!.id);
+    });
+  });
+});