moving

benchmarks/gepa/scripts/faza-1/run-mutation-oracle.ts

@@ -0,0 +1,438 @@
+#!/usr/bin/env tsx
+/**
+ * GEPA Faza 1 — mutation oracle runner.
+ *
+ * Per launch decision §G step 7 + manifest v7 §gepa.mutation_oracle +
+ * §mutation_oracle_design + Amendment 2 §4 (forked Qwen vs non-Qwen templates).
+ *
+ * For each of 5 shapes, generate 2 mutation candidates via Opus 4.7. Each
+ * candidate is validated via mutation-validator.ts (cell-semantic preservation).
+ * Output: 10 candidate files at packages/agent/src/prompt-shapes/gepa-evolved/<shape>-gen1-v<N>.ts
+ *
+ * Approach:
+ * - Use JSON-mode response_format (Amendment 4 lesson) for reliable parsing
+ * - Opus outputs JSON with the 5 method body strings + new evidence_link
+ * - Runner assembles TS file from fixed template (preserves cell semantics by construction)
+ * - Validator confirms types.ts + MULTI_STEP_ACTION_CONTRACT SHAs unchanged
+ *
+ * Cost projection: 10 calls × ~$0.15 = ~$1.50
+ *
+ * Failure handling per brief §5: 2 consecutive invalid mutations from oracle
+ * → halt-and-PM.
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { validateCandidate, type ValidatorVerdict } from '../../src/faza-1/mutation-validator.js';
+import { classifyShape, type TemplateClass } from '../../src/faza-1/mutation-oracle-fork.js';
+import { type ShapeName, QWEN_TARGETED_SHAPES } from '../../src/faza-1/types.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const REPO_ROOT = path.resolve(__dirname, '../../../..');
+
+const PROMPT_SHAPES_DIR = path.join(REPO_ROOT, 'packages/agent/src/prompt-shapes');
+const TYPES_FILE = path.join(PROMPT_SHAPES_DIR, 'types.ts');
+const GEPA_EVOLVED_DIR = path.join(PROMPT_SHAPES_DIR, 'gepa-evolved');
+const ORACLE_LOG = path.join(REPO_ROOT, 'benchmarks/results/gepa-faza1/gen-1/mutation-oracle-run.log');
+const OUT_MANIFEST = path.join(REPO_ROOT, 'benchmarks/results/gepa-faza1/gen-1/mutation-oracle-manifest.json');
+
+const LITELLM_URL = process.env.LITELLM_URL ?? 'http://localhost:4000';
+const ORACLE_MODEL = 'claude-opus-4-7';
+const ORACLE_MAX_TOKENS = 8000;
+const N_MUTATIONS_PER_SHAPE = 2;
+
+const SHAPES: ShapeName[] = ['claude', 'qwen-thinking', 'qwen-non-thinking', 'gpt', 'generic-simple'];
+
+const PHASE_4_3_FAILURE_MODES = `Top T2 categories from Phase 4.3 verdict (decisions/2026-04-28-phase-4-3-rescore-delta-report.md):
+1. unsupported-specifics (10 of 26 T2 hits): hallucinated specifics or overreach beyond materials
+2. missed / didn't-consider / shallow (9 of 26 T2 hits): incomplete coverage of source documents
+3. conflation / weak-synthesis (5 of 26 T2 hits): risks blended together rather than separated
+4. wrong-entity / off-topic (sparse): minor framing errors
+
+Phase 4.5 retrieval-engagement signal: Qwen retrieves 1.33×/task vs Opus 2.33×/task on byte-identical tool surface. H3 corpus NULL-baseline replicated this (mean retrievals = 1.05).`;
+
+// ── Logging ────────────────────────────────────────────────────────────────
+
+function log(msg: string): void {
+  const line = `[${new Date().toISOString()}] ${msg}\n`;
+  try { fs.appendFileSync(ORACLE_LOG, line); } catch { /* dir may not exist yet */ }
+  process.stderr.write(line);
+}
+
+// ── Baseline shape inspection (extract metadata for prompt) ────────────────
+
+interface BaselineShapeMetadata {
+  description: string;
+  modelClass: string;
+  defaultThinking: boolean;
+  defaultMaxTokens: number;
+  shapeFileContent: string;  // full file text
+}
+
+function loadBaselineShape(shapeName: ShapeName): BaselineShapeMetadata {
+  const shapeFile = path.join(PROMPT_SHAPES_DIR, `${shapeName}.ts`);
+  const content = fs.readFileSync(shapeFile, 'utf-8');
+
+  const description = content.match(/description: '([^']+)'/)?.[1] ?? '';
+  const modelClass = content.match(/modelClass: '([^']+)'/)?.[1] ?? shapeName;
+  const defaultThinking = content.match(/defaultThinking: (true|false|undefined)/)?.[1] === 'true';
+  const defaultMaxTokens = Number(content.match(/defaultMaxTokens: (\d+)/)?.[1] ?? 4096);
+
+  return { description, modelClass, defaultThinking, defaultMaxTokens, shapeFileContent: content };
+}
+
+// ── Oracle prompt (JSON-mode) ──────────────────────────────────────────────
+
+function buildOraclePrompt(shapeName: ShapeName, baseline: BaselineShapeMetadata, mutationIdx: number): string {
+  const cls: TemplateClass = classifyShape(shapeName);
+  const isQwen = QWEN_TARGETED_SHAPES.has(shapeName);
+
+  const qwenGuidance = `For Qwen-targeted shape mutation:
+- Emphasize multi-turn retrieval over single-shot retrieval. Phrase like "Continue retrieving until you have evidence from at least 2 distinct queries before finalizing."
+- Add anti-premature-finalization scaffolding. Phrase like "Before finalizing, ask: what gap in evidence remains? Issue another retrieval if any gap exists."
+- Encourage iterative refinement of retrieval queries based on prior turn results.
+- Goal: push mean retrieval_calls per task from current 1.0 baseline toward >= 1.5 (escape Amendment 2 penalty zone) and ideally >= 2.0 (Opus parity proxy).`;
+
+  const nonQwenGuidance = `For non-Qwen shape mutation:
+- Standard mutation guidance per brief §3.3 — evolve reasoning scaffold, planning step structure, chain-of-thought triggers.
+- Restructure implicit reasoning prompts (e.g., "think step by step" variants, planning bullets).
+- Refine where the model is prompted to articulate reasoning before producing output.
+- Improve multi-step task decomposition explicitness.`;
+
+  return `You are an expert prompt engineer. Generate ONE mutated variant of the prompt-shape below, evolving reasoning scaffold + retrieval-engagement guidance while preserving cell semantics.
+
+## Target shape
+- Name: ${shapeName}
+- Class: ${cls}
+- Mutation variant index: ${mutationIdx} (you are generating mutation #${mutationIdx} of 2 for this shape)
+
+## Baseline metadata (LOCKED — do NOT change these)
+- description: ${baseline.description}
+- modelClass: ${baseline.modelClass}
+- defaultThinking: ${baseline.defaultThinking}
+- defaultMaxTokens: ${baseline.defaultMaxTokens}
+
+## Phase 4.3 + Phase 4.5 failure modes to address
+${PHASE_4_3_FAILURE_MODES}
+
+## Mutation guidance
+${isQwen ? qwenGuidance : nonQwenGuidance}
+
+## Cell semantic boundaries (LOCKED — violation = REJECTED candidate)
+You may NOT modify:
+- The MULTI_STEP_ACTION_CONTRACT constant (lives in types.ts; bytes are SHA-pinned)
+- The JSON action contract format ({"action": "retrieve" | "finalize", ...})
+- Task framing (persona/question/materials section labels)
+- Imports block
+- Locked metadata fields above
+
+You MAY modify:
+- The 5 method bodies (string-building only): systemPromptSolo, systemPromptMultiStep, soloUserPrompt, multiStepKickoffUserPrompt, retrievalInjectionUserPrompt
+- The evidence_link metadata (you MUST update to point to GEPA Gen 1 results: "benchmarks/results/gepa-faza1/gen-1/mutation-oracle-run.log + Phase 4.5 + Amendment 2 §3 retrieval-engagement bonus")
+
+## Baseline shape file (your input)
+
+\`\`\`typescript
+${baseline.shapeFileContent}
+\`\`\`
+
+## Your output: JSON object
+
+Output a single JSON object with these fields. Each method body field should be a TypeScript expression that evaluates to a string (the prompt text). Use the same approach as the baseline (e.g., array.join('\\n')). Variable references like \${persona}, \${question}, \${maxSteps}, \${maxRetrievalsPerStep}, \${input.persona}, etc. must be preserved verbatim where the baseline used them.
+
+\`\`\`json
+{
+  "evidenceLink": "<updated evidence_link string referencing Gen 1 results + Phase 4.5/Amendment 2>",
+  "systemPromptSolo": "<TypeScript expression returning system prompt for !isMultiStep — typically a join of strings or template literal; reference {persona}>",
+  "systemPromptMultiStep": "<TypeScript expression returning system prompt for isMultiStep — must reference MULTI_STEP_ACTION_CONTRACT verbatim, {persona}, {question}, {maxSteps}, {maxRetrievalsPerStep}>",
+  "soloUserPrompt": "<TypeScript expression returning user prompt with {input.persona}, {input.materials}, {input.question}>",
+  "multiStepKickoffUserPrompt": "<TypeScript expression returning kickoff user message; baseline uses 'Begin. Output your first action JSON now.' — your variant should request engagement scaffolding for Qwen shapes>",
+  "retrievalInjectionUserPrompt": "<TypeScript expression returning retrieval-injection user message with {input.query}, {input.resultCount}, {input.results}>"
+}
+\`\`\`
+
+CRITICAL: each field's value must be a STRING containing valid TypeScript code that, when wrapped in \`return (\${value})\`, would compile + return a string. The simplest valid pattern is template literals or .join('\\n') over an array of strings.
+
+Output ONLY the JSON object. No prose. No code fences.`;
+}
+
+// ── LLM call (JSON-mode, no temperature per Amendment 4 lesson) ────────────
+
+interface OracleCallResult {
+  content: string;
+  inTokens: number;
+  outTokens: number;
+  costUsd: number;
+  latencyMs: number;
+  error?: string;
+}
+
+async function callOpusOracle(prompt: string): Promise<OracleCallResult> {
+  const masterKey = process.env.LITELLM_MASTER_KEY;
+  if (!masterKey) throw new Error('LITELLM_MASTER_KEY env not set');
+
+  const payload = {
+    model: ORACLE_MODEL,
+    messages: [{ role: 'user', content: prompt }],
+    max_tokens: ORACLE_MAX_TOKENS,
+    response_format: { type: 'json_object' },
+    // temperature omitted per Amendment 4 (Anthropic deprecates with JSON mode)
+  };
+
+  const started = Date.now();
+  let lastErr: string | undefined;
+  for (let attempt = 0; attempt < 2; attempt++) {
+    try {
+      const resp = await fetch(`${LITELLM_URL}/chat/completions`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${masterKey}` },
+        body: JSON.stringify(payload),
+      });
+      const d: any = await resp.json();
+      if ('error' in d) {
+        lastErr = String(d.error?.message ?? JSON.stringify(d.error)).slice(0, 200);
+        if (attempt < 1) await new Promise(r => setTimeout(r, 1500));
+        continue;
+      }
+      const content = d.choices?.[0]?.message?.content ?? '';
+      const usage = d.usage ?? {};
+      const inTok = usage.prompt_tokens ?? 0;
+      const outTok = usage.completion_tokens ?? 0;
+      const costUsd = (inTok * 15.0 + outTok * 75.0) / 1_000_000;
+      return { content, inTokens: inTok, outTokens: outTok, costUsd, latencyMs: Date.now() - started };
+    } catch (e) {
+      lastErr = `${(e as Error).name}: ${(e as Error).message}`.slice(0, 200);
+      if (attempt < 1) await new Promise(r => setTimeout(r, 1500));
+    }
+  }
+  return { content: '', inTokens: 0, outTokens: 0, costUsd: 0, latencyMs: Date.now() - started, error: lastErr };
+}
+
+// ── JSON parsing + TS file assembly ────────────────────────────────────────
+
+interface MutationFields {
+  evidenceLink: string;
+  systemPromptSolo: string;
+  systemPromptMultiStep: string;
+  soloUserPrompt: string;
+  multiStepKickoffUserPrompt: string;
+  retrievalInjectionUserPrompt: string;
+}
+
+function parseOracleOutput(content: string): MutationFields | { error: string } {
+  let s = content.trim();
+  if (s.startsWith('```')) s = s.replace(/^```[a-z]*\n?/, '').replace(/```\s*$/, '');
+  const firstBrace = s.indexOf('{');
+  const lastBrace = s.lastIndexOf('}');
+  if (firstBrace < 0 || lastBrace < 0) {
+    return { error: `no JSON object found in output (length=${content.length})` };
+  }
+  try {
+    const obj = JSON.parse(s.slice(firstBrace, lastBrace + 1));
+    const required = ['evidenceLink', 'systemPromptSolo', 'systemPromptMultiStep', 'soloUserPrompt', 'multiStepKickoffUserPrompt', 'retrievalInjectionUserPrompt'];
+    for (const k of required) {
+      if (typeof obj[k] !== 'string' || obj[k].length === 0) {
+        return { error: `field "${k}" missing or empty` };
+      }
+    }
+    return obj as MutationFields;
+  } catch (e) {
+    return { error: `JSON parse failed: ${(e as Error).message}` };
+  }
+}
+
+function buildShapeFile(shapeName: ShapeName, baseline: BaselineShapeMetadata, fields: MutationFields, mutationIdx: number): string {
+  const exportName = `${shapeName.replace(/-([a-z])/g, (_, c) => c.toUpperCase())}Gen1V${mutationIdx}Shape`;
+  const tsName = `${shapeName}-gen1-v${mutationIdx}`;
+  return `/**
+ * GEPA Faza 1 — Gen 1 mutation #${mutationIdx} of ${shapeName}.
+ *
+ * Generated by Opus 4.7 mutation oracle per Amendment 2 §4 forked template
+ * (${classifyShape(shapeName)} branch). Cell-semantic boundaries preserved
+ * via mutation-validator.ts SHA pins.
+ *
+ * Evidence: ${fields.evidenceLink}
+ *
+ * Baseline anchor: packages/agent/src/prompt-shapes/${shapeName}.ts (manifest v7
+ * §gepa.mutation_validator.baseline_shape_shas[${shapeName}.ts]).
+ */
+
+import {
+  type PromptShape,
+  type SystemPromptInput,
+  type SoloUserPromptInput,
+  type MultiStepKickoffInput,
+  type RetrievalInjectionInput,
+  MULTI_STEP_ACTION_CONTRACT,
+} from '../types.js';
+
+export const ${exportName}: PromptShape = {
+  name: '${tsName}',
+  metadata: {
+    description: '${baseline.description.replace(/'/g, "\\'")}',
+    modelClass: '${baseline.modelClass}',
+    evidence_link: ${JSON.stringify(fields.evidenceLink)},
+    defaultThinking: ${baseline.defaultThinking},
+    defaultMaxTokens: ${baseline.defaultMaxTokens},
+  },
+
+  systemPrompt(input: SystemPromptInput): string {
+    const { persona, question, isMultiStep, maxSteps = 5, maxRetrievalsPerStep = 8 } = input;
+    if (!isMultiStep) {
+      return ${fields.systemPromptSolo};
+    }
+    return ${fields.systemPromptMultiStep};
+  },
+
+  soloUserPrompt(input: SoloUserPromptInput): string {
+    return ${fields.soloUserPrompt};
+  },
+
+  multiStepKickoffUserPrompt(_input: MultiStepKickoffInput): string {
+    return ${fields.multiStepKickoffUserPrompt};
+  },
+
+  retrievalInjectionUserPrompt(input: RetrievalInjectionInput): string {
+    return ${fields.retrievalInjectionUserPrompt};
+  },
+};
+`;
+}
+
+// ── Validate via runtime sanity (compile check) + structural check ─────────
+
+function validateAssembledFile(filepath: string, baselineShapeName: ShapeName): { valid: boolean; reason?: string } {
+  // Check file parses as TypeScript by attempting a require-style import
+  // For Faza 1 simplicity, just check that the file:
+  // 1. Imports from types.js (mutation-validator's import preservation check)
+  // 2. Contains the required locked metadata fields
+  // 3. Exports a single PromptShape
+  // 4. Doesn't break the cell-semantic boundary (types.ts/MULTI_STEP_ACTION_CONTRACT SHAs unchanged)
+  // The mutation-validator.ts does the SHA checks; we use it via validateCandidate.
+
+  const verdict: ValidatorVerdict = validateCandidate({
+    candidateShapeFilePath: filepath,
+    baselineShapeName: `${baselineShapeName}.ts` as any,
+    typesFilePath: TYPES_FILE,
+    expectShapeDiff: true,
+  });
+
+  if (!verdict.valid) {
+    return { valid: false, reason: `validator violations: ${verdict.violations.map(v => `${v.category}: ${v.detail}`).join('; ')}` };
+  }
+  return { valid: true };
+}
+
+// ── Main: generate 10 mutations ────────────────────────────────────────────
+
+interface MutationManifestEntry {
+  shape: ShapeName;
+  mutationIdx: number;
+  filename: string;
+  costUsd: number;
+  latencyMs: number;
+  tsName: string;
+  validatorVerdict: 'valid' | 'invalid_after_retry';
+  error?: string;
+}
+
+async function main(): Promise<void> {
+  fs.mkdirSync(GEPA_EVOLVED_DIR, { recursive: true });
+  fs.mkdirSync(path.dirname(ORACLE_LOG), { recursive: true });
+  if (!fs.existsSync(ORACLE_LOG)) fs.writeFileSync(ORACLE_LOG, '');
+
+  log(`[start] mutation oracle for ${SHAPES.length} shapes × ${N_MUTATIONS_PER_SHAPE} mutations`);
+
+  const manifest: MutationManifestEntry[] = [];
+  let cumulativeCost = 0;
+  let consecutiveInvalidGlobal = 0;
+
+  for (const shapeName of SHAPES) {
+    const baseline = loadBaselineShape(shapeName);
+    log(`[${shapeName}] baseline loaded; modelClass=${baseline.modelClass} description="${baseline.description.slice(0, 60)}..."`);
+
+    for (let mutIdx = 1; mutIdx <= N_MUTATIONS_PER_SHAPE; mutIdx++) {
+      const tsName = `${shapeName}-gen1-v${mutIdx}`;
+      const outFile = path.join(GEPA_EVOLVED_DIR, `${tsName}.ts`);
+
+      if (fs.existsSync(outFile)) {
+        log(`[${tsName}] already exists; skipping`);
+        manifest.push({ shape: shapeName, mutationIdx: mutIdx, filename: outFile, costUsd: 0, latencyMs: 0, tsName, validatorVerdict: 'valid' });
+        continue;
+      }
+
+      // Try once; if invalid, retry once with structural feedback. After 2 fails → mark invalid_after_retry.
+      let valid = false;
+      let totalCost = 0;
+      let totalLatency = 0;
+      let errMsg: string | undefined;
+
+      for (let attempt = 0; attempt < 2 && !valid; attempt++) {
+        const prompt = buildOraclePrompt(shapeName, baseline, mutIdx);
+        log(`[${tsName}] oracle call attempt ${attempt + 1}; prompt_len=${prompt.length}c`);
+        const llm = await callOpusOracle(prompt);
+        totalCost += llm.costUsd;
+        totalLatency += llm.latencyMs;
+
+        if (llm.error) {
+          errMsg = `LLM error: ${llm.error}`;
+          log(`[${tsName}] ${errMsg}`);
+          continue;
+        }
+        const parsed = parseOracleOutput(llm.content);
+        if ('error' in parsed) {
+          errMsg = `parse error: ${parsed.error}`;
+          log(`[${tsName}] ${errMsg}; first 200c: ${llm.content.slice(0, 200)}`);
+          continue;
+        }
+
+        const tsContent = buildShapeFile(shapeName, baseline, parsed, mutIdx);
+        fs.writeFileSync(outFile, tsContent, 'utf-8');
+
+        const v = validateAssembledFile(outFile, shapeName);
+        if (v.valid) {
+          valid = true;
+          log(`[${tsName}] OK; cost=$${llm.costUsd.toFixed(4)}; latency=${llm.latencyMs}ms; file=${path.basename(outFile)}`);
+        } else {
+          errMsg = `validator failed: ${v.reason}`;
+          log(`[${tsName}] ${errMsg}`);
+          fs.unlinkSync(outFile);
+        }
+      }
+
+      cumulativeCost += totalCost;
+      manifest.push({
+        shape: shapeName, mutationIdx: mutIdx, filename: outFile,
+        costUsd: totalCost, latencyMs: totalLatency, tsName,
+        validatorVerdict: valid ? 'valid' : 'invalid_after_retry',
+        error: valid ? undefined : errMsg,
+      });
+
+      if (valid) {
+        consecutiveInvalidGlobal = 0;
+      } else {
+        consecutiveInvalidGlobal++;
+        if (consecutiveInvalidGlobal >= 2) {
+          log(`[HALT] 2 consecutive invalid mutations from oracle (per brief §5) — stopping`);
+          break;
+        }
+      }
+    }
+    if (consecutiveInvalidGlobal >= 2) break;
+  }
+
+  fs.writeFileSync(OUT_MANIFEST, JSON.stringify({
+    totalMutationsAttempted: manifest.length,
+    totalValid: manifest.filter(m => m.validatorVerdict === 'valid').length,
+    totalInvalid: manifest.filter(m => m.validatorVerdict !== 'valid').length,
+    cumulativeCostUsd: +cumulativeCost.toFixed(6),
+    entries: manifest,
+    completedAtIso: new Date().toISOString(),
+  }, null, 2));
+  log(`[done] ${manifest.length} mutations attempted; ${manifest.filter(m => m.validatorVerdict === 'valid').length} valid; cumulative $${cumulativeCost.toFixed(4)}`);
+}
+
+main().catch(e => { console.error('FATAL:', e); process.exit(2); });