moving

scripts/run-pilot-2026-04-26.ts

@@ -0,0 +1,951 @@
+#!/usr/bin/env tsx
+/**
+ * Agentic Knowledge Work Pilot — N=3 direction validator
+ *
+ * Pilot ID:           agentic-knowledge-work-pilot-2026-04-26
+ * Manifest anchor:    pilot-2026-04-26-v1
+ * Cost ceiling:       $7.00 hard / $6.00 halt (per amendment §6)
+ * Per-cell halt:      $0.50 (Cells B/D)
+ * Wall budget:        7-10h (per amendment §6)
+ *
+ * Authority:
+ *   - cc1-brief.md (predecessor, audit-immutable)
+ *   - cc1-brief-amendment-2026-04-26.md (binding execution doc)
+ *   - judge-rubric.md (Likert 1-5 × 6 dimensions × trio ensemble)
+ *
+ * §11 frozen path compliance: this script is a NEW wrapper at scripts/.
+ * It does NOT touch any §11 frozen path. It uses @waggle/core (in-tree
+ * memory substrate) and direct HTTP to LiteLLM. No imports from
+ * benchmarks/harness/src/* (LoCoMo wrapper untouched).
+ *
+ * Usage:
+ *   npx tsx scripts/run-pilot-2026-04-26.ts --smoke                      # Task 1 only, all 4 cells
+ *   npx tsx scripts/run-pilot-2026-04-26.ts --task task-2 --all-cells    # Single task, all cells
+ *   npx tsx scripts/run-pilot-2026-04-26.ts --task task-3 --cell B       # Single task + single cell
+ *   npx tsx scripts/run-pilot-2026-04-26.ts --all-tasks --all-cells      # Full pilot (12 cells)
+ *   npx tsx scripts/run-pilot-2026-04-26.ts --dry-run --smoke            # No API calls; sanity-check parsing
+ */
+
+import * as fs from 'node:fs';
+import * as fsp from 'node:fs/promises';
+import * as path from 'node:path';
+import * as crypto from 'node:crypto';
+import { fileURLToPath } from 'node:url';
+import { execFileSync } from 'node:child_process';
+
+import {
+  MindDB,
+  FrameStore,
+  SessionStore,
+  HybridSearch,
+  createOllamaEmbedder,
+  type Embedder,
+} from '@waggle/core';
+
+// Phase 2.2 — pilot wrapper now consumes the unified agent loop from
+// @waggle/agent (Phase 2.1 commit a599a07). Local re-implementations of
+// runCellSolo / runCellMultiStep / parseAgentAction are removed; this
+// wrapper provides only the LlmCallFn + RetrievalSearchFn adapters, plus
+// the pilot-specific orchestration (task loading, cell loop, judge ensemble,
+// cost accounting, JSONL output, audit chain).
+import {
+  runSoloAgent,
+  runRetrievalAgentLoop,
+  type LlmCallFn,
+  type LlmCallInput,
+  type LlmCallResult as AgentLlmCallResult,
+  type RetrievalSearchFn,
+  type AgentRunResult,
+} from '@waggle/agent';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// ─────────────────────────────────────────────────────────────────────────
+// Constants
+// ─────────────────────────────────────────────────────────────────────────
+
+const REPO_ROOT = path.resolve(__dirname, '..');
+const PILOT_ID = 'agentic-knowledge-work-pilot-2026-04-26';
+const MANIFEST_ANCHOR = 'pilot-2026-04-26-v1';
+const BRIEF_DIR = 'D:/Projects/PM-Waggle-OS/briefs/2026-04-26-agentic-knowledge-work-pilot';
+const CC1_BRIEF_PATH = path.join(BRIEF_DIR, 'cc1-brief.md');
+const AMENDMENT_PATH = path.join(BRIEF_DIR, 'cc1-brief-amendment-2026-04-26.md');
+const JUDGE_RUBRIC_PATH = path.join(BRIEF_DIR, 'judge-rubric.md');
+
+const OUT_DIR = path.join(REPO_ROOT, 'benchmarks', 'results', 'pilot-2026-04-26');
+const PROMPTS_ARCHIVE_DIR = path.join(OUT_DIR, 'prompts-archive');
+const RUN_LOG_PATH = path.join(OUT_DIR, 'pilot-run.log');
+const SCRATCH_DIR = path.join(REPO_ROOT, 'tmp', 'pilot-2026-04-26');
+
+const LITELLM_URL = 'http://localhost:4000';
+const OLLAMA_URL = 'http://localhost:11434';
+const EMBEDDER_MODEL = 'nomic-embed-text';
+
+// Amendment v2 §4 (PM-revised cost ceiling — methodology priority over budget tightness):
+const COST_CAP_USD = 20.0;
+const COST_HALT_USD = 17.0;
+const PER_CELL_HARD_HALT_USD = 1.0;
+const PER_CALL_SANITY_USD = 0.4;  // hard halt + ping (was $0.50 sanity ping in v1)
+
+const MAX_STEPS = 5;
+const MAX_RETRIEVALS_PER_STEP = 8;
+const MAX_JUDGE_RETRIES = 3;
+
+const TASK_FILES: Record<string, string> = {
+  'task-1': path.join(BRIEF_DIR, 'task-1-strategic-synthesis.md'),
+  'task-2': path.join(BRIEF_DIR, 'task-2-cross-thread-coordination.md'),
+  'task-3': path.join(BRIEF_DIR, 'task-3-decision-support.md'),
+};
+
+// Default Qwen config — overridable via --qwen-alias / --qwen-max-tokens / --qwen-thinking.
+// Amendment v2 §2 binding: alias=qwen3.6-35b-a3b-via-dashscope-direct, thinking=on, max_tokens=16000.
+const DEFAULT_QWEN_ALIAS = 'qwen3.6-35b-a3b-via-dashscope-direct';
+const DEFAULT_QWEN_MAX_TOKENS = 16000;
+const DEFAULT_QWEN_THINKING_ON = true;
+
+function buildCells(qwenAlias: string) {
+  return {
+    A: { model: 'claude-opus-4-7', mode: 'solo' as const, label: 'Opus solo' },
+    B: { model: 'claude-opus-4-7', mode: 'multistep' as const, label: 'Opus + memory + harness' },
+    C: { model: qwenAlias, mode: 'solo' as const, label: 'Qwen solo' },
+    D: { model: qwenAlias, mode: 'multistep' as const, label: 'Qwen + memory + harness' },
+  };
+}
+const CELLS = buildCells(DEFAULT_QWEN_ALIAS);
+
+// Module-level mutable Qwen opts — set from CLI args in main(); applied by
+// the LlmCallFn adapter when the model alias matches Qwen. Subject calls go
+// through the agent loop; judge calls go through llmCall directly with their
+// own opts (max_tokens=3000, thinking=false per amendment v2).
+const RUNTIME_QWEN_OPTS = {
+  maxTokens: DEFAULT_QWEN_MAX_TOKENS,
+  thinking: DEFAULT_QWEN_THINKING_ON,
+};
+
+const JUDGES = ['claude-opus-4-7', 'gpt-5.4', 'minimax-m27-via-openrouter'] as const;
+
+const MODEL_PRICING: Record<string, { in: number; out: number }> = {
+  'claude-opus-4-7':                     { in: 15.0,   out: 75.0 },
+  'qwen3.6-35b-a3b-via-openrouter':      { in: 0.6,    out: 2.4 },
+  'gpt-5.4':                             { in: 2.5,    out: 10.0 },
+  'minimax-m27-via-openrouter':          { in: 0.7,    out: 2.8 },
+};
+
+// ─────────────────────────────────────────────────────────────────────────
+// Types
+// ─────────────────────────────────────────────────────────────────────────
+
+interface TaskMaterials {
+  taskId: string;
+  rawText: string;
+  persona: string;
+  question: string;
+  materialsConcat: string;
+  materialFrames: { title: string; body: string }[];
+}
+
+interface CellResult {
+  taskId: string;
+  cellId: 'A' | 'B' | 'C' | 'D';
+  model: string;
+  configuration: 'solo' | 'memory-harness';
+  candidateResponse: string;
+  candidateLatencyMs: number;
+  candidateTokensIn: number;
+  candidateTokensOut: number;
+  candidateCostUsd: number;
+  loopExhausted: boolean;
+  stepsTaken: number;
+  retrievalCalls: number;
+  errors: string[];
+}
+
+interface JudgeVerdict {
+  completeness: number;
+  accuracy: number;
+  synthesis: number;
+  judgment: number;
+  actionability: number;
+  structure: number;
+  rationale: string;
+  overall_verdict: string;
+  mean: number;
+}
+
+interface JudgeRecord extends JudgeVerdict {
+  judge_model: string;
+  judge_cost_usd: number;
+  judge_latency_ms: number;
+  judge_retries: number;
+}
+
+interface CellJsonlRecord {
+  task_id: string;
+  cell_id: 'A' | 'B' | 'C' | 'D';
+  model: string;
+  configuration: 'solo' | 'memory-harness';
+  candidate_response: string;
+  candidate_latency_ms: number;
+  candidate_tokens_in: number;
+  candidate_tokens_out: number;
+  candidate_cost_usd: number;
+  loop_exhausted: boolean;
+  steps_taken: number;
+  retrieval_calls: number;
+  judge_opus: JudgeVerdict;
+  judge_gpt: JudgeVerdict;
+  judge_minimax: JudgeVerdict;
+  trio_mean: number;
+  trio_strict_pass: boolean;
+  trio_critical_fail: boolean;
+  manifest_anchor: string;
+  head_sha: string;
+  ts_iso: string;
+  cell_cost_usd: number;
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// Logging
+// ─────────────────────────────────────────────────────────────────────────
+
+function logLine(msg: string): void {
+  const line = `[${new Date().toISOString()}] ${msg}\n`;
+  try { fs.appendFileSync(RUN_LOG_PATH, line); } catch { /* dir not yet created */ }
+  process.stderr.write(line);
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// CLI
+// ─────────────────────────────────────────────────────────────────────────
+
+interface Args {
+  smoke: boolean;
+  allTasks: boolean;
+  allCells: boolean;
+  task?: string;
+  cell?: 'A' | 'B' | 'C' | 'D';
+  dryRun: boolean;
+  help: boolean;
+  // Amendment v2 §7 flags
+  qwenAlias: string;
+  qwenMaxTokens: number;
+  qwenThinking: boolean;
+  retryCellAMinimax: boolean;
+  restartCells?: string;  // e.g. "task-1-C,task-1-D"
+}
+
+function parseArgs(argv: string[]): Args {
+  const out: Args = {
+    smoke: false, allTasks: false, allCells: false, dryRun: false, help: false,
+    qwenAlias: DEFAULT_QWEN_ALIAS,
+    qwenMaxTokens: DEFAULT_QWEN_MAX_TOKENS,
+    qwenThinking: DEFAULT_QWEN_THINKING_ON,
+    retryCellAMinimax: false,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const flag = argv[i];
+    const next = argv[i + 1];
+    switch (flag) {
+      case '--smoke': out.smoke = true; break;
+      case '--all-tasks': out.allTasks = true; break;
+      case '--all-cells': out.allCells = true; break;
+      case '--task': out.task = next; i++; break;
+      case '--cell': out.cell = next as Args['cell']; i++; break;
+      case '--dry-run': out.dryRun = true; break;
+      case '--qwen-alias': out.qwenAlias = next; i++; break;
+      case '--qwen-max-tokens': out.qwenMaxTokens = Number(next); i++; break;
+      case '--qwen-thinking': out.qwenThinking = (next ?? '').toLowerCase() !== 'off'; i++; break;
+      case '--retry-cell-a-minimax': out.retryCellAMinimax = true; break;
+      case '--restart-cells': out.restartCells = next; i++; break;
+      case '--help': case '-h': out.help = true; break;
+    }
+  }
+  return out;
+}
+
+function printHelp(): void {
+  console.log(`
+Agentic Knowledge Work Pilot — N=3 direction validator
+
+Flags:
+  --smoke              Run Task 1 only, all 4 cells (A/B/C/D). HALT after.
+  --all-tasks          Run tasks 1, 2, 3
+  --all-cells          Run cells A, B, C, D for the selected task(s)
+  --task <id>          Run a specific task (task-1, task-2, task-3)
+  --cell <id>          Run a specific cell (A, B, C, D)
+  --dry-run            Skip API calls; verify parsing + scaffolding only
+
+Amendment v2 §7 flags:
+  --qwen-alias <a>     Override Qwen alias (default: ${DEFAULT_QWEN_ALIAS})
+  --qwen-max-tokens N  Override Qwen max_tokens (default: ${DEFAULT_QWEN_MAX_TOKENS})
+  --qwen-thinking on|off  Explicit Qwen thinking flag (default: on)
+  --retry-cell-a-minimax  Surgical MiniMax retry against existing Cell A response
+  --restart-cells <list>  Comma list e.g. "task-1-C,task-1-D" — invalidate + re-run
+
+  -h, --help           This text
+`);
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// Pre-flight + audit-trail SHAs
+// ─────────────────────────────────────────────────────────────────────────
+
+function sha256File(filepath: string): string {
+  const buf = fs.readFileSync(filepath);
+  return crypto.createHash('sha256').update(buf).digest('hex');
+}
+
+function gitHead(): string {
+  try {
+    return execFileSync('git', ['rev-parse', 'HEAD'], { cwd: REPO_ROOT }).toString().trim();
+  } catch {
+    return 'UNKNOWN';
+  }
+}
+
+function preflight(): { headSha: string; amendmentSha: string; briefSha: string; rubricSha: string } {
+  return {
+    headSha: gitHead(),
+    amendmentSha: sha256File(AMENDMENT_PATH),
+    briefSha: sha256File(CC1_BRIEF_PATH),
+    rubricSha: sha256File(JUDGE_RUBRIC_PATH),
+  };
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// Task materials loader
+// ─────────────────────────────────────────────────────────────────────────
+
+function loadTaskMaterials(taskId: string): TaskMaterials {
+  const filepath = TASK_FILES[taskId];
+  if (!filepath) throw new Error(`unknown task: ${taskId}`);
+  const raw = fs.readFileSync(filepath, 'utf-8');
+  const endIdx = raw.indexOf('## End of materials');
+  if (endIdx < 0) throw new Error(`task ${taskId}: no '## End of materials' delimiter`);
+  const stripped = raw.slice(0, endIdx).trimEnd();
+
+  const personaMatch = stripped.match(/\*\*Persona:\*\*\s*([\s\S]*?)\n\n\*\*Scenario:\*\*/);
+  const scenarioMatch = stripped.match(/\*\*Scenario:\*\*\s*([\s\S]*?)\n\n\*\*Question to answer:\*\*/);
+  const questionMatch = stripped.match(/\*\*Question to answer:\*\*\s*\n>?\s*([\s\S]*?)(?:\n\n|\n\*\*Materials)/);
+
+  const persona = personaMatch ? personaMatch[1].trim() : '';
+  const scenario = scenarioMatch ? scenarioMatch[1].trim() : '';
+  const question = questionMatch ? questionMatch[1].replace(/^"|"$/g, '').trim() : '';
+
+  const sectionRegex = /^##\s+(DOC|THREAD|MEMO)\s+(\d+)\s*[—-]?\s*(.*?)$/gm;
+  const materialFrames: { title: string; body: string }[] = [];
+  const matches = [...stripped.matchAll(sectionRegex)];
+  for (let i = 0; i < matches.length; i++) {
+    const m = matches[i];
+    const title = `${m[1]} ${m[2]}${m[3] ? ' — ' + m[3] : ''}`;
+    const start = (m.index ?? 0) + m[0].length;
+    const end = i + 1 < matches.length ? (matches[i + 1].index ?? stripped.length) : stripped.length;
+    const body = stripped.slice(start, end).trim();
+    materialFrames.push({ title, body });
+  }
+
+  const materialsConcat = materialFrames.map(f => `## ${f.title}\n\n${f.body}`).join('\n\n---\n\n');
+
+  return {
+    taskId,
+    rawText: stripped,
+    persona: `Persona: ${persona}\n\nScenario: ${scenario}`,
+    question,
+    materialsConcat,
+    materialFrames,
+  };
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// LiteLLM HTTP client with cost tracking
+// ─────────────────────────────────────────────────────────────────────────
+
+// LlmCallFn adapter — Phase 2.2 refactor. Conforms to @waggle/agent's
+// LlmCallFn signature so the agent loop can call it directly. Handles
+// per-model accommodations (Opus temp=1.0, GPT/MiniMax omit temperature,
+// Qwen extra_body.enable_thinking explicit per amendment v2 §2).
+//
+// For Qwen subject calls, applies RUNTIME_QWEN_OPTS (set from CLI flags)
+// when the caller doesn't override. Judge callers pass thinking=false and
+// maxTokens=3000 explicitly.
+const llmCall: LlmCallFn = async (input: LlmCallInput): Promise<AgentLlmCallResult> => {
+  const masterKey = process.env.LITELLM_MASTER_KEY;
+  if (!masterKey) throw new Error('LITELLM_MASTER_KEY env not set');
+
+  const { model, messages } = input;
+  const isQwen = model.includes('qwen');
+  const maxTokens = input.maxTokens ?? (isQwen ? RUNTIME_QWEN_OPTS.maxTokens : 4096);
+  const thinking = input.thinking ?? (isQwen ? RUNTIME_QWEN_OPTS.thinking : true);
+  const temperature = input.temperature ?? 0.3;
+
+  const payload: Record<string, unknown> = { model, messages, max_tokens: maxTokens };
+
+  if (model.startsWith('claude-opus')) {
+    payload.temperature = 1.0;
+  } else if (model === 'gpt-5.4' || model === 'minimax-m27-via-openrouter') {
+    // omit temperature — reasoning model defaults
+  } else {
+    payload.temperature = temperature;
+  }
+
+  // Amendment v2 §2: ALWAYS pass enable_thinking explicitly for Qwen — do not rely on default.
+  if (isQwen) {
+    payload.extra_body = { enable_thinking: thinking };
+  }
+
+  const started = Date.now();
+  let lastErr: string | undefined;
+  for (let attempt = 0; attempt < 2; attempt++) {
+    try {
+      const resp = await fetch(`${LITELLM_URL}/chat/completions`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${masterKey}` },
+        body: JSON.stringify(payload),
+      });
+      const d: any = await resp.json();
+      if ('error' in d) {
+        lastErr = String(d.error?.message ?? JSON.stringify(d.error)).slice(0, 200);
+        if (attempt < 1) await new Promise(r => setTimeout(r, 1500));
+        continue;
+      }
+      const content = d.choices?.[0]?.message?.content ?? '';
+      const usage = d.usage ?? {};
+      const inTok = usage.prompt_tokens ?? 0;
+      const outTok = usage.completion_tokens ?? 0;
+      const pricing = MODEL_PRICING[model] ?? { in: 1, out: 4 };
+      const costUsd = (inTok * pricing.in + outTok * pricing.out) / 1_000_000;
+      return { content, inTokens: inTok, outTokens: outTok, costUsd, latencyMs: Date.now() - started };
+    } catch (e) {
+      lastErr = `${(e as Error).name}: ${(e as Error).message}`.slice(0, 200);
+      if (attempt < 1) await new Promise(r => setTimeout(r, 1500));
+      continue;
+    }
+  }
+  return {
+    content: '', inTokens: 0, outTokens: 0, costUsd: 0,
+    latencyMs: Date.now() - started,
+    error: lastErr ?? 'unknown error',
+  };
+};
+
+// ─────────────────────────────────────────────────────────────────────────
+// Cell A/C: solo single-shot
+// ─────────────────────────────────────────────────────────────────────────
+
+async function runCellSolo(cell: 'A' | 'C', task: TaskMaterials): Promise<CellResult> {
+  const config = CELLS[cell];
+  logLine(`[cell ${task.taskId}/${cell}] solo call → ${config.model} (via @waggle/agent runSoloAgent)`);
+
+  // Delegate to the unified agent loop from packages/agent (Phase 2.1 a599a07).
+  // Prompt assembly + per-model framing is handled by the prompt-shape selected
+  // for config.model. Pilot wrapper provides only the LlmCallFn adapter.
+  const result: AgentRunResult = await runSoloAgent({
+    modelAlias: config.model,
+    persona: task.persona,
+    question: task.question,
+    materials: task.materialsConcat,
+    llmCall,
+    contextTag: `${task.taskId}/${cell}`,
+    // No normalization-side schema change — keep raw response in JSONL for
+    // backwards compat with original pilot artifacts.
+  });
+
+  return {
+    taskId: task.taskId,
+    cellId: cell,
+    model: config.model,
+    configuration: 'solo',
+    candidateResponse: result.rawResponse,
+    candidateLatencyMs: result.totalLatencyMs,
+    candidateTokensIn: result.totalTokensIn,
+    candidateTokensOut: result.totalTokensOut,
+    candidateCostUsd: result.totalCostUsd,
+    loopExhausted: result.loopExhausted,
+    stepsTaken: result.stepsTaken,
+    retrievalCalls: result.retrievalCalls,
+    errors: [...result.errors],
+  };
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// Cell B/D: multi-step retrieval-augmented loop
+// ─────────────────────────────────────────────────────────────────────────
+
+async function runCellMultiStep(cell: 'B' | 'D', task: TaskMaterials, embedder: Embedder): Promise<CellResult> {
+  const config = CELLS[cell];
+
+  // Per-cell SessionStore + HybridSearch setup — this scaffolding stays in
+  // the pilot wrapper because per-task corpus isolation is pilot-specific.
+  const dbPath = path.join(SCRATCH_DIR, `per-task-${task.taskId}-cell-${cell}.sqlite`);
+  if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
+  const db = new MindDB(dbPath);
+  const frames = new FrameStore(db);
+  const sessions = new SessionStore(db);
+  const hybrid = new HybridSearch(db, embedder);
+
+  const gopId = `${PILOT_ID}-${task.taskId}-${cell}`;
+  sessions.ensure(gopId, undefined, `Pilot session for ${task.taskId} cell ${cell}`);
+
+  for (const m of task.materialFrames) {
+    const content = `## ${m.title}\n\n${m.body}`;
+    frames.createIFrame(gopId, content, 'important', 'system');
+  }
+  logLine(`[cell ${task.taskId}/${cell}] ingested ${task.materialFrames.length} frames into ${dbPath}`);
+
+  // RetrievalSearchFn adapter — wraps HybridSearch.search for the agent loop.
+  // The agent loop calls this via config.search; the adapter formats the hits
+  // into a single string for prompt injection.
+  const searchAdapter: RetrievalSearchFn = async ({ query, limit }) => {
+    const hits = await hybrid.search(query, { limit, gopId });
+    const formatted = hits.length > 0
+      ? hits.map((sr, i) => `[result ${i + 1}, score ${sr.finalScore.toFixed(3)}]\n${sr.frame.content}`).join('\n\n---\n\n')
+      : '';
+    return { formattedResults: formatted, resultCount: hits.length };
+  };
+
+  // Delegate to the unified agent loop from packages/agent (Phase 2.1 a599a07).
+  // Per-call halt + per-cell halt + MAX_STEPS + force-finalize all enforced inside.
+  const result: AgentRunResult = await runRetrievalAgentLoop({
+    modelAlias: config.model,
+    persona: task.persona,
+    question: task.question,
+    llmCall,
+    search: searchAdapter,
+    maxSteps: MAX_STEPS,
+    maxRetrievalsPerStep: MAX_RETRIEVALS_PER_STEP,
+    perCallHaltUsd: PER_CALL_SANITY_USD,
+    perCellHaltUsd: PER_CELL_HARD_HALT_USD,
+    contextTag: `${task.taskId}/${cell}`,
+  });
+
+  // MindDB does not expose a public close() — let GC reclaim. The sqlite file
+  // remains on disk in tmp/ for post-run inspection (gitignored).
+  return {
+    taskId: task.taskId,
+    cellId: cell,
+    model: config.model,
+    configuration: 'memory-harness',
+    candidateResponse: result.rawResponse,
+    candidateLatencyMs: result.totalLatencyMs,
+    candidateTokensIn: result.totalTokensIn,
+    candidateTokensOut: result.totalTokensOut,
+    candidateCostUsd: result.totalCostUsd,
+    loopExhausted: result.loopExhausted,
+    stepsTaken: result.stepsTaken,
+    retrievalCalls: result.retrievalCalls,
+    errors: [...result.errors],
+  };
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// Trio judging
+// ─────────────────────────────────────────────────────────────────────────
+
+const JUDGE_PROMPT_TEMPLATE = `You are evaluating an AI agent's response to a complex knowledge work task. The persona, scenario, materials, and question are provided. The response was generated under one of four configurations (revealed only after scoring): {model_only | model + memory + agent harness} × {Opus 4.7 | Qwen 3.6 35B-A3B}.
+
+You do NOT know which configuration produced this response. Score blind.
+
+Read the persona/scenario/question (provided), skim the materials (provided), then read the response carefully (provided).
+
+Score the response on six dimensions, Likert 1-5:
+
+1. COMPLETENESS — engagement with all material
+2. ACCURACY — faithfulness to source materials, no hallucinations
+3. SYNTHESIS — connections across inputs, not isolated treatment
+4. JUDGMENT — defensible recommendations, tradeoffs acknowledged
+5. ACTIONABILITY — would the persona act on this tomorrow
+6. STRUCTURE — organization and readability
+
+Output JSON only:
+{
+  "completeness": <1-5>,
+  "accuracy": <1-5>,
+  "synthesis": <1-5>,
+  "judgment": <1-5>,
+  "actionability": <1-5>,
+  "structure": <1-5>,
+  "rationale": "<1-2 sentences explaining the lowest scoring dimension>",
+  "overall_verdict": "<one of: PASS_STRONG | PASS_ADEQUATE | FAIL_WEAK | FAIL_CRITICAL>"
+}
+
+PASS_STRONG: mean >= 4.0
+PASS_ADEQUATE: mean 3.5-3.99
+FAIL_WEAK: mean 2.5-3.49
+FAIL_CRITICAL: mean < 2.5
+
+[PERSONA + SCENARIO + QUESTION]
+###PERSONA_SCENARIO_QUESTION###
+
+[MATERIALS]
+###MATERIALS###
+
+[RESPONSE TO EVALUATE]
+###RESPONSE###`;
+
+function buildJudgePrompt(task: TaskMaterials, response: string): string {
+  return JUDGE_PROMPT_TEMPLATE
+    .replace('###PERSONA_SCENARIO_QUESTION###', `${task.persona}\n\nQUESTION: ${task.question}`)
+    .replace('###MATERIALS###', task.materialsConcat)
+    .replace('###RESPONSE###', response);
+}
+
+function parseJudgeJson(text: string): JudgeVerdict | null {
+  const m = text.match(/\{[\s\S]*\}/);
+  if (!m) return null;
+  try {
+    const obj = JSON.parse(m[0]);
+    const dims = ['completeness', 'accuracy', 'synthesis', 'judgment', 'actionability', 'structure'] as const;
+    for (const d of dims) {
+      if (typeof obj[d] !== 'number' || obj[d] < 1 || obj[d] > 5) return null;
+    }
+    const mean = dims.reduce((s, d) => s + obj[d], 0) / dims.length;
+    return {
+      completeness: obj.completeness, accuracy: obj.accuracy, synthesis: obj.synthesis,
+      judgment: obj.judgment, actionability: obj.actionability, structure: obj.structure,
+      rationale: typeof obj.rationale === 'string' ? obj.rationale : '',
+      overall_verdict: typeof obj.overall_verdict === 'string' ? obj.overall_verdict : '',
+      mean,
+    };
+  } catch {
+    return null;
+  }
+}
+
+const ZERO_VERDICT: JudgeVerdict = {
+  completeness: 0, accuracy: 0, synthesis: 0, judgment: 0, actionability: 0, structure: 0,
+  rationale: '__JUDGE_FAILED__', overall_verdict: 'FAIL_CRITICAL', mean: 0,
+};
+
+async function runJudge(judgeModel: string, prompt: string): Promise<JudgeRecord> {
+  let lastError = '';
+  let totalCost = 0, totalLatency = 0;
+  // Amendment v2 PM-decision (post second-smoke): max_tokens 1024 → 3000 to address
+  // MiniMax solo-cell failure pattern (dense memo responses likely overflowed 1024 mid-JSON).
+  for (let attempt = 0; attempt < MAX_JUDGE_RETRIES; attempt++) {
+    const r = await llmCall({ model: judgeModel, messages: [{ role: 'user', content: prompt }], maxTokens: 3000, thinking: false });
+    totalCost += r.costUsd; totalLatency += r.latencyMs;
+    if (r.error) { lastError = `attempt ${attempt + 1}: ${r.error}`; continue; }
+    const parsed = parseJudgeJson(r.content);
+    if (parsed) {
+      return {
+        ...parsed,
+        judge_model: judgeModel, judge_cost_usd: totalCost,
+        judge_latency_ms: totalLatency, judge_retries: attempt,
+      };
+    }
+    lastError = `attempt ${attempt + 1}: malformed JSON: ${r.content.slice(0, 100)}`;
+  }
+  logLine(`[judge ${judgeModel}] FAILED after ${MAX_JUDGE_RETRIES}: ${lastError}`);
+  return {
+    ...ZERO_VERDICT,
+    judge_model: judgeModel, judge_cost_usd: totalCost,
+    judge_latency_ms: totalLatency, judge_retries: MAX_JUDGE_RETRIES,
+    rationale: `__JUDGE_FAILED__: ${lastError}`,
+  };
+}
+
+async function judgeWithTrio(task: TaskMaterials, response: string): Promise<{
+  records: JudgeRecord[]; trioMean: number; strictPass: boolean; criticalFail: boolean; cost: number;
+}> {
+  const prompt = buildJudgePrompt(task, response);
+  const records = await Promise.all(JUDGES.map(j => runJudge(j, prompt)));
+  const validMeans = records.filter(r => r.mean > 0).map(r => r.mean);
+  const trioMean = validMeans.length === 3
+    ? validMeans.reduce((s, m) => s + m, 0) / 3
+    : validMeans.length > 0 ? validMeans.reduce((s, m) => s + m, 0) / validMeans.length : 0;
+  const strictPass = records.filter(r => r.mean >= 3.5).length >= 2;
+  const criticalFail = records.filter(r => r.mean < 2.0 && r.mean > 0).length >= 2;
+  const cost = records.reduce((s, r) => s + r.judge_cost_usd, 0);
+  return { records, trioMean, strictPass, criticalFail, cost };
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// Output writers
+// ─────────────────────────────────────────────────────────────────────────
+
+function stripJudge(j: JudgeRecord): JudgeVerdict {
+  return {
+    completeness: j.completeness, accuracy: j.accuracy, synthesis: j.synthesis,
+    judgment: j.judgment, actionability: j.actionability, structure: j.structure,
+    rationale: j.rationale, overall_verdict: j.overall_verdict, mean: j.mean,
+  };
+}
+
+function findJudge(records: JudgeRecord[], model: string): JudgeRecord {
+  const r = records.find(x => x.judge_model === model);
+  if (!r) throw new Error(`judge record missing for ${model}`);
+  return r;
+}
+
+function writeCellJsonl(
+  cell: CellResult,
+  judges: { records: JudgeRecord[]; trioMean: number; strictPass: boolean; criticalFail: boolean; cost: number },
+  audit: { headSha: string }
+): CellJsonlRecord {
+  const rec: CellJsonlRecord = {
+    task_id: cell.taskId,
+    cell_id: cell.cellId,
+    model: cell.model,
+    configuration: cell.configuration,
+    candidate_response: cell.candidateResponse,
+    candidate_latency_ms: cell.candidateLatencyMs,
+    candidate_tokens_in: cell.candidateTokensIn,
+    candidate_tokens_out: cell.candidateTokensOut,
+    candidate_cost_usd: cell.candidateCostUsd,
+    loop_exhausted: cell.loopExhausted,
+    steps_taken: cell.stepsTaken,
+    retrieval_calls: cell.retrievalCalls,
+    judge_opus: stripJudge(findJudge(judges.records, 'claude-opus-4-7')),
+    judge_gpt: stripJudge(findJudge(judges.records, 'gpt-5.4')),
+    judge_minimax: stripJudge(findJudge(judges.records, 'minimax-m27-via-openrouter')),
+    trio_mean: judges.trioMean,
+    trio_strict_pass: judges.strictPass,
+    trio_critical_fail: judges.criticalFail,
+    manifest_anchor: MANIFEST_ANCHOR,
+    head_sha: audit.headSha,
+    ts_iso: new Date().toISOString(),
+    cell_cost_usd: cell.candidateCostUsd + judges.cost,
+  };
+  const outPath = path.join(OUT_DIR, `pilot-${cell.taskId}-${cell.cellId}.jsonl`);
+  fs.writeFileSync(outPath, JSON.stringify(rec) + '\n', 'utf-8');
+  logLine(`[cell ${cell.taskId}/${cell.cellId}] wrote ${path.basename(outPath)} trio_mean=${judges.trioMean.toFixed(2)} strict=${judges.strictPass} critical=${judges.criticalFail} cell_cost=$${rec.cell_cost_usd.toFixed(4)}`);
+  return rec;
+}
+
+function writeSummary(records: CellJsonlRecord[], cumulativeCost: number, startTs: string, endTs: string): void {
+  const byTask: Record<string, Record<string, number>> = {};
+  let criticalFailures = 0;
+  for (const r of records) {
+    byTask[r.task_id] = byTask[r.task_id] ?? {};
+    byTask[r.task_id][`cell_${r.cell_id}_trio_mean`] = r.trio_mean;
+    if (r.trio_critical_fail) criticalFailures += 1;
+  }
+  const perTask: Record<string, unknown> = {};
+  let h2Pass = 0, h3Pass = 0, h4Pass = 0;
+  for (const tid of Object.keys(byTask)) {
+    const t = byTask[tid];
+    const a = t.cell_A_trio_mean ?? 0;
+    const b = t.cell_B_trio_mean ?? 0;
+    const c = t.cell_C_trio_mean ?? 0;
+    const d = t.cell_D_trio_mean ?? 0;
+    const h2 = b - a, h3 = d - c, h4 = d - a;
+    const h2Dir = h2 >= 0.30, h3Dir = h3 >= 0.30, h4Dir = d >= a;
+    if (h2Dir) h2Pass += 1; if (h3Dir) h3Pass += 1; if (h4Dir) h4Pass += 1;
+    perTask[tid] = {
+      cell_A_trio_mean: a, cell_B_trio_mean: b, cell_C_trio_mean: c, cell_D_trio_mean: d,
+      h2_delta_opus: +h2.toFixed(4), h3_delta_qwen: +h3.toFixed(4), h4_delta_sovereignty: +h4.toFixed(4),
+      h2_directional_pass: h2Dir, h3_directional_pass: h3Dir, h4_directional_pass: h4Dir,
+    };
+  }
+  const verdict = (h2Pass >= 2 && h3Pass >= 2 && h4Pass >= 2 && criticalFailures === 0) ? 'PASS' : 'FAIL';
+  const summary = {
+    pilot_id: PILOT_ID,
+    manifest_anchor: MANIFEST_ANCHOR,
+    execution_window_utc: `${startTs} to ${endTs}`,
+    total_cost_usd: +cumulativeCost.toFixed(6),
+    total_judge_calls: records.length * 3,
+    total_candidate_calls: records.length,
+    n_cells: records.length,
+    results_per_task: perTask,
+    aggregate: {
+      h2_pass_count: h2Pass,
+      h3_pass_count: h3Pass,
+      h4_pass_count: h4Pass,
+      critical_failures: criticalFailures,
+      pilot_verdict: verdict,
+    },
+  };
+  fs.writeFileSync(path.join(OUT_DIR, 'pilot-summary.json'), JSON.stringify(summary, null, 2), 'utf-8');
+  logLine(`[summary] verdict=${verdict} h2=${h2Pass}/3 h3=${h3Pass}/3 h4=${h4Pass}/3 critical=${criticalFailures} cost=$${cumulativeCost.toFixed(4)}`);
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// Amendment v2 §3.1 — Cell A MiniMax surgical retry
+// ─────────────────────────────────────────────────────────────────────────
+
+async function retryCellAMinimax(audit: { headSha: string }): Promise<void> {
+  const jsonlPath = path.join(OUT_DIR, 'pilot-task-1-A.jsonl');
+  if (!fs.existsSync(jsonlPath)) {
+    logLine(`[retry-minimax] FAIL: ${jsonlPath} not found — Cell A must exist first`);
+    return;
+  }
+  const rec: CellJsonlRecord = JSON.parse(fs.readFileSync(jsonlPath, 'utf-8').trim());
+  const task = loadTaskMaterials('task-1');
+  const prompt = buildJudgePrompt(task, rec.candidate_response);
+  logLine(`[retry-minimax] Cell A — calling minimax-m27-via-openrouter against existing candidate (${rec.candidate_response.length}c)`);
+  const newJudge = await runJudge('minimax-m27-via-openrouter', prompt);
+  if (newJudge.mean > 0) {
+    rec.judge_minimax = stripJudge(newJudge);
+    const validMeans = [rec.judge_opus.mean, rec.judge_gpt.mean, rec.judge_minimax.mean].filter(m => m > 0);
+    rec.trio_mean = validMeans.length > 0 ? validMeans.reduce((s, m) => s + m, 0) / validMeans.length : 0;
+    const strictCount = [rec.judge_opus, rec.judge_gpt, rec.judge_minimax].filter(j => j.mean >= 3.5).length;
+    const criticalCount = [rec.judge_opus, rec.judge_gpt, rec.judge_minimax].filter(j => j.mean < 2.0 && j.mean > 0).length;
+    rec.trio_strict_pass = strictCount >= 2;
+    rec.trio_critical_fail = criticalCount >= 2;
+    (rec as unknown as Record<string, unknown>).judge_minimax_retried_at = new Date().toISOString();
+    rec.cell_cost_usd += newJudge.judge_cost_usd;
+    rec.head_sha = audit.headSha;
+    fs.writeFileSync(jsonlPath, JSON.stringify(rec) + '\n', 'utf-8');
+    logLine(`[retry-minimax] SUCCESS — Cell A judge_minimax=${newJudge.mean.toFixed(2)} new trio_mean=${rec.trio_mean.toFixed(3)} cost=$${newJudge.judge_cost_usd.toFixed(4)}`);
+  } else {
+    logLine(`[retry-minimax] FAIL again — Cell A retains 2-judge fallback. cost=$${newJudge.judge_cost_usd.toFixed(4)}`);
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// Amendment v2 §3 — Restart cells (invalidate + re-run)
+// ─────────────────────────────────────────────────────────────────────────
+
+async function restartCells(
+  cellList: string,
+  embedder: Embedder,
+  audit: { headSha: string }
+): Promise<{ records: CellJsonlRecord[]; cost: number }> {
+  const invalidatedDir = path.join(OUT_DIR, 'invalidated');
+  fs.mkdirSync(invalidatedDir, { recursive: true });
+
+  const targets = cellList.split(',').map(s => s.trim()).filter(Boolean);
+  let totalCost = 0;
+  const records: CellJsonlRecord[] = [];
+  for (const target of targets) {
+    const m = target.match(/^(task-\d+)-([ABCD])$/);
+    if (!m) {
+      logLine(`[restart] skip malformed target: ${target}`);
+      continue;
+    }
+    const taskId = m[1];
+    const cellId = m[2] as 'A' | 'B' | 'C' | 'D';
+    const original = path.join(OUT_DIR, `pilot-${taskId}-${cellId}.jsonl`);
+    const dest = path.join(invalidatedDir, `pilot-${taskId}-${cellId}.invalidated-${new Date().toISOString().replace(/[:.]/g, '-')}.jsonl`);
+    if (fs.existsSync(original)) {
+      fs.renameSync(original, dest);
+      logLine(`[restart] moved original to ${path.basename(dest)}`);
+    }
+    const task = loadTaskMaterials(taskId);
+    const cellResult = (cellId === 'A' || cellId === 'C')
+      ? await runCellSolo(cellId, task)
+      : await runCellMultiStep(cellId, task, embedder);
+    const judges = await judgeWithTrio(task, cellResult.candidateResponse);
+    totalCost += cellResult.candidateCostUsd + judges.cost;
+    const rec = writeCellJsonl(cellResult, judges, { headSha: audit.headSha });
+    records.push(rec);
+  }
+  return { records, cost: totalCost };
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// Main
+// ─────────────────────────────────────────────────────────────────────────
+
+async function main(): Promise<void> {
+  const args = parseArgs(process.argv.slice(2));
+  if (args.help) { printHelp(); return; }
+
+  fs.mkdirSync(OUT_DIR, { recursive: true });
+  fs.mkdirSync(PROMPTS_ARCHIVE_DIR, { recursive: true });
+  fs.mkdirSync(SCRATCH_DIR, { recursive: true });
+
+  // Amendment v2 §7 (PM-revised post second-smoke): always append to run log; only
+  // create on first run. JSONLs are atomic per-cell so log loss is recoverable, but
+  // multi-kick pilot requires log continuity across phases (smoke → restart → tasks 2+3).
+  if (!fs.existsSync(RUN_LOG_PATH)) fs.writeFileSync(RUN_LOG_PATH, '');
+
+  // Apply amendment v2 §2 Qwen config (or CLI overrides).
+  RUNTIME_QWEN_OPTS.maxTokens = args.qwenMaxTokens;
+  RUNTIME_QWEN_OPTS.thinking = args.qwenThinking;
+  CELLS.C.model = args.qwenAlias;
+  CELLS.D.model = args.qwenAlias;
+
+  const startTs = new Date().toISOString();
+  const audit = preflight();
+  // Amendment v2 §8: capture v2 SHA in addition to v1 + brief + rubric + HEAD.
+  const amendmentV2Path = path.join(BRIEF_DIR, 'cc1-brief-amendment-v2-2026-04-26.md');
+  const amendmentV2Sha = fs.existsSync(amendmentV2Path) ? sha256File(amendmentV2Path) : 'NOT_PRESENT';
+  logLine(`[pilot] amendment_v2_doc_sha256 = ${amendmentV2Sha}`);
+  logLine(`[pilot] amendment_v1_doc_sha256 = ${audit.amendmentSha}`);
+  logLine(`[pilot] cc1_brief_sha256 = ${audit.briefSha}`);
+  logLine(`[pilot] judge_rubric_sha256 = ${audit.rubricSha}`);
+  logLine(`[pilot] head_sha = ${audit.headSha}`);
+  logLine(`[pilot] manifest_anchor = ${MANIFEST_ANCHOR}`);
+  logLine(`[pilot] cost_cap = $${COST_CAP_USD}, halt = $${COST_HALT_USD}, per_cell_halt = $${PER_CELL_HARD_HALT_USD}`);
+  logLine(`[pilot] qwen_alias = ${args.qwenAlias}`);
+  logLine(`[pilot] qwen_max_tokens = ${args.qwenMaxTokens}`);
+  logLine(`[pilot] qwen_thinking = ${args.qwenThinking ? 'on' : 'off'}`);
+
+  // Amendment v2 §7 partial-run paths (no full pilot loop).
+  if (args.retryCellAMinimax || args.restartCells) {
+    const embedder = createOllamaEmbedder({ baseUrl: OLLAMA_URL, model: EMBEDDER_MODEL });
+    let partialCost = 0;
+    if (args.retryCellAMinimax) {
+      await retryCellAMinimax({ headSha: audit.headSha });
+    }
+    if (args.restartCells) {
+      const r = await restartCells(args.restartCells, embedder, { headSha: audit.headSha });
+      partialCost += r.cost;
+    }
+    logLine(`[partial-run] complete; partial_cost=$${partialCost.toFixed(4)}`);
+    // Re-emit summary from current JSONL set (covers retained + restarted records).
+    const allFiles = fs.readdirSync(OUT_DIR).filter(f => f.match(/^pilot-task-\d+-[ABCD]\.jsonl$/));
+    const allRecords: CellJsonlRecord[] = [];
+    for (const f of allFiles) {
+      const r = JSON.parse(fs.readFileSync(path.join(OUT_DIR, f), 'utf-8').trim());
+      allRecords.push(r);
+    }
+    const cumCost = allRecords.reduce((s, r) => s + r.cell_cost_usd, 0);
+    writeSummary(allRecords, cumCost, startTs, new Date().toISOString());
+    return;
+  }
+
+  let taskIds: string[];
+  if (args.smoke) taskIds = ['task-1'];
+  else if (args.allTasks) taskIds = ['task-1', 'task-2', 'task-3'];
+  else if (args.task) taskIds = [args.task];
+  else { console.error('Specify one of: --smoke, --all-tasks, --task <id>, --retry-cell-a-minimax, --restart-cells <list>'); process.exit(1); }
+
+  const cellIds: ('A' | 'B' | 'C' | 'D')[] = (args.allCells || args.smoke)
+    ? ['A', 'B', 'C', 'D']
+    : args.cell ? [args.cell] : ['A', 'B', 'C', 'D'];
+
+  if (args.dryRun) {
+    logLine(`[dry-run] would run ${taskIds.length} task(s) × ${cellIds.length} cell(s) = ${taskIds.length * cellIds.length} cells`);
+    for (const tid of taskIds) {
+      const t = loadTaskMaterials(tid);
+      logLine(`[dry-run] ${tid}: frames=${t.materialFrames.length} persona=${t.persona.length}c materials=${t.materialsConcat.length}c question=${t.question.length}c`);
+      for (const f of t.materialFrames) logLine(`  - frame: ${f.title} (${f.body.length}c)`);
+    }
+    return;
+  }
+
+  const embedder = createOllamaEmbedder({ baseUrl: OLLAMA_URL, model: EMBEDDER_MODEL });
+
+  let cumulativeCost = 0;
+  const allRecords: CellJsonlRecord[] = [];
+
+  for (const tid of taskIds) {
+    const task = loadTaskMaterials(tid);
+    logLine(`[task ${tid}] loaded ${task.materialFrames.length} frames`);
+    for (const cid of cellIds) {
+      if (cumulativeCost >= COST_HALT_USD) {
+        logLine(`[HALT] cumulative $${cumulativeCost.toFixed(4)} >= $${COST_HALT_USD}`);
+        break;
+      }
+      const cellResult = (cid === 'A' || cid === 'C')
+        ? await runCellSolo(cid, task)
+        : await runCellMultiStep(cid, task, embedder);
+      const judges = await judgeWithTrio(task, cellResult.candidateResponse);
+      cumulativeCost += cellResult.candidateCostUsd + judges.cost;
+      const rec = writeCellJsonl(cellResult, judges, { headSha: audit.headSha });
+      allRecords.push(rec);
+      logLine(`[cumulative] $${cumulativeCost.toFixed(4)} / $${COST_CAP_USD}`);
+    }
+  }
+
+  const endTs = new Date().toISOString();
+  writeSummary(allRecords, cumulativeCost, startTs, endTs);
+}
+
+main().catch((e) => {
+  console.error('FATAL:', e);
+  process.exit(2);
+});