#!/usr/bin/env tsx /** * Four-cell ablation harness — CLI entry. * * Usage (via `npm run bench -- ...` from repo root, or direct `tsx`): * * --cell raw Single cell * --cell filtered * --cell compressed * --cell full-context * --all-cells Run all four sequentially (same dataset/seed) * --control verbose-fixed Diagnostic control (not a cell) * * --dataset locomo locomo | longmemeval | synthetic * --limit N N instances. --full sets Infinity. * --full Alias for --limit Infinity. * --model qwen3.6-35b-a3b Must match an id in config/models.json * --seed N Default: 42 (reproducibility artifact req). * --budget USD Hard USD cap. Default: Infinity. * --output path.jsonl Default: ../results/--.jsonl * --dry-run Stub LLM calls. Default: true when * LITELLM_URL is not set. * * Env: * LITELLM_URL default http://localhost:4000 * LITELLM_API_KEY default sk-waggle-dev */ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import url from 'node:url'; import { generateTurnId, logTurnEvent } from '@waggle/agent'; import type { CellName, ControlName, DatasetSpec, JsonlRecord, ModelSpec, RunConfig, RunKind, } from './types.js'; import { getDatasetVersion, loadDataset, loadPreflightSampleLock, sampleInstances } from './datasets.js'; import { createLlmClient } from './llm.js'; import { CANONICAL_MANIFEST_PATH, computeBenchSpecManifestHash, emitPreregistrationManifest, getRunnerVersion, readManifestLockedDate, sanitizeArgv, type JudgeModelManifestEntry, type PreregistrationManifestPayload, } from './preregistration.js'; import { JsonlWriter, buildAggregate, scoreAccuracy, percentile } from './metrics.js'; import { cells, isCellName, isControlName } from './cells.js'; import { controls } from './controls.js'; import { createJudgeLlmClient, type JudgeClientCostEntry } from './judge-client.js'; import { runJudge, type JudgeConfig } from './judge-runner.js'; // Sprint 12 Task 2.5 Stage 1 (2026-04-23) — substrate hookup for retrieval + // agentic cells. Substrate is created once in main() when the cell roster // includes `retrieval` or `agentic`, pre-ingested from raw LoCoMo, then // threaded through RunConfig → CellInput to each iteration. import { createSubstrate, type Substrate } from './substrate.js'; import { extractTurnsFromLocomoRaw, ingestLoCoMoCorpus } from './ingest.js'; // Sprint 12 Task 2.5 Stage 1.5 §7.2 — consecutive fetch-transport failure // halt. Prevents the v2 full-context saturation cascade from burning the // full instance budget on dead calls. import { StreakTracker } from './streak-tracker.js'; // Sprint 12 Task 2.5 Stage 1.5 §7.3 — pre-cell health check. Probes every // upstream route the run depends on before spending instance-budget on them. import { preCellHealthCheck } from './health-check.js'; // Sprint 12 Task 2.5 Stage 1.5 §7.4 — single-runner PID+heartbeat lock. // Encodes `concurrent_runners: FORBIDDEN` at the process level. Uses a // single global sentinel under benchmarks/results/ so two harness processes // can never both run even against different datasets/models (they still // share the LiteLLM proxy + provider bridges — the v2 saturation root cause). import { acquireRunnerLock, type LockHandle } from './runner-lock.js'; // ── Arg parsing ──────────────────────────────────────────────────────────── interface ParsedArgs { cell?: string; allCells: boolean; control?: string; dataset: string; limit: number; model: string; seed: number; budget: number; output?: string; dryRun?: boolean; sampleLock?: string; /** When set, each cell/control call also triggers a judge call after * the model answer returns. Value is the single-judge model id * (e.g. `claude-sonnet-4-6`). Default: undefined → judging skipped. */ judge?: string; /** Comma-separated list of judge model ids for ensemble mode. First * id is the tie-breaker (Sonnet by taxonomy §6 convention). Takes * precedence over `--judge` when both are set. */ judgeEnsemble?: string[]; // ── Sprint 12 Task 1 Blocker #3 — pre-registration flags ───────────────── /** SHA-256 hex of the bench-spec manifest YAML. When omitted, the * emitter resolves the manifest path (env → sibling PM-Waggle-OS → throw) * and computes the hash at run start. When provided, skips the lookup * entirely — useful for tests and replication runs where the source * YAML is not available at the runtime cwd. */ manifestHash?: string; /** Suppresses `bench.preregistration.manifest_hash` pino-style event * when `false`. Default: true. Tests pass `--no-emit-preregistration-event` * to keep the log surface quiet. */ emitPreregistrationEvent: boolean; /** Multi-value cell selection (repeatable `--per-cell raw --per-cell filtered`). * Overrides `--cell` and `--all-cells` when at least one value is set. * Also materializes into the `per_cell` field of the pre-registration * manifest payload regardless of how cells were chosen (falls back to * derivation from --all-cells or --cell when unset). */ perCell?: string[]; /** Judge tie-break strategy name surfaced into the pre-registration * payload. Canonical A3 LOCK § B2 values: `quadri-vendor` (fourth * vendor fires on 1-1-1) and `pm-escalation` (2-2 escalates to PM). * `majority` is accepted as an explicit-default alias. */ judgeTiebreak?: string; // ── Sprint 12 Task 2.5 Stage 1 (2026-04-23) — substrate flags ──────────── /** Path to the raw LoCoMo archive (`locomo10.json`). Required when the cell * roster includes `retrieval` or `agentic`. Relative paths resolve against * the current working directory. */ locomoRawPath?: string; /** Retrieval top-K (default 10, GATE-S0 lock). Threaded into CellInput. */ retrievalTopK?: number; /** Agentic hard turn cap (default 3, GATE-S0 lock). */ agenticMaxTurns?: number; /** Agentic AbortController timeout in ms (default 180_000). */ agenticTimeoutMs?: number; } function parseArgs(argv: string[]): ParsedArgs { const out: ParsedArgs = { allCells: false, dataset: 'synthetic', limit: 10, model: 'qwen3.6-35b-a3b', seed: 42, budget: Number.POSITIVE_INFINITY, emitPreregistrationEvent: true, }; const VALID_TIEBREAK = new Set(['quadri-vendor', 'pm-escalation', 'majority']); for (let i = 0; i < argv.length; i++) { const flag = argv[i]; const next = argv[i + 1]; switch (flag) { case '--cell': out.cell = next; i++; break; case '--all-cells': out.allCells = true; break; case '--control': out.control = next; i++; break; case '--dataset': out.dataset = next; i++; break; case '--limit': out.limit = Number(next); i++; break; case '--full': out.limit = Number.POSITIVE_INFINITY; break; case '--model': out.model = next; i++; break; case '--seed': out.seed = Number(next); i++; break; case '--budget': out.budget = Number(next); i++; break; case '--output': out.output = next; i++; break; case '--dry-run': out.dryRun = true; break; case '--live': out.dryRun = false; break; case '--sample-lock': out.sampleLock = next; i++; break; case '--judge': out.judge = next; i++; break; case '--judge-ensemble': out.judgeEnsemble = (next ?? '').split(',').map(s => s.trim()).filter(Boolean); i++; break; // ── Sprint 12 Task 1 Blocker #3 flags ─────────────────────────── case '--manifest-hash': if (typeof next !== 'string' || !/^[0-9a-f]{64}$/i.test(next)) { throw new Error( `Invalid --manifest-hash value: expected 64-char lowercase SHA-256 hex, got ${next ?? '(missing)'}`, ); } out.manifestHash = next.toLowerCase(); i++; break; case '--emit-preregistration-event': out.emitPreregistrationEvent = true; break; case '--no-emit-preregistration-event': out.emitPreregistrationEvent = false; break; case '--per-cell': if (typeof next !== 'string' || next.length === 0) { throw new Error(`Invalid --per-cell value: expected cell name, got ${next ?? '(missing)'}`); } out.perCell = out.perCell ?? []; out.perCell.push(next); i++; break; case '--judge-tiebreak': if (typeof next !== 'string' || !VALID_TIEBREAK.has(next)) { throw new Error( `Invalid --judge-tiebreak value: expected one of ${[...VALID_TIEBREAK].join(' | ')}, got ${next ?? '(missing)'}`, ); } out.judgeTiebreak = next; i++; break; // ── Sprint 12 Task 2.5 Stage 1 substrate flags ─────────────────────── case '--locomo-raw-path': if (typeof next !== 'string' || next.length === 0) { throw new Error(`Invalid --locomo-raw-path value: expected path, got ${next ?? '(missing)'}`); } out.locomoRawPath = next; i++; break; case '--retrieval-top-k': { const n = Number(next); if (!Number.isInteger(n) || n <= 0 || n > 100) { throw new Error(`Invalid --retrieval-top-k value: expected 1..100 integer, got ${next ?? '(missing)'}`); } out.retrievalTopK = n; i++; break; } case '--agentic-max-turns': { const n = Number(next); if (!Number.isInteger(n) || n <= 0 || n > 10) { throw new Error(`Invalid --agentic-max-turns value: expected 1..10 integer, got ${next ?? '(missing)'}`); } out.agenticMaxTurns = n; i++; break; } case '--agentic-timeout-ms': { const n = Number(next); if (!Number.isFinite(n) || n < 1000) { throw new Error(`Invalid --agentic-timeout-ms value: expected ≥1000 ms, got ${next ?? '(missing)'}`); } out.agenticTimeoutMs = n; i++; break; } case '--help': case '-h': printHelp(); process.exit(0); } } return out; } function printHelp(): void { console.log(`Four-cell ablation harness. Usage: waggle-bench --cell --dataset --limit N --model waggle-bench --all-cells --dataset --limit N --model [--budget USD] waggle-bench --control verbose-fixed --dataset --limit N --model Flags: --cell Single cell name --all-cells Run all four cells sequentially --control Diagnostic control (e.g. verbose-fixed) --dataset locomo | longmemeval | synthetic (default synthetic) --limit N Instance count cap (default 10) --full --limit Infinity --model id Model id from config/models.json (default qwen3.6-35b-a3b) --seed N PRNG seed (default 42) --budget USD Hard USD cap (default Infinity) --output path JSONL output path (default auto) --dry-run Stub LLM calls (default if LITELLM_URL unset) --live Force real LLM calls even if LITELLM_URL unset --sample-lock P Path to a committed sample-lock JSON. Bypasses the dataset adapter and loads instances directly from the lock. Runtime asserts category distribution matches 13/13/12/12 for the Stage 2 preflight gate; fails fast otherwise. --judge MODEL Enable single-judge mode. After each cell call, invokes failure-mode-judge with the given model (e.g. claude-sonnet-4-6) and populates the judge fields on the JSONL record. Skip the flag to run without judging. --judge-ensemble M1,M2,... Enable 3+-judge ensemble mode. First model is the tie- breaker. Takes precedence over --judge. Real API spend — keep within the brief's $5 alarm per run. Pre-registration flags (Sprint 12 Task 1 Blocker #3): --manifest-hash Override bench-spec manifest SHA-256 (64-char hex). When omitted, resolves BENCH_SPEC_MANIFEST_PATH env or falls back to sibling ../PM-Waggle-OS/decisions/ path. --emit-preregistration-event Explicitly enable the bench.preregistration.manifest_hash event (default: enabled). --no-emit-preregistration-event Suppress the event (useful for smoke tests). --per-cell NAME Multi-value cell selection. Repeat for multiple cells (--per-cell raw --per-cell filtered). Overrides --cell and --all-cells when at least one value is supplied. --judge-tiebreak STRATEGY Tie-break strategy surfaced into the pre-registration payload. One of: quadri-vendor | pm-escalation | majority. Default: quadri-vendor (per A3 LOCK § B2). Task 2.5 Stage 1 substrate flags (2026-04-23): --locomo-raw-path P Path to raw LoCoMo archive (snap-research/locomo10.json). REQUIRED when --cell is 'retrieval' or 'agentic'. Main() builds an ephemeral :memory: MindDB + HybridSearch pair, ingests every turn as one I-frame (frame-per-turn per GATE-S0 lock), then tears it down at end of run. --retrieval-top-k N Top-K turn frames the retrieval cell recalls per question. Default 10 (GATE-S0 lock). Range 1..100. --agentic-max-turns N Hard turn cap for the agentic cell's inner agent-loop. Default 3 (GATE-S0 lock). Range 1..10. --agentic-timeout-ms N AbortController timeout for the agentic cell. Default 180_000 ms. Minimum 1000 ms. --help, -h This text `); } // ── Config loaders ───────────────────────────────────────────────────────── function harnessRoot(): string { // __filename equivalent for ESM. When compiled to dist/runner.js this // resolves to the dist dir; when run via tsx it resolves to src/runner.ts. const here = url.fileURLToPath(import.meta.url); // `harness/dist/runner.js` → harness dir; `harness/src/runner.ts` → harness dir. return path.resolve(path.dirname(here), '..'); } function loadModels(): Record { const cfg = path.join(harnessRoot(), 'config', 'models.json'); return JSON.parse(fs.readFileSync(cfg, 'utf-8')) as Record; } function loadDatasets(): Record { const cfg = path.join(harnessRoot(), 'config', 'datasets.json'); return JSON.parse(fs.readFileSync(cfg, 'utf-8')) as Record; } // ── Run driver ───────────────────────────────────────────────────────────── async function runOne(config: RunConfig): Promise { const startedAt = new Date().toISOString(); const dataRoot = path.join(harnessRoot(), '..', 'data'); // Sample-lock path, when provided, takes precedence over the dataset adapter // and enforces the Stage 2 13/13/12/12 distribution inside // loadPreflightSampleLock. Failure to match → throw before any LLM call, // which is the Task-1 acceptance requirement. const all = config.sampleLockPath ? loadPreflightSampleLock( path.isAbsolute(config.sampleLockPath) ? config.sampleLockPath : path.resolve(process.cwd(), config.sampleLockPath), ) : loadDataset(config.dataset, dataRoot); // Sprint 12 Task 1 Blocker #1: attach dataset_version (SHA-256 of canonical // archive) to every emitted JSONL record. Sample-lock runs get the // lock-file hash; regular runs get the dataset archive hash; synthetic // runs get the static `synthetic-scaffold-v1` string. Computed once per // runOne call to avoid per-instance disk I/O. const datasetVersion = config.sampleLockPath ? computeFileHash( path.isAbsolute(config.sampleLockPath) ? config.sampleLockPath : path.resolve(process.cwd(), config.sampleLockPath), ) : getDatasetVersion(config.dataset, dataRoot); // Sprint 12 Task 1 Blocker #3: emit `bench.preregistration.manifest_hash` // once per runOne before the first instance iteration. Suppressed when // the caller sets `emitPreregistrationEvent: false` (test default). if (config.emitPreregistrationEvent !== false) { emitPreregistrationManifest( assemblePreregistrationPayload(config, datasetVersion, all.length), ); } // When a sample lock drives the run, honor instance order verbatim — the // lock file IS the deterministic sample. Cell comparisons require identical // order across cells, and `sampleInstances` would re-shuffle the lock. const sampled = config.sampleLockPath ? (Number.isFinite(config.limit) && config.limit < all.length ? all.slice(0, config.limit) : all.slice()) : sampleInstances(all, config.seed, config.limit); const writer = new JsonlWriter(config.outputPath); const llm = createLlmClient({ dryRun: config.dryRun, litellmUrl: config.litellmUrl, litellmApiKey: config.litellmApiKey, }); let totalCost = 0; let budgetStoppedAt: number | null = null; const latenciesByInstance: number[] = []; // §7.2 Stage 1.5 streak halt — track consecutive fetch-transport failures. // Tracker is cell-local: each cell gets a fresh counter so a streak in the // previous cell doesn't poison the next. const streakTracker = new StreakTracker(); let streakHaltAt: number | null = null; let streakHaltSummary: string | null = null; for (let i = 0; i < sampled.length; i++) { if (totalCost >= config.budgetUsd) { budgetStoppedAt = i; break; } const instance = sampled[i]; const turnId = generateTurnId(); // Task 2.5 Stage 1: cells and controls have diverged CellInput/ControlInput // shapes — cells now accept optional substrate + litellm + per-cell knobs // that controls don't need. Split the dispatch so each side gets only the // fields it understands. let result; if (config.run.kind === 'cell') { const cellFn = cells[config.run.name as CellName]; result = await cellFn({ instance, model: config.model, llm, turnId, substrate: config.substrate, litellm: config.litellm, retrievalTopK: config.retrievalTopK, agenticMaxTurns: config.agenticMaxTurns, agenticTimeoutMs: config.agenticTimeoutMs, }); } else { const controlFn = controls[config.run.name as ControlName]; result = await controlFn({ instance, model: config.model, llm, turnId }); } latenciesByInstance.push(result.latencyMs); const accuracy = result.failureMode ? 0 : scoreAccuracy(result.text, instance.expected); totalCost += result.costUsd; // Sprint 11 A2: emit structured llm.response event tagged with turnId. // `reasoningShape` + char count become the canonical observability // surface; reasoning content itself goes to JSONL only, never to logs, // per design doc §2.3/§2.4 exclusion rules. logTurnEvent(turnId, { stage: 'llm.response', cell: config.run.name, model: config.model.id, textChars: result.text.length, latencyMs: result.latencyMs, costUsd: result.costUsd, failureMode: result.failureMode, reasoningShape: result.reasoningShape ?? 'none', reasoningChars: result.reasoningContent?.length ?? 0, }); // Drift alarm per ratification §Q3: thinking=on was requested but no // reasoning field present — observable without failing the run. if (result.reasoningShape === 'unknown') { logTurnEvent(turnId, { stage: 'llm.response.reasoning_content_shape_unknown', cell: config.run.name, model: config.model.id, litellmModel: config.model.litellmModel, }); } // Sprint 9 Task 2: when a judge is configured, grade the answer // in-line. The judge runs even when the cell call itself failed // (failureMode !== null) because the transcript still has value // for calibration; the aggregator can filter if needed. A judge // error never aborts the batch — runJudge swallows and annotates. let judgePayload: import('./judge-runner.js').JudgePayload | null = null; if (config.judgeConfig && !result.failureMode) { judgePayload = await runJudge( { question: instance.question, // scoreAccuracy picks the first expected answer; keep parity // so the judge sees the same ground truth. Remaining entries // in `expected` are alternate phrasings, not independent // facts, and are irrelevant to the §4 judge prompt. groundTruth: instance.expected[0] ?? '', contextExcerpt: instance.context, modelAnswer: result.text, }, config.judgeConfig, ); } // p50 / p95 are computed over the running window of observed latencies // so the JSONL record carries real-time percentiles (not flat per-row). // Aggregate metrics.ts recomputes globals for the summary. const record: JsonlRecord = { turnId, cell: config.run.name, instance_id: instance.instance_id, model: config.model.id, seed: config.seed, accuracy, p50_latency_ms: percentile(latenciesByInstance, 50), p95_latency_ms: percentile(latenciesByInstance, 95), usd_per_query: round(result.costUsd, 6), failure_mode: result.failureMode, dataset_version: datasetVersion, ...(judgePayload && { model_answer: judgePayload.model_answer, judge_verdict: judgePayload.judge_verdict, judge_failure_mode: judgePayload.judge_failure_mode, judge_rationale: judgePayload.judge_rationale, judge_model: judgePayload.judge_model, judge_timestamp: judgePayload.judge_timestamp, judge_ensemble: judgePayload.judge_ensemble, // B2 fold-in observability fields. ...(judgePayload.tie_break_path !== undefined && { tie_break_path: judgePayload.tie_break_path }), ...(judgePayload.tie_break_fourth_vendor !== undefined && { tie_break_fourth_vendor: judgePayload.tie_break_fourth_vendor }), // Sprint 12 Task 2 §2.1 A3 namespace split (LOCKED 2026-04-23) — // authoritative A3 LOCK § 6 taxonomy columns. `undefined` a3_failure_code // (e.g. PM_ESCALATION skipped instance) is spread conditionally so // the absence of the key is preserved, matching the aggregator's // "skipped" semantics. ...(judgePayload.a3_failure_code !== undefined && { a3_failure_code: judgePayload.a3_failure_code }), ...(judgePayload.a3_rationale !== undefined && { a3_rationale: judgePayload.a3_rationale }), }), // Sprint 11 A2: reasoning_content fields — same-row persistence // keyed by `turnId` per ratification §Q4. Chars stay separate for // aggregation even when content is stripped on the read path. ...(result.reasoningContent !== undefined && { reasoning_content: result.reasoningContent, reasoning_content_chars: result.reasoningContent.length, }), ...(result.reasoningShape !== undefined && { reasoning_shape: result.reasoningShape, }), // Sprint 12 Task 1 Blocker #3 / B3 addendum § 4: copy target-model // pinning surface fields from `config/models.json` onto every row so // any single JSONL line carries its own audit-replication anchor. ...(config.model.pinning_surface !== undefined && { model_pinning_surface: config.model.pinning_surface, model_pinning_carve_out_reason: config.model.pinning_surface_carve_out_reason ?? null, }), // B3 addendum § 4 leaves `model_revision_hash` null unless the provider // exposes one. Future providers (e.g. OpenRouter with revision_id) // will populate this via a provider-specific hook in Session 3+. model_revision_hash: null, }; writer.write(record); // §7.2 Stage 1.5 streak halt — check AFTER writing the current record so // the failure-forensic row persists. Break out of the loop; the error is // thrown AFTER writer.close() below so JSONL + summary files flush // cleanly. Caller (main) surfaces the error to the operator. if (streakTracker.record(result.failureMode)) { streakHaltAt = i + 1; streakHaltSummary = streakTracker.summary(); break; } } await writer.close(); const finishedAt = new Date().toISOString(); const summary = buildAggregate(config, writer.all(), startedAt, finishedAt, budgetStoppedAt); const summaryPath = config.outputPath.replace(/\.jsonl$/, '.summary.json'); fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2), 'utf-8'); // Compact stdout summary — machine-parseable prefix makes CI grep easy. console.log( `[bench:summary] kind=${summary.run.kind} name=${summary.run.name} ` + `model=${summary.run.model} seed=${summary.run.seed} ` + `n=${summary.counts.total} completed=${summary.counts.completed} ` + `failed=${summary.counts.failed} ` + `accuracy=${summary.metrics.meanAccuracy} ` + `p50=${summary.metrics.p50LatencyMs}ms p95=${summary.metrics.p95LatencyMs}ms ` + `cost=$${summary.metrics.totalUsd} ` + `jsonl=${config.outputPath}`, ); // §7.2 Stage 1.5 — throw AFTER writer + summary flushed so partial data // persists for forensic review (same pattern as budget-stop, except budget // is a soft stop that doesn't throw). The surfaced error lets main() // decide whether to continue other cells or bail the whole invocation. if (streakHaltAt !== null) { throw new Error( `[bench:halt] cell '${config.run.name}' aborted at instance ${streakHaltAt}/${sampled.length} ` + `due to consecutive fetch-transport failures (${streakHaltSummary}). ` + `Likely upstream saturation — check LiteLLM proxy + provider health before re-running. ` + `Partial JSONL persisted at ${config.outputPath}.`, ); } } function round(n: number, decimals: number): number { const f = Math.pow(10, decimals); return Math.round(n * f) / f; } /** SHA-256 hex of a file's bytes. Used for sample-lock version stamping * when the run is driven by a committed lock file rather than the regular * dataset archive. */ function computeFileHash(absolutePath: string): string { const buf = fs.readFileSync(absolutePath); return crypto.createHash('sha256').update(buf).digest('hex'); } /** * Resolve the CLI-requested judge-model roster against `config/models.json` * so every pre-registration emit carries B3-addendum pinning fields. Keeps * the registry lookup in main() — runOne only forwards the materialised * list. * * Unknown model ids are passed through with `floating_alias` defaults and * an explicit carve-out reason that names the ID. This keeps the run going * while loudly surfacing the gap to downstream audit consumers. */ function resolveJudgeModelsForPreregistration( models: Record, args: ParsedArgs, ): JudgeModelManifestEntry[] { const ids: string[] = []; if (args.judgeEnsemble && args.judgeEnsemble.length > 0) { ids.push(...args.judgeEnsemble); } else if (args.judge) { ids.push(args.judge); } return ids.map((id, idx) => { const spec = models[id]; if (!spec) { return { model_id: id, provider: 'unknown', judge_role: (idx === 0 ? 'primary' : idx === 1 ? 'secondary' : 'tertiary'), pinning_surface: 'floating_alias' as const, pinning_surface_carve_out_reason: `Model id '${id}' not found in benchmarks/harness/config/models.json — pinning surface cannot be resolved; treating as floating_alias for audit safety`, }; } return { model_id: spec.id, provider: spec.provider, judge_role: spec.judge_role ?? (idx === 0 ? 'primary' : idx === 1 ? 'secondary' : 'tertiary'), pinning_surface: spec.pinning_surface ?? 'floating_alias', pinning_surface_carve_out_reason: spec.pinning_surface_carve_out_reason ?? null, }; }); } /** * Build the `PreregistrationManifestPayload` from the RunConfig + derived * per-run context. Split out from `runOne` so tests can exercise the * assembly without spinning up a full run. * * Judge models surfacing: in Sub-deliverable A the list is derived from * `config.judgeConfig.models` (ensemble) or `config.judgeConfig.model` * (single-judge), with placeholder pinning fields. Sub-deliverable C * tightens per-judge `pinning_surface` + `carve_out_reason` by looking * each model up in `config/models.json`. Until C lands, judges emit as * `floating_alias` + null carve-out (schema-valid placeholder). */ function assemblePreregistrationPayload( config: RunConfig, datasetVersion: string, datasetInstanceCount: number, ): PreregistrationManifestPayload { const manifestHash = config.manifestHash ?? computeBenchSpecManifestHash(); // `readManifestLockedDate` throws if the YAML can't be located and no // override is provided. If the caller supplied `manifestHash` explicitly // (test path), we still try to resolve the YAML for `locked_at`; if // resolution fails under that branch, fall back to 'unknown' so tests // that don't carry the YAML keep working. let manifestLockedAt: string; try { manifestLockedAt = readManifestLockedDate(); } catch { manifestLockedAt = 'unknown'; } const perCell = config.perCellList ?? [config.run.name]; // Sub-deliverable C: `judgeModelsResolved` is pre-materialised in main() // from args.judgeEnsemble against config/models.json. When judging is // disabled the list is empty (schema-valid per brief § 3.1). const judgeModels: JudgeModelManifestEntry[] = config.judgeModelsResolved ?? []; return { manifest_hash: manifestHash, manifest_path: CANONICAL_MANIFEST_PATH, manifest_locked_at: manifestLockedAt, dataset_version: datasetVersion, dataset_path: config.dataset.dataPath, dataset_instance_count: datasetInstanceCount, per_cell: perCell, judge_tiebreak: config.judgeTiebreak ?? 'quadri-vendor', judge_models: judgeModels, emitted_at: new Date().toISOString(), runner_version: getRunnerVersion(), runner_invocation: { argv: sanitizeArgv(process.argv), cwd: process.cwd(), }, }; } // ── Main ─────────────────────────────────────────────────────────────────── function buildRuns(args: ParsedArgs): RunKind[] { const runs: RunKind[] = []; // Sprint 12 Task 1 Blocker #3: `--per-cell` (multi-value) overrides the // single-cell / all-cells dispatch when at least one value is supplied. // Sprint 12 Task 2.5 Stage 2-Retry §1.5: valid-name list extended to // include retrieval + agentic (Stage 1) + no-context (Stage 2-Retry). // `--all-cells` is preserved at the Sprint 9 quartet for backward // compatibility per PM ratification; `scripts/run-mini-locomo.ts // --v3-cells` expands to the Stage 2-Retry 5-cell list via --per-cell. const VALID_CELLS = 'raw | filtered | compressed | full-context | retrieval | agentic | no-context'; if (args.perCell && args.perCell.length > 0) { for (const name of args.perCell) { if (!isCellName(name)) { throw new Error(`Unknown cell: ${name}. Valid: ${VALID_CELLS}`); } runs.push({ kind: 'cell', name }); } return runs; } if (args.allCells) { (['raw', 'filtered', 'compressed', 'full-context'] as CellName[]).forEach(n => { runs.push({ kind: 'cell', name: n }); }); } else if (args.control) { if (!isControlName(args.control)) { throw new Error(`Unknown control: ${args.control}. Valid: verbose-fixed`); } runs.push({ kind: 'control', name: args.control }); } else if (args.cell) { if (!isCellName(args.cell)) { throw new Error(`Unknown cell: ${args.cell}. Valid: ${VALID_CELLS}`); } runs.push({ kind: 'cell', name: args.cell }); } else { throw new Error('Must specify one of --cell , --all-cells, --per-cell , or --control . Try --help.'); } return runs; } function defaultOutputPath(kind: RunKind, dataset: string): string { const ts = new Date().toISOString().replace(/[:.]/g, '-'); const resultsDir = path.join(harnessRoot(), '..', 'results'); return path.join(resultsDir, `${kind.name}-${dataset}-${ts}.jsonl`); } async function main(): Promise { const args = parseArgs(process.argv.slice(2)); const models = loadModels(); const datasets = loadDatasets(); const model = models[args.model]; if (!model) { throw new Error(`Unknown model: ${args.model}. Valid ids: ${Object.keys(models).join(', ')}`); } const dataset = datasets[args.dataset]; if (!dataset) { throw new Error(`Unknown dataset: ${args.dataset}. Valid ids: ${Object.keys(datasets).join(', ')}`); } const dryRun = args.dryRun ?? !process.env.LITELLM_URL; const litellmUrl = process.env.LITELLM_URL ?? 'http://localhost:4000'; const litellmApiKey = process.env.LITELLM_API_KEY ?? 'sk-waggle-dev'; // Build judge config (Sprint 9 Task 2). `--judge-ensemble` wins when // both flags are present. In dry-run mode judge is force-skipped — // calling a real LiteLLM judge while the cell path is stubbed would // produce mixed-signal JSONL that's hard to interpret. const judgeCosts: JudgeClientCostEntry[] = []; let judgeConfig: JudgeConfig | undefined; if (!dryRun) { if (args.judgeEnsemble && args.judgeEnsemble.length > 0) { const clients = new Map(); for (const m of args.judgeEnsemble) { clients.set( m, createJudgeLlmClient({ litellmUrl, litellmApiKey, model: m, onCall: entry => judgeCosts.push(entry), }), ); } // Sprint 11 B2 fold-in (2026-04-22): when the primary ensemble is // exactly 3 vendors (the Sprint 10 Task 2.2 ratified trio — Opus 4.7 // + GPT-5.4 + Gemini 3.1-Pro), auto-wire xai/grok-4.20 as the // fourth-vendor tie-breaker per decisions/2026-04-22-tie-break-policy-locked.md. // Skip wiring when: (a) the ensemble isn't 3-vendor, (b) grok-4.20 is // already in the primary list (would create dual-role ambiguity). // // Sprint 12 Task 2 §2.3 (2026-04-23): split the models.json key (used // to construct the LiteLLM client with the correct route resolution) // from the OpenRouter canonical audit slug (used as the // `tie_break_fourth_vendor` JSONL column + pino `fourth_vendor_slug`). // Without the split, the JSONL audit log would report the harness- // local key `'grok-4.20'` and diverge from the §2.2-verified // OpenRouter slug `x-ai/grok-4.20`. const TIE_BREAKER_MODEL_KEY = 'grok-4.20'; // models.json key (LiteLLM route) const TIE_BREAKER_AUDIT_SLUG = 'x-ai/grok-4.20'; // OpenRouter canonical (§2.2 verified 2026-04-23) const shouldWireTieBreaker = args.judgeEnsemble.length === 3 && !args.judgeEnsemble.includes(TIE_BREAKER_MODEL_KEY); let tieBreakerClient: import('./judge-client.js').LlmClient | undefined; if (shouldWireTieBreaker) { tieBreakerClient = createJudgeLlmClient({ litellmUrl, litellmApiKey, model: TIE_BREAKER_MODEL_KEY, onCall: entry => judgeCosts.push(entry), }); } judgeConfig = { kind: 'ensemble', models: args.judgeEnsemble, clients, ...(shouldWireTieBreaker && tieBreakerClient && { tieBreakerModel: TIE_BREAKER_AUDIT_SLUG, tieBreakerClient, }), }; } else if (args.judge) { judgeConfig = { kind: 'single', model: args.judge, client: createJudgeLlmClient({ litellmUrl, litellmApiKey, model: args.judge, onCall: entry => judgeCosts.push(entry), }), }; } } const runs = buildRuns(args); // Sprint 12 Task 1 Blocker #3: invocation-level cell list fed into every // runOne so the pre-registration payload reports the full scope rather // than a single runOne's slice. Stripping kind='control' entries from the // list would hide verbose-fixed-only invocations; report them as-is. const perCellList = runs.map(r => r.name); // Sub-deliverable C: materialise judge-model roster with B3 addendum // pinning fields so every pre-registration emit carries audit-anchor // metadata without repeating the models.json lookup per runOne. const judgeModelsResolved = resolveJudgeModelsForPreregistration(models, args); // Sprint 12 Task 2.5 Stage 1: when the cell roster includes retrieval or // agentic, build an ephemeral substrate and pre-ingest every LoCoMo turn as // an I-frame (frame-per-turn per GATE-S0 lock). Substrate is shared across // every cell in this invocation so the corpus is indexed once, not per-cell. // Lifecycle is owned here; `close()` fires in the `finally` block below. const substrateNeeded = runs.some( r => r.kind === 'cell' && (r.name === 'retrieval' || r.name === 'agentic'), ); let substrate: Substrate | null = null; let runnerLock: LockHandle | null = null; try { // §7.4 Stage 1.5 — acquire the global single-runner lock FIRST. Fail // fast before any substrate ingest or health-probe cost is spent. // Sentinel lives under benchmarks/results/ so any two runner invocations // contend regardless of their --output paths. const lockSentinel = path.join(harnessRoot(), '..', 'results', '.benchmark-runner'); runnerLock = acquireRunnerLock(lockSentinel); console.log(`[bench:lock] acquired ${runnerLock.lockPath} (pid=${process.pid})`); // §7.3 Stage 1.5 — pre-cell health check. Probes /health/liveliness + a // short ping call per (subject + judge ensemble). Skipped in dry-run // mode (no upstream to probe). Any 5xx or network error aborts before // any instance budget is spent. if (!dryRun) { const judgePingModels = args.judgeEnsemble ? [...args.judgeEnsemble] : (args.judge ? [args.judge] : []); const hc = await preCellHealthCheck({ litellmUrl, litellmApiKey, subjectModel: model.litellmModel, judgeModels: judgePingModels, }); if (!hc.ok) { const summary = hc.failures.map(f => `${f.endpoint} → ${f.error}`).join('; '); throw new Error( `[bench:health-check] FAILED in ${hc.durationMs}ms against ${litellmUrl}: ${summary}. ` + `Aborting before any instance budget is spent. Check LiteLLM proxy + provider keys.`, ); } console.log( `[bench:health-check] OK — probed subject + ${judgePingModels.length} judge(s) in ${hc.durationMs}ms`, ); } if (substrateNeeded) { if (dryRun) { // Retrieval + agentic aren't meaningful in dry-run mode (the agent // loop would 404 against the stub LLM, and the embedder would still // need a live Ollama server). Fail loudly rather than ship silently // broken results. throw new Error( 'retrieval/agentic cells cannot run in dry-run mode. Remove --dry-run ' + 'or set LITELLM_URL to activate live mode.', ); } if (!args.locomoRawPath) { throw new Error( 'retrieval/agentic cells require --locomo-raw-path . ' + 'Point it at the raw snap-research/locomo10.json archive.', ); } const rawPath = path.isAbsolute(args.locomoRawPath) ? args.locomoRawPath : path.resolve(process.cwd(), args.locomoRawPath); substrate = createSubstrate(); // default: :memory: + ollama-embedder const substrateStart = Date.now(); const turns = extractTurnsFromLocomoRaw(rawPath); const stats = await ingestLoCoMoCorpus( substrate.db, substrate.search, substrate.frames, substrate.sessions, turns, ); console.log( `[bench:substrate] turns_found=${turns.length} frames_created=${stats.count} ` + `ingest_ms=${stats.ingestMs} index_ms=${stats.indexMs} total_ms=${Date.now() - substrateStart}`, ); } for (const run of runs) { const outputPath = args.output ?? defaultOutputPath(run, args.dataset); await runOne({ run, dataset, model, limit: args.limit, seed: args.seed, budgetUsd: args.budget, outputPath, dryRun, litellmUrl, litellmApiKey, sampleLockPath: args.sampleLock, judgeConfig, onJudgeCall: entry => judgeCosts.push(entry), manifestHash: args.manifestHash, emitPreregistrationEvent: args.emitPreregistrationEvent, perCellList, judgeTiebreak: args.judgeTiebreak, judgeModelsResolved, // Task 2.5 Stage 1 — substrate deps (undefined for non-substrate cells) substrate: substrate ?? undefined, litellm: { url: litellmUrl, apiKey: litellmApiKey }, retrievalTopK: args.retrievalTopK, agenticMaxTurns: args.agenticMaxTurns, agenticTimeoutMs: args.agenticTimeoutMs, }); } // Surface judge spend separately from cell spend so the brief's "$5 // alarm per run" guardrail applies cleanly to the judge-layer budget. if (judgeCosts.length > 0) { const judgeTotalUsd = judgeCosts.reduce((sum, e) => sum + e.usd, 0); const judgeOk = judgeCosts.filter(e => e.ok).length; console.log( `[bench:judge-summary] calls=${judgeCosts.length} ok=${judgeOk} ` + `failed=${judgeCosts.length - judgeOk} total_usd=$${judgeTotalUsd.toFixed(6)}`, ); } } finally { if (substrate) { substrate.close(); } if (runnerLock) { runnerLock.release(); } } } // Only run main() when invoked directly (not when imported by tests). const isMain = typeof process !== 'undefined' && Array.isArray(process.argv) && process.argv[1] !== undefined && url.fileURLToPath(import.meta.url) === path.resolve(process.argv[1]); if (isMain) { main().catch(err => { console.error('[bench:error]', err?.message ?? err); process.exit(1); }); } // Exported for the smoke tests. export { main, parseArgs, buildRuns, runOne, defaultOutputPath };