980 lines
43 KiB
TypeScript
980 lines
43 KiB
TypeScript
#!/usr/bin/env tsx
|
|
/**
|
|
* Four-cell ablation harness — CLI entry.
|
|
*
|
|
* Usage (via `npm run bench -- ...` from repo root, or direct `tsx`):
|
|
*
|
|
* --cell raw Single cell
|
|
* --cell filtered
|
|
* --cell compressed
|
|
* --cell full-context
|
|
* --all-cells Run all four sequentially (same dataset/seed)
|
|
* --control verbose-fixed Diagnostic control (not a cell)
|
|
*
|
|
* --dataset locomo locomo | longmemeval | synthetic
|
|
* --limit N N instances. --full sets Infinity.
|
|
* --full Alias for --limit Infinity.
|
|
* --model qwen3.6-35b-a3b Must match an id in config/models.json
|
|
* --seed N Default: 42 (reproducibility artifact req).
|
|
* --budget USD Hard USD cap. Default: Infinity.
|
|
* --output path.jsonl Default: ../results/<cell>-<dataset>-<ts>.jsonl
|
|
* --dry-run Stub LLM calls. Default: true when
|
|
* LITELLM_URL is not set.
|
|
*
|
|
* Env:
|
|
* LITELLM_URL default http://localhost:4000
|
|
* LITELLM_API_KEY default sk-waggle-dev
|
|
*/
|
|
|
|
import crypto from 'node:crypto';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import url from 'node:url';
|
|
import { generateTurnId, logTurnEvent } from '@waggle/agent';
|
|
import type {
|
|
CellName, ControlName, DatasetSpec, JsonlRecord, ModelSpec, RunConfig, RunKind,
|
|
} from './types.js';
|
|
import { getDatasetVersion, loadDataset, loadPreflightSampleLock, sampleInstances } from './datasets.js';
|
|
import { createLlmClient } from './llm.js';
|
|
import {
|
|
CANONICAL_MANIFEST_PATH,
|
|
computeBenchSpecManifestHash,
|
|
emitPreregistrationManifest,
|
|
getRunnerVersion,
|
|
readManifestLockedDate,
|
|
sanitizeArgv,
|
|
type JudgeModelManifestEntry,
|
|
type PreregistrationManifestPayload,
|
|
} from './preregistration.js';
|
|
import { JsonlWriter, buildAggregate, scoreAccuracy, percentile } from './metrics.js';
|
|
import { cells, isCellName, isControlName } from './cells.js';
|
|
import { controls } from './controls.js';
|
|
import { createJudgeLlmClient, type JudgeClientCostEntry } from './judge-client.js';
|
|
import { runJudge, type JudgeConfig } from './judge-runner.js';
|
|
// Sprint 12 Task 2.5 Stage 1 (2026-04-23) — substrate hookup for retrieval +
|
|
// agentic cells. Substrate is created once in main() when the cell roster
|
|
// includes `retrieval` or `agentic`, pre-ingested from raw LoCoMo, then
|
|
// threaded through RunConfig → CellInput to each iteration.
|
|
import { createSubstrate, type Substrate } from './substrate.js';
|
|
import { extractTurnsFromLocomoRaw, ingestLoCoMoCorpus } from './ingest.js';
|
|
// Sprint 12 Task 2.5 Stage 1.5 §7.2 — consecutive fetch-transport failure
|
|
// halt. Prevents the v2 full-context saturation cascade from burning the
|
|
// full instance budget on dead calls.
|
|
import { StreakTracker } from './streak-tracker.js';
|
|
// Sprint 12 Task 2.5 Stage 1.5 §7.3 — pre-cell health check. Probes every
|
|
// upstream route the run depends on before spending instance-budget on them.
|
|
import { preCellHealthCheck } from './health-check.js';
|
|
// Sprint 12 Task 2.5 Stage 1.5 §7.4 — single-runner PID+heartbeat lock.
|
|
// Encodes `concurrent_runners: FORBIDDEN` at the process level. Uses a
|
|
// single global sentinel under benchmarks/results/ so two harness processes
|
|
// can never both run even against different datasets/models (they still
|
|
// share the LiteLLM proxy + provider bridges — the v2 saturation root cause).
|
|
import { acquireRunnerLock, type LockHandle } from './runner-lock.js';
|
|
|
|
// ── Arg parsing ────────────────────────────────────────────────────────────
|
|
|
|
interface ParsedArgs {
|
|
cell?: string;
|
|
allCells: boolean;
|
|
control?: string;
|
|
dataset: string;
|
|
limit: number;
|
|
model: string;
|
|
seed: number;
|
|
budget: number;
|
|
output?: string;
|
|
dryRun?: boolean;
|
|
sampleLock?: string;
|
|
/** When set, each cell/control call also triggers a judge call after
|
|
* the model answer returns. Value is the single-judge model id
|
|
* (e.g. `claude-sonnet-4-6`). Default: undefined → judging skipped. */
|
|
judge?: string;
|
|
/** Comma-separated list of judge model ids for ensemble mode. First
|
|
* id is the tie-breaker (Sonnet by taxonomy §6 convention). Takes
|
|
* precedence over `--judge` when both are set. */
|
|
judgeEnsemble?: string[];
|
|
// ── Sprint 12 Task 1 Blocker #3 — pre-registration flags ─────────────────
|
|
/** SHA-256 hex of the bench-spec manifest YAML. When omitted, the
|
|
* emitter resolves the manifest path (env → sibling PM-Waggle-OS → throw)
|
|
* and computes the hash at run start. When provided, skips the lookup
|
|
* entirely — useful for tests and replication runs where the source
|
|
* YAML is not available at the runtime cwd. */
|
|
manifestHash?: string;
|
|
/** Suppresses `bench.preregistration.manifest_hash` pino-style event
|
|
* when `false`. Default: true. Tests pass `--no-emit-preregistration-event`
|
|
* to keep the log surface quiet. */
|
|
emitPreregistrationEvent: boolean;
|
|
/** Multi-value cell selection (repeatable `--per-cell raw --per-cell filtered`).
|
|
* Overrides `--cell` and `--all-cells` when at least one value is set.
|
|
* Also materializes into the `per_cell` field of the pre-registration
|
|
* manifest payload regardless of how cells were chosen (falls back to
|
|
* derivation from --all-cells or --cell when unset). */
|
|
perCell?: string[];
|
|
/** Judge tie-break strategy name surfaced into the pre-registration
|
|
* payload. Canonical A3 LOCK § B2 values: `quadri-vendor` (fourth
|
|
* vendor fires on 1-1-1) and `pm-escalation` (2-2 escalates to PM).
|
|
* `majority` is accepted as an explicit-default alias. */
|
|
judgeTiebreak?: string;
|
|
// ── Sprint 12 Task 2.5 Stage 1 (2026-04-23) — substrate flags ────────────
|
|
/** Path to the raw LoCoMo archive (`locomo10.json`). Required when the cell
|
|
* roster includes `retrieval` or `agentic`. Relative paths resolve against
|
|
* the current working directory. */
|
|
locomoRawPath?: string;
|
|
/** Retrieval top-K (default 10, GATE-S0 lock). Threaded into CellInput. */
|
|
retrievalTopK?: number;
|
|
/** Agentic hard turn cap (default 3, GATE-S0 lock). */
|
|
agenticMaxTurns?: number;
|
|
/** Agentic AbortController timeout in ms (default 180_000). */
|
|
agenticTimeoutMs?: number;
|
|
}
|
|
|
|
function parseArgs(argv: string[]): ParsedArgs {
|
|
const out: ParsedArgs = {
|
|
allCells: false,
|
|
dataset: 'synthetic',
|
|
limit: 10,
|
|
model: 'qwen3.6-35b-a3b',
|
|
seed: 42,
|
|
budget: Number.POSITIVE_INFINITY,
|
|
emitPreregistrationEvent: true,
|
|
};
|
|
const VALID_TIEBREAK = new Set(['quadri-vendor', 'pm-escalation', 'majority']);
|
|
for (let i = 0; i < argv.length; i++) {
|
|
const flag = argv[i];
|
|
const next = argv[i + 1];
|
|
switch (flag) {
|
|
case '--cell': out.cell = next; i++; break;
|
|
case '--all-cells': out.allCells = true; break;
|
|
case '--control': out.control = next; i++; break;
|
|
case '--dataset': out.dataset = next; i++; break;
|
|
case '--limit': out.limit = Number(next); i++; break;
|
|
case '--full': out.limit = Number.POSITIVE_INFINITY; break;
|
|
case '--model': out.model = next; i++; break;
|
|
case '--seed': out.seed = Number(next); i++; break;
|
|
case '--budget': out.budget = Number(next); i++; break;
|
|
case '--output': out.output = next; i++; break;
|
|
case '--dry-run': out.dryRun = true; break;
|
|
case '--live': out.dryRun = false; break;
|
|
case '--sample-lock': out.sampleLock = next; i++; break;
|
|
case '--judge': out.judge = next; i++; break;
|
|
case '--judge-ensemble':
|
|
out.judgeEnsemble = (next ?? '').split(',').map(s => s.trim()).filter(Boolean);
|
|
i++;
|
|
break;
|
|
// ── Sprint 12 Task 1 Blocker #3 flags ───────────────────────────
|
|
case '--manifest-hash':
|
|
if (typeof next !== 'string' || !/^[0-9a-f]{64}$/i.test(next)) {
|
|
throw new Error(
|
|
`Invalid --manifest-hash value: expected 64-char lowercase SHA-256 hex, got ${next ?? '(missing)'}`,
|
|
);
|
|
}
|
|
out.manifestHash = next.toLowerCase();
|
|
i++;
|
|
break;
|
|
case '--emit-preregistration-event':
|
|
out.emitPreregistrationEvent = true;
|
|
break;
|
|
case '--no-emit-preregistration-event':
|
|
out.emitPreregistrationEvent = false;
|
|
break;
|
|
case '--per-cell':
|
|
if (typeof next !== 'string' || next.length === 0) {
|
|
throw new Error(`Invalid --per-cell value: expected cell name, got ${next ?? '(missing)'}`);
|
|
}
|
|
out.perCell = out.perCell ?? [];
|
|
out.perCell.push(next);
|
|
i++;
|
|
break;
|
|
case '--judge-tiebreak':
|
|
if (typeof next !== 'string' || !VALID_TIEBREAK.has(next)) {
|
|
throw new Error(
|
|
`Invalid --judge-tiebreak value: expected one of ${[...VALID_TIEBREAK].join(' | ')}, got ${next ?? '(missing)'}`,
|
|
);
|
|
}
|
|
out.judgeTiebreak = next;
|
|
i++;
|
|
break;
|
|
// ── Sprint 12 Task 2.5 Stage 1 substrate flags ───────────────────────
|
|
case '--locomo-raw-path':
|
|
if (typeof next !== 'string' || next.length === 0) {
|
|
throw new Error(`Invalid --locomo-raw-path value: expected path, got ${next ?? '(missing)'}`);
|
|
}
|
|
out.locomoRawPath = next;
|
|
i++;
|
|
break;
|
|
case '--retrieval-top-k': {
|
|
const n = Number(next);
|
|
if (!Number.isInteger(n) || n <= 0 || n > 100) {
|
|
throw new Error(`Invalid --retrieval-top-k value: expected 1..100 integer, got ${next ?? '(missing)'}`);
|
|
}
|
|
out.retrievalTopK = n;
|
|
i++;
|
|
break;
|
|
}
|
|
case '--agentic-max-turns': {
|
|
const n = Number(next);
|
|
if (!Number.isInteger(n) || n <= 0 || n > 10) {
|
|
throw new Error(`Invalid --agentic-max-turns value: expected 1..10 integer, got ${next ?? '(missing)'}`);
|
|
}
|
|
out.agenticMaxTurns = n;
|
|
i++;
|
|
break;
|
|
}
|
|
case '--agentic-timeout-ms': {
|
|
const n = Number(next);
|
|
if (!Number.isFinite(n) || n < 1000) {
|
|
throw new Error(`Invalid --agentic-timeout-ms value: expected ≥1000 ms, got ${next ?? '(missing)'}`);
|
|
}
|
|
out.agenticTimeoutMs = n;
|
|
i++;
|
|
break;
|
|
}
|
|
case '--help':
|
|
case '-h':
|
|
printHelp();
|
|
process.exit(0);
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function printHelp(): void {
|
|
console.log(`Four-cell ablation harness.
|
|
|
|
Usage:
|
|
waggle-bench --cell <raw|filtered|compressed|full-context> --dataset <id> --limit N --model <id>
|
|
waggle-bench --all-cells --dataset <id> --limit N --model <id> [--budget USD]
|
|
waggle-bench --control verbose-fixed --dataset <id> --limit N --model <id>
|
|
|
|
Flags:
|
|
--cell Single cell name
|
|
--all-cells Run all four cells sequentially
|
|
--control Diagnostic control (e.g. verbose-fixed)
|
|
--dataset locomo | longmemeval | synthetic (default synthetic)
|
|
--limit N Instance count cap (default 10)
|
|
--full --limit Infinity
|
|
--model id Model id from config/models.json (default qwen3.6-35b-a3b)
|
|
--seed N PRNG seed (default 42)
|
|
--budget USD Hard USD cap (default Infinity)
|
|
--output path JSONL output path (default auto)
|
|
--dry-run Stub LLM calls (default if LITELLM_URL unset)
|
|
--live Force real LLM calls even if LITELLM_URL unset
|
|
--sample-lock P Path to a committed sample-lock JSON. Bypasses the dataset
|
|
adapter and loads instances directly from the lock. Runtime
|
|
asserts category distribution matches 13/13/12/12 for the
|
|
Stage 2 preflight gate; fails fast otherwise.
|
|
--judge MODEL Enable single-judge mode. After each cell call, invokes
|
|
failure-mode-judge with the given model (e.g.
|
|
claude-sonnet-4-6) and populates the judge fields on the
|
|
JSONL record. Skip the flag to run without judging.
|
|
--judge-ensemble M1,M2,...
|
|
Enable 3+-judge ensemble mode. First model is the tie-
|
|
breaker. Takes precedence over --judge. Real API spend —
|
|
keep within the brief's $5 alarm per run.
|
|
|
|
Pre-registration flags (Sprint 12 Task 1 Blocker #3):
|
|
--manifest-hash <sha> Override bench-spec manifest SHA-256 (64-char hex).
|
|
When omitted, resolves BENCH_SPEC_MANIFEST_PATH env or
|
|
falls back to sibling ../PM-Waggle-OS/decisions/ path.
|
|
--emit-preregistration-event
|
|
Explicitly enable the bench.preregistration.manifest_hash
|
|
event (default: enabled).
|
|
--no-emit-preregistration-event
|
|
Suppress the event (useful for smoke tests).
|
|
--per-cell NAME Multi-value cell selection. Repeat for multiple cells
|
|
(--per-cell raw --per-cell filtered). Overrides --cell
|
|
and --all-cells when at least one value is supplied.
|
|
--judge-tiebreak STRATEGY
|
|
Tie-break strategy surfaced into the pre-registration
|
|
payload. One of: quadri-vendor | pm-escalation | majority.
|
|
Default: quadri-vendor (per A3 LOCK § B2).
|
|
|
|
Task 2.5 Stage 1 substrate flags (2026-04-23):
|
|
--locomo-raw-path P
|
|
Path to raw LoCoMo archive (snap-research/locomo10.json).
|
|
REQUIRED when --cell is 'retrieval' or 'agentic'. Main()
|
|
builds an ephemeral :memory: MindDB + HybridSearch pair,
|
|
ingests every turn as one I-frame (frame-per-turn per
|
|
GATE-S0 lock), then tears it down at end of run.
|
|
--retrieval-top-k N
|
|
Top-K turn frames the retrieval cell recalls per question.
|
|
Default 10 (GATE-S0 lock). Range 1..100.
|
|
--agentic-max-turns N
|
|
Hard turn cap for the agentic cell's inner agent-loop.
|
|
Default 3 (GATE-S0 lock). Range 1..10.
|
|
--agentic-timeout-ms N
|
|
AbortController timeout for the agentic cell. Default
|
|
180_000 ms. Minimum 1000 ms.
|
|
|
|
--help, -h This text
|
|
`);
|
|
}
|
|
|
|
// ── Config loaders ─────────────────────────────────────────────────────────
|
|
|
|
function harnessRoot(): string {
|
|
// __filename equivalent for ESM. When compiled to dist/runner.js this
|
|
// resolves to the dist dir; when run via tsx it resolves to src/runner.ts.
|
|
const here = url.fileURLToPath(import.meta.url);
|
|
// `harness/dist/runner.js` → harness dir; `harness/src/runner.ts` → harness dir.
|
|
return path.resolve(path.dirname(here), '..');
|
|
}
|
|
|
|
function loadModels(): Record<string, ModelSpec> {
|
|
const cfg = path.join(harnessRoot(), 'config', 'models.json');
|
|
return JSON.parse(fs.readFileSync(cfg, 'utf-8')) as Record<string, ModelSpec>;
|
|
}
|
|
|
|
function loadDatasets(): Record<string, DatasetSpec> {
|
|
const cfg = path.join(harnessRoot(), 'config', 'datasets.json');
|
|
return JSON.parse(fs.readFileSync(cfg, 'utf-8')) as Record<string, DatasetSpec>;
|
|
}
|
|
|
|
// ── Run driver ─────────────────────────────────────────────────────────────
|
|
|
|
async function runOne(config: RunConfig): Promise<void> {
|
|
const startedAt = new Date().toISOString();
|
|
const dataRoot = path.join(harnessRoot(), '..', 'data');
|
|
// Sample-lock path, when provided, takes precedence over the dataset adapter
|
|
// and enforces the Stage 2 13/13/12/12 distribution inside
|
|
// loadPreflightSampleLock. Failure to match → throw before any LLM call,
|
|
// which is the Task-1 acceptance requirement.
|
|
const all = config.sampleLockPath
|
|
? loadPreflightSampleLock(
|
|
path.isAbsolute(config.sampleLockPath)
|
|
? config.sampleLockPath
|
|
: path.resolve(process.cwd(), config.sampleLockPath),
|
|
)
|
|
: loadDataset(config.dataset, dataRoot);
|
|
|
|
// Sprint 12 Task 1 Blocker #1: attach dataset_version (SHA-256 of canonical
|
|
// archive) to every emitted JSONL record. Sample-lock runs get the
|
|
// lock-file hash; regular runs get the dataset archive hash; synthetic
|
|
// runs get the static `synthetic-scaffold-v1` string. Computed once per
|
|
// runOne call to avoid per-instance disk I/O.
|
|
const datasetVersion = config.sampleLockPath
|
|
? computeFileHash(
|
|
path.isAbsolute(config.sampleLockPath)
|
|
? config.sampleLockPath
|
|
: path.resolve(process.cwd(), config.sampleLockPath),
|
|
)
|
|
: getDatasetVersion(config.dataset, dataRoot);
|
|
|
|
// Sprint 12 Task 1 Blocker #3: emit `bench.preregistration.manifest_hash`
|
|
// once per runOne before the first instance iteration. Suppressed when
|
|
// the caller sets `emitPreregistrationEvent: false` (test default).
|
|
if (config.emitPreregistrationEvent !== false) {
|
|
emitPreregistrationManifest(
|
|
assemblePreregistrationPayload(config, datasetVersion, all.length),
|
|
);
|
|
}
|
|
// When a sample lock drives the run, honor instance order verbatim — the
|
|
// lock file IS the deterministic sample. Cell comparisons require identical
|
|
// order across cells, and `sampleInstances` would re-shuffle the lock.
|
|
const sampled = config.sampleLockPath
|
|
? (Number.isFinite(config.limit) && config.limit < all.length ? all.slice(0, config.limit) : all.slice())
|
|
: sampleInstances(all, config.seed, config.limit);
|
|
|
|
const writer = new JsonlWriter(config.outputPath);
|
|
const llm = createLlmClient({
|
|
dryRun: config.dryRun,
|
|
litellmUrl: config.litellmUrl,
|
|
litellmApiKey: config.litellmApiKey,
|
|
});
|
|
|
|
let totalCost = 0;
|
|
let budgetStoppedAt: number | null = null;
|
|
const latenciesByInstance: number[] = [];
|
|
// §7.2 Stage 1.5 streak halt — track consecutive fetch-transport failures.
|
|
// Tracker is cell-local: each cell gets a fresh counter so a streak in the
|
|
// previous cell doesn't poison the next.
|
|
const streakTracker = new StreakTracker();
|
|
let streakHaltAt: number | null = null;
|
|
let streakHaltSummary: string | null = null;
|
|
|
|
for (let i = 0; i < sampled.length; i++) {
|
|
if (totalCost >= config.budgetUsd) {
|
|
budgetStoppedAt = i;
|
|
break;
|
|
}
|
|
const instance = sampled[i];
|
|
const turnId = generateTurnId();
|
|
|
|
// Task 2.5 Stage 1: cells and controls have diverged CellInput/ControlInput
|
|
// shapes — cells now accept optional substrate + litellm + per-cell knobs
|
|
// that controls don't need. Split the dispatch so each side gets only the
|
|
// fields it understands.
|
|
let result;
|
|
if (config.run.kind === 'cell') {
|
|
const cellFn = cells[config.run.name as CellName];
|
|
result = await cellFn({
|
|
instance,
|
|
model: config.model,
|
|
llm,
|
|
turnId,
|
|
substrate: config.substrate,
|
|
litellm: config.litellm,
|
|
retrievalTopK: config.retrievalTopK,
|
|
agenticMaxTurns: config.agenticMaxTurns,
|
|
agenticTimeoutMs: config.agenticTimeoutMs,
|
|
});
|
|
} else {
|
|
const controlFn = controls[config.run.name as ControlName];
|
|
result = await controlFn({ instance, model: config.model, llm, turnId });
|
|
}
|
|
latenciesByInstance.push(result.latencyMs);
|
|
|
|
const accuracy = result.failureMode ? 0 : scoreAccuracy(result.text, instance.expected);
|
|
totalCost += result.costUsd;
|
|
|
|
// Sprint 11 A2: emit structured llm.response event tagged with turnId.
|
|
// `reasoningShape` + char count become the canonical observability
|
|
// surface; reasoning content itself goes to JSONL only, never to logs,
|
|
// per design doc §2.3/§2.4 exclusion rules.
|
|
logTurnEvent(turnId, {
|
|
stage: 'llm.response',
|
|
cell: config.run.name,
|
|
model: config.model.id,
|
|
textChars: result.text.length,
|
|
latencyMs: result.latencyMs,
|
|
costUsd: result.costUsd,
|
|
failureMode: result.failureMode,
|
|
reasoningShape: result.reasoningShape ?? 'none',
|
|
reasoningChars: result.reasoningContent?.length ?? 0,
|
|
});
|
|
// Drift alarm per ratification §Q3: thinking=on was requested but no
|
|
// reasoning field present — observable without failing the run.
|
|
if (result.reasoningShape === 'unknown') {
|
|
logTurnEvent(turnId, {
|
|
stage: 'llm.response.reasoning_content_shape_unknown',
|
|
cell: config.run.name,
|
|
model: config.model.id,
|
|
litellmModel: config.model.litellmModel,
|
|
});
|
|
}
|
|
|
|
// Sprint 9 Task 2: when a judge is configured, grade the answer
|
|
// in-line. The judge runs even when the cell call itself failed
|
|
// (failureMode !== null) because the transcript still has value
|
|
// for calibration; the aggregator can filter if needed. A judge
|
|
// error never aborts the batch — runJudge swallows and annotates.
|
|
let judgePayload: import('./judge-runner.js').JudgePayload | null = null;
|
|
if (config.judgeConfig && !result.failureMode) {
|
|
judgePayload = await runJudge(
|
|
{
|
|
question: instance.question,
|
|
// scoreAccuracy picks the first expected answer; keep parity
|
|
// so the judge sees the same ground truth. Remaining entries
|
|
// in `expected` are alternate phrasings, not independent
|
|
// facts, and are irrelevant to the §4 judge prompt.
|
|
groundTruth: instance.expected[0] ?? '',
|
|
contextExcerpt: instance.context,
|
|
modelAnswer: result.text,
|
|
},
|
|
config.judgeConfig,
|
|
);
|
|
}
|
|
|
|
// p50 / p95 are computed over the running window of observed latencies
|
|
// so the JSONL record carries real-time percentiles (not flat per-row).
|
|
// Aggregate metrics.ts recomputes globals for the summary.
|
|
const record: JsonlRecord = {
|
|
turnId,
|
|
cell: config.run.name,
|
|
instance_id: instance.instance_id,
|
|
model: config.model.id,
|
|
seed: config.seed,
|
|
accuracy,
|
|
p50_latency_ms: percentile(latenciesByInstance, 50),
|
|
p95_latency_ms: percentile(latenciesByInstance, 95),
|
|
usd_per_query: round(result.costUsd, 6),
|
|
failure_mode: result.failureMode,
|
|
dataset_version: datasetVersion,
|
|
...(judgePayload && {
|
|
model_answer: judgePayload.model_answer,
|
|
judge_verdict: judgePayload.judge_verdict,
|
|
judge_failure_mode: judgePayload.judge_failure_mode,
|
|
judge_rationale: judgePayload.judge_rationale,
|
|
judge_model: judgePayload.judge_model,
|
|
judge_timestamp: judgePayload.judge_timestamp,
|
|
judge_ensemble: judgePayload.judge_ensemble,
|
|
// B2 fold-in observability fields.
|
|
...(judgePayload.tie_break_path !== undefined && { tie_break_path: judgePayload.tie_break_path }),
|
|
...(judgePayload.tie_break_fourth_vendor !== undefined && { tie_break_fourth_vendor: judgePayload.tie_break_fourth_vendor }),
|
|
// Sprint 12 Task 2 §2.1 A3 namespace split (LOCKED 2026-04-23) —
|
|
// authoritative A3 LOCK § 6 taxonomy columns. `undefined` a3_failure_code
|
|
// (e.g. PM_ESCALATION skipped instance) is spread conditionally so
|
|
// the absence of the key is preserved, matching the aggregator's
|
|
// "skipped" semantics.
|
|
...(judgePayload.a3_failure_code !== undefined && { a3_failure_code: judgePayload.a3_failure_code }),
|
|
...(judgePayload.a3_rationale !== undefined && { a3_rationale: judgePayload.a3_rationale }),
|
|
}),
|
|
// Sprint 11 A2: reasoning_content fields — same-row persistence
|
|
// keyed by `turnId` per ratification §Q4. Chars stay separate for
|
|
// aggregation even when content is stripped on the read path.
|
|
...(result.reasoningContent !== undefined && {
|
|
reasoning_content: result.reasoningContent,
|
|
reasoning_content_chars: result.reasoningContent.length,
|
|
}),
|
|
...(result.reasoningShape !== undefined && {
|
|
reasoning_shape: result.reasoningShape,
|
|
}),
|
|
// Sprint 12 Task 1 Blocker #3 / B3 addendum § 4: copy target-model
|
|
// pinning surface fields from `config/models.json` onto every row so
|
|
// any single JSONL line carries its own audit-replication anchor.
|
|
...(config.model.pinning_surface !== undefined && {
|
|
model_pinning_surface: config.model.pinning_surface,
|
|
model_pinning_carve_out_reason: config.model.pinning_surface_carve_out_reason ?? null,
|
|
}),
|
|
// B3 addendum § 4 leaves `model_revision_hash` null unless the provider
|
|
// exposes one. Future providers (e.g. OpenRouter with revision_id)
|
|
// will populate this via a provider-specific hook in Session 3+.
|
|
model_revision_hash: null,
|
|
};
|
|
writer.write(record);
|
|
|
|
// §7.2 Stage 1.5 streak halt — check AFTER writing the current record so
|
|
// the failure-forensic row persists. Break out of the loop; the error is
|
|
// thrown AFTER writer.close() below so JSONL + summary files flush
|
|
// cleanly. Caller (main) surfaces the error to the operator.
|
|
if (streakTracker.record(result.failureMode)) {
|
|
streakHaltAt = i + 1;
|
|
streakHaltSummary = streakTracker.summary();
|
|
break;
|
|
}
|
|
}
|
|
|
|
await writer.close();
|
|
const finishedAt = new Date().toISOString();
|
|
const summary = buildAggregate(config, writer.all(), startedAt, finishedAt, budgetStoppedAt);
|
|
const summaryPath = config.outputPath.replace(/\.jsonl$/, '.summary.json');
|
|
fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2), 'utf-8');
|
|
|
|
// Compact stdout summary — machine-parseable prefix makes CI grep easy.
|
|
console.log(
|
|
`[bench:summary] kind=${summary.run.kind} name=${summary.run.name} ` +
|
|
`model=${summary.run.model} seed=${summary.run.seed} ` +
|
|
`n=${summary.counts.total} completed=${summary.counts.completed} ` +
|
|
`failed=${summary.counts.failed} ` +
|
|
`accuracy=${summary.metrics.meanAccuracy} ` +
|
|
`p50=${summary.metrics.p50LatencyMs}ms p95=${summary.metrics.p95LatencyMs}ms ` +
|
|
`cost=$${summary.metrics.totalUsd} ` +
|
|
`jsonl=${config.outputPath}`,
|
|
);
|
|
|
|
// §7.2 Stage 1.5 — throw AFTER writer + summary flushed so partial data
|
|
// persists for forensic review (same pattern as budget-stop, except budget
|
|
// is a soft stop that doesn't throw). The surfaced error lets main()
|
|
// decide whether to continue other cells or bail the whole invocation.
|
|
if (streakHaltAt !== null) {
|
|
throw new Error(
|
|
`[bench:halt] cell '${config.run.name}' aborted at instance ${streakHaltAt}/${sampled.length} ` +
|
|
`due to consecutive fetch-transport failures (${streakHaltSummary}). ` +
|
|
`Likely upstream saturation — check LiteLLM proxy + provider health before re-running. ` +
|
|
`Partial JSONL persisted at ${config.outputPath}.`,
|
|
);
|
|
}
|
|
}
|
|
|
|
function round(n: number, decimals: number): number {
|
|
const f = Math.pow(10, decimals);
|
|
return Math.round(n * f) / f;
|
|
}
|
|
|
|
/** SHA-256 hex of a file's bytes. Used for sample-lock version stamping
|
|
* when the run is driven by a committed lock file rather than the regular
|
|
* dataset archive. */
|
|
function computeFileHash(absolutePath: string): string {
|
|
const buf = fs.readFileSync(absolutePath);
|
|
return crypto.createHash('sha256').update(buf).digest('hex');
|
|
}
|
|
|
|
/**
|
|
* Resolve the CLI-requested judge-model roster against `config/models.json`
|
|
* so every pre-registration emit carries B3-addendum pinning fields. Keeps
|
|
* the registry lookup in main() — runOne only forwards the materialised
|
|
* list.
|
|
*
|
|
* Unknown model ids are passed through with `floating_alias` defaults and
|
|
* an explicit carve-out reason that names the ID. This keeps the run going
|
|
* while loudly surfacing the gap to downstream audit consumers.
|
|
*/
|
|
function resolveJudgeModelsForPreregistration(
|
|
models: Record<string, ModelSpec>,
|
|
args: ParsedArgs,
|
|
): JudgeModelManifestEntry[] {
|
|
const ids: string[] = [];
|
|
if (args.judgeEnsemble && args.judgeEnsemble.length > 0) {
|
|
ids.push(...args.judgeEnsemble);
|
|
} else if (args.judge) {
|
|
ids.push(args.judge);
|
|
}
|
|
return ids.map((id, idx) => {
|
|
const spec = models[id];
|
|
if (!spec) {
|
|
return {
|
|
model_id: id,
|
|
provider: 'unknown',
|
|
judge_role: (idx === 0 ? 'primary' : idx === 1 ? 'secondary' : 'tertiary'),
|
|
pinning_surface: 'floating_alias' as const,
|
|
pinning_surface_carve_out_reason: `Model id '${id}' not found in benchmarks/harness/config/models.json — pinning surface cannot be resolved; treating as floating_alias for audit safety`,
|
|
};
|
|
}
|
|
return {
|
|
model_id: spec.id,
|
|
provider: spec.provider,
|
|
judge_role: spec.judge_role ?? (idx === 0 ? 'primary' : idx === 1 ? 'secondary' : 'tertiary'),
|
|
pinning_surface: spec.pinning_surface ?? 'floating_alias',
|
|
pinning_surface_carve_out_reason: spec.pinning_surface_carve_out_reason ?? null,
|
|
};
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Build the `PreregistrationManifestPayload` from the RunConfig + derived
|
|
* per-run context. Split out from `runOne` so tests can exercise the
|
|
* assembly without spinning up a full run.
|
|
*
|
|
* Judge models surfacing: in Sub-deliverable A the list is derived from
|
|
* `config.judgeConfig.models` (ensemble) or `config.judgeConfig.model`
|
|
* (single-judge), with placeholder pinning fields. Sub-deliverable C
|
|
* tightens per-judge `pinning_surface` + `carve_out_reason` by looking
|
|
* each model up in `config/models.json`. Until C lands, judges emit as
|
|
* `floating_alias` + null carve-out (schema-valid placeholder).
|
|
*/
|
|
function assemblePreregistrationPayload(
|
|
config: RunConfig,
|
|
datasetVersion: string,
|
|
datasetInstanceCount: number,
|
|
): PreregistrationManifestPayload {
|
|
const manifestHash = config.manifestHash ?? computeBenchSpecManifestHash();
|
|
// `readManifestLockedDate` throws if the YAML can't be located and no
|
|
// override is provided. If the caller supplied `manifestHash` explicitly
|
|
// (test path), we still try to resolve the YAML for `locked_at`; if
|
|
// resolution fails under that branch, fall back to 'unknown' so tests
|
|
// that don't carry the YAML keep working.
|
|
let manifestLockedAt: string;
|
|
try {
|
|
manifestLockedAt = readManifestLockedDate();
|
|
} catch {
|
|
manifestLockedAt = 'unknown';
|
|
}
|
|
|
|
const perCell = config.perCellList ?? [config.run.name];
|
|
|
|
// Sub-deliverable C: `judgeModelsResolved` is pre-materialised in main()
|
|
// from args.judgeEnsemble against config/models.json. When judging is
|
|
// disabled the list is empty (schema-valid per brief § 3.1).
|
|
const judgeModels: JudgeModelManifestEntry[] = config.judgeModelsResolved ?? [];
|
|
|
|
return {
|
|
manifest_hash: manifestHash,
|
|
manifest_path: CANONICAL_MANIFEST_PATH,
|
|
manifest_locked_at: manifestLockedAt,
|
|
dataset_version: datasetVersion,
|
|
dataset_path: config.dataset.dataPath,
|
|
dataset_instance_count: datasetInstanceCount,
|
|
per_cell: perCell,
|
|
judge_tiebreak: config.judgeTiebreak ?? 'quadri-vendor',
|
|
judge_models: judgeModels,
|
|
emitted_at: new Date().toISOString(),
|
|
runner_version: getRunnerVersion(),
|
|
runner_invocation: {
|
|
argv: sanitizeArgv(process.argv),
|
|
cwd: process.cwd(),
|
|
},
|
|
};
|
|
}
|
|
|
|
// ── Main ───────────────────────────────────────────────────────────────────
|
|
|
|
function buildRuns(args: ParsedArgs): RunKind[] {
|
|
const runs: RunKind[] = [];
|
|
// Sprint 12 Task 1 Blocker #3: `--per-cell` (multi-value) overrides the
|
|
// single-cell / all-cells dispatch when at least one value is supplied.
|
|
// Sprint 12 Task 2.5 Stage 2-Retry §1.5: valid-name list extended to
|
|
// include retrieval + agentic (Stage 1) + no-context (Stage 2-Retry).
|
|
// `--all-cells` is preserved at the Sprint 9 quartet for backward
|
|
// compatibility per PM ratification; `scripts/run-mini-locomo.ts
|
|
// --v3-cells` expands to the Stage 2-Retry 5-cell list via --per-cell.
|
|
const VALID_CELLS = 'raw | filtered | compressed | full-context | retrieval | agentic | no-context';
|
|
if (args.perCell && args.perCell.length > 0) {
|
|
for (const name of args.perCell) {
|
|
if (!isCellName(name)) {
|
|
throw new Error(`Unknown cell: ${name}. Valid: ${VALID_CELLS}`);
|
|
}
|
|
runs.push({ kind: 'cell', name });
|
|
}
|
|
return runs;
|
|
}
|
|
if (args.allCells) {
|
|
(['raw', 'filtered', 'compressed', 'full-context'] as CellName[]).forEach(n => {
|
|
runs.push({ kind: 'cell', name: n });
|
|
});
|
|
} else if (args.control) {
|
|
if (!isControlName(args.control)) {
|
|
throw new Error(`Unknown control: ${args.control}. Valid: verbose-fixed`);
|
|
}
|
|
runs.push({ kind: 'control', name: args.control });
|
|
} else if (args.cell) {
|
|
if (!isCellName(args.cell)) {
|
|
throw new Error(`Unknown cell: ${args.cell}. Valid: ${VALID_CELLS}`);
|
|
}
|
|
runs.push({ kind: 'cell', name: args.cell });
|
|
} else {
|
|
throw new Error('Must specify one of --cell <name>, --all-cells, --per-cell <name>, or --control <name>. Try --help.');
|
|
}
|
|
return runs;
|
|
}
|
|
|
|
function defaultOutputPath(kind: RunKind, dataset: string): string {
|
|
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
|
const resultsDir = path.join(harnessRoot(), '..', 'results');
|
|
return path.join(resultsDir, `${kind.name}-${dataset}-${ts}.jsonl`);
|
|
}
|
|
|
|
async function main(): Promise<void> {
|
|
const args = parseArgs(process.argv.slice(2));
|
|
const models = loadModels();
|
|
const datasets = loadDatasets();
|
|
|
|
const model = models[args.model];
|
|
if (!model) {
|
|
throw new Error(`Unknown model: ${args.model}. Valid ids: ${Object.keys(models).join(', ')}`);
|
|
}
|
|
const dataset = datasets[args.dataset];
|
|
if (!dataset) {
|
|
throw new Error(`Unknown dataset: ${args.dataset}. Valid ids: ${Object.keys(datasets).join(', ')}`);
|
|
}
|
|
|
|
const dryRun = args.dryRun ?? !process.env.LITELLM_URL;
|
|
const litellmUrl = process.env.LITELLM_URL ?? 'http://localhost:4000';
|
|
const litellmApiKey = process.env.LITELLM_API_KEY ?? 'sk-waggle-dev';
|
|
|
|
// Build judge config (Sprint 9 Task 2). `--judge-ensemble` wins when
|
|
// both flags are present. In dry-run mode judge is force-skipped —
|
|
// calling a real LiteLLM judge while the cell path is stubbed would
|
|
// produce mixed-signal JSONL that's hard to interpret.
|
|
const judgeCosts: JudgeClientCostEntry[] = [];
|
|
let judgeConfig: JudgeConfig | undefined;
|
|
if (!dryRun) {
|
|
if (args.judgeEnsemble && args.judgeEnsemble.length > 0) {
|
|
const clients = new Map<string, import('./judge-client.js').LlmClient>();
|
|
for (const m of args.judgeEnsemble) {
|
|
clients.set(
|
|
m,
|
|
createJudgeLlmClient({
|
|
litellmUrl,
|
|
litellmApiKey,
|
|
model: m,
|
|
onCall: entry => judgeCosts.push(entry),
|
|
}),
|
|
);
|
|
}
|
|
|
|
// Sprint 11 B2 fold-in (2026-04-22): when the primary ensemble is
|
|
// exactly 3 vendors (the Sprint 10 Task 2.2 ratified trio — Opus 4.7
|
|
// + GPT-5.4 + Gemini 3.1-Pro), auto-wire xai/grok-4.20 as the
|
|
// fourth-vendor tie-breaker per decisions/2026-04-22-tie-break-policy-locked.md.
|
|
// Skip wiring when: (a) the ensemble isn't 3-vendor, (b) grok-4.20 is
|
|
// already in the primary list (would create dual-role ambiguity).
|
|
//
|
|
// Sprint 12 Task 2 §2.3 (2026-04-23): split the models.json key (used
|
|
// to construct the LiteLLM client with the correct route resolution)
|
|
// from the OpenRouter canonical audit slug (used as the
|
|
// `tie_break_fourth_vendor` JSONL column + pino `fourth_vendor_slug`).
|
|
// Without the split, the JSONL audit log would report the harness-
|
|
// local key `'grok-4.20'` and diverge from the §2.2-verified
|
|
// OpenRouter slug `x-ai/grok-4.20`.
|
|
const TIE_BREAKER_MODEL_KEY = 'grok-4.20'; // models.json key (LiteLLM route)
|
|
const TIE_BREAKER_AUDIT_SLUG = 'x-ai/grok-4.20'; // OpenRouter canonical (§2.2 verified 2026-04-23)
|
|
const shouldWireTieBreaker =
|
|
args.judgeEnsemble.length === 3 && !args.judgeEnsemble.includes(TIE_BREAKER_MODEL_KEY);
|
|
let tieBreakerClient: import('./judge-client.js').LlmClient | undefined;
|
|
if (shouldWireTieBreaker) {
|
|
tieBreakerClient = createJudgeLlmClient({
|
|
litellmUrl,
|
|
litellmApiKey,
|
|
model: TIE_BREAKER_MODEL_KEY,
|
|
onCall: entry => judgeCosts.push(entry),
|
|
});
|
|
}
|
|
|
|
judgeConfig = {
|
|
kind: 'ensemble',
|
|
models: args.judgeEnsemble,
|
|
clients,
|
|
...(shouldWireTieBreaker && tieBreakerClient && {
|
|
tieBreakerModel: TIE_BREAKER_AUDIT_SLUG,
|
|
tieBreakerClient,
|
|
}),
|
|
};
|
|
} else if (args.judge) {
|
|
judgeConfig = {
|
|
kind: 'single',
|
|
model: args.judge,
|
|
client: createJudgeLlmClient({
|
|
litellmUrl,
|
|
litellmApiKey,
|
|
model: args.judge,
|
|
onCall: entry => judgeCosts.push(entry),
|
|
}),
|
|
};
|
|
}
|
|
}
|
|
|
|
const runs = buildRuns(args);
|
|
// Sprint 12 Task 1 Blocker #3: invocation-level cell list fed into every
|
|
// runOne so the pre-registration payload reports the full scope rather
|
|
// than a single runOne's slice. Stripping kind='control' entries from the
|
|
// list would hide verbose-fixed-only invocations; report them as-is.
|
|
const perCellList = runs.map(r => r.name);
|
|
// Sub-deliverable C: materialise judge-model roster with B3 addendum
|
|
// pinning fields so every pre-registration emit carries audit-anchor
|
|
// metadata without repeating the models.json lookup per runOne.
|
|
const judgeModelsResolved = resolveJudgeModelsForPreregistration(models, args);
|
|
|
|
// Sprint 12 Task 2.5 Stage 1: when the cell roster includes retrieval or
|
|
// agentic, build an ephemeral substrate and pre-ingest every LoCoMo turn as
|
|
// an I-frame (frame-per-turn per GATE-S0 lock). Substrate is shared across
|
|
// every cell in this invocation so the corpus is indexed once, not per-cell.
|
|
// Lifecycle is owned here; `close()` fires in the `finally` block below.
|
|
const substrateNeeded = runs.some(
|
|
r => r.kind === 'cell' && (r.name === 'retrieval' || r.name === 'agentic'),
|
|
);
|
|
let substrate: Substrate | null = null;
|
|
let runnerLock: LockHandle | null = null;
|
|
try {
|
|
// §7.4 Stage 1.5 — acquire the global single-runner lock FIRST. Fail
|
|
// fast before any substrate ingest or health-probe cost is spent.
|
|
// Sentinel lives under benchmarks/results/ so any two runner invocations
|
|
// contend regardless of their --output paths.
|
|
const lockSentinel = path.join(harnessRoot(), '..', 'results', '.benchmark-runner');
|
|
runnerLock = acquireRunnerLock(lockSentinel);
|
|
console.log(`[bench:lock] acquired ${runnerLock.lockPath} (pid=${process.pid})`);
|
|
|
|
// §7.3 Stage 1.5 — pre-cell health check. Probes /health/liveliness + a
|
|
// short ping call per (subject + judge ensemble). Skipped in dry-run
|
|
// mode (no upstream to probe). Any 5xx or network error aborts before
|
|
// any instance budget is spent.
|
|
if (!dryRun) {
|
|
const judgePingModels = args.judgeEnsemble
|
|
? [...args.judgeEnsemble]
|
|
: (args.judge ? [args.judge] : []);
|
|
const hc = await preCellHealthCheck({
|
|
litellmUrl,
|
|
litellmApiKey,
|
|
subjectModel: model.litellmModel,
|
|
judgeModels: judgePingModels,
|
|
});
|
|
if (!hc.ok) {
|
|
const summary = hc.failures.map(f => `${f.endpoint} → ${f.error}`).join('; ');
|
|
throw new Error(
|
|
`[bench:health-check] FAILED in ${hc.durationMs}ms against ${litellmUrl}: ${summary}. ` +
|
|
`Aborting before any instance budget is spent. Check LiteLLM proxy + provider keys.`,
|
|
);
|
|
}
|
|
console.log(
|
|
`[bench:health-check] OK — probed subject + ${judgePingModels.length} judge(s) in ${hc.durationMs}ms`,
|
|
);
|
|
}
|
|
|
|
if (substrateNeeded) {
|
|
if (dryRun) {
|
|
// Retrieval + agentic aren't meaningful in dry-run mode (the agent
|
|
// loop would 404 against the stub LLM, and the embedder would still
|
|
// need a live Ollama server). Fail loudly rather than ship silently
|
|
// broken results.
|
|
throw new Error(
|
|
'retrieval/agentic cells cannot run in dry-run mode. Remove --dry-run ' +
|
|
'or set LITELLM_URL to activate live mode.',
|
|
);
|
|
}
|
|
if (!args.locomoRawPath) {
|
|
throw new Error(
|
|
'retrieval/agentic cells require --locomo-raw-path <path>. ' +
|
|
'Point it at the raw snap-research/locomo10.json archive.',
|
|
);
|
|
}
|
|
const rawPath = path.isAbsolute(args.locomoRawPath)
|
|
? args.locomoRawPath
|
|
: path.resolve(process.cwd(), args.locomoRawPath);
|
|
substrate = createSubstrate(); // default: :memory: + ollama-embedder
|
|
const substrateStart = Date.now();
|
|
const turns = extractTurnsFromLocomoRaw(rawPath);
|
|
const stats = await ingestLoCoMoCorpus(
|
|
substrate.db, substrate.search, substrate.frames, substrate.sessions, turns,
|
|
);
|
|
console.log(
|
|
`[bench:substrate] turns_found=${turns.length} frames_created=${stats.count} ` +
|
|
`ingest_ms=${stats.ingestMs} index_ms=${stats.indexMs} total_ms=${Date.now() - substrateStart}`,
|
|
);
|
|
}
|
|
|
|
for (const run of runs) {
|
|
const outputPath = args.output ?? defaultOutputPath(run, args.dataset);
|
|
await runOne({
|
|
run,
|
|
dataset,
|
|
model,
|
|
limit: args.limit,
|
|
seed: args.seed,
|
|
budgetUsd: args.budget,
|
|
outputPath,
|
|
dryRun,
|
|
litellmUrl,
|
|
litellmApiKey,
|
|
sampleLockPath: args.sampleLock,
|
|
judgeConfig,
|
|
onJudgeCall: entry => judgeCosts.push(entry),
|
|
manifestHash: args.manifestHash,
|
|
emitPreregistrationEvent: args.emitPreregistrationEvent,
|
|
perCellList,
|
|
judgeTiebreak: args.judgeTiebreak,
|
|
judgeModelsResolved,
|
|
// Task 2.5 Stage 1 — substrate deps (undefined for non-substrate cells)
|
|
substrate: substrate ?? undefined,
|
|
litellm: { url: litellmUrl, apiKey: litellmApiKey },
|
|
retrievalTopK: args.retrievalTopK,
|
|
agenticMaxTurns: args.agenticMaxTurns,
|
|
agenticTimeoutMs: args.agenticTimeoutMs,
|
|
});
|
|
}
|
|
|
|
// Surface judge spend separately from cell spend so the brief's "$5
|
|
// alarm per run" guardrail applies cleanly to the judge-layer budget.
|
|
if (judgeCosts.length > 0) {
|
|
const judgeTotalUsd = judgeCosts.reduce((sum, e) => sum + e.usd, 0);
|
|
const judgeOk = judgeCosts.filter(e => e.ok).length;
|
|
console.log(
|
|
`[bench:judge-summary] calls=${judgeCosts.length} ok=${judgeOk} ` +
|
|
`failed=${judgeCosts.length - judgeOk} total_usd=$${judgeTotalUsd.toFixed(6)}`,
|
|
);
|
|
}
|
|
} finally {
|
|
if (substrate) {
|
|
substrate.close();
|
|
}
|
|
if (runnerLock) {
|
|
runnerLock.release();
|
|
}
|
|
}
|
|
}
|
|
|
|
// Only run main() when invoked directly (not when imported by tests).
|
|
const isMain =
|
|
typeof process !== 'undefined' &&
|
|
Array.isArray(process.argv) &&
|
|
process.argv[1] !== undefined &&
|
|
url.fileURLToPath(import.meta.url) === path.resolve(process.argv[1]);
|
|
|
|
if (isMain) {
|
|
main().catch(err => {
|
|
console.error('[bench:error]', err?.message ?? err);
|
|
process.exit(1);
|
|
});
|
|
}
|
|
|
|
// Exported for the smoke tests.
|
|
export { main, parseArgs, buildRuns, runOne, defaultOutputPath };
|