moving
Some checks failed
Installer Smoke / installer-smoke (push) Has been cancelled

This commit is contained in:
Oleg Maslov
2026-09-02 10:10:29 +02:00
commit 0c3e2ead3b
3841 changed files with 970576 additions and 0 deletions

View File

@@ -0,0 +1,110 @@
/**
* BEAM 1M — per-conversation content→date map for the v2 answer prompt.
*
* The raw-turn minds (minds-1M) store each frame's content as
* `"${role}: ${text}"` with NO dates. BEAM's chat.json anchors each SESSION
* (batch) with a SINGLE `time_anchor` on its opening `main_question`; every
* later turn in that session inherits it. This module rebuilds, per
* conversation, a Map from the exact frame-content string to an ISO date so the
* v2 path in beam-run-1m.ts can prefix each retrieved memory with
* `"[YYYY-MM-DD] role: ..."` — giving the contradiction / temporal rules real
* dates to reason over. v1 never calls this (byte-identical undated behaviour).
*
* Keying MUST match beam-ingest-1m.ts::flattenChatJson + ingest-beam.ts exactly:
* key = `${String(msg.role).toLowerCase()}: ${String(msg.content).trim()}`
* for every NON-EMPTY message, walked in batches → turn-groups → messages order.
*/
import fs from 'node:fs';
const MONTHS: Record<string, string> = {
january: '01', february: '02', march: '03', april: '04', may: '05', june: '06',
july: '07', august: '08', september: '09', october: '10', november: '11', december: '12',
};
/** "March-01-2024" → "2024-03-01". Returns null when unparseable. */
export function normalizeTimeAnchor(raw: string | null | undefined): string | null {
if (!raw) return null;
const parts = String(raw).trim().split('-');
if (parts.length !== 3) return null;
const mm = MONTHS[parts[0].toLowerCase()];
const dd = parts[1].padStart(2, '0');
const yyyy = parts[2];
if (!mm || !/^\d{4}$/.test(yyyy) || !/^\d{2}$/.test(dd)) return null;
return `${yyyy}-${mm}-${dd}`;
}
interface RawMsg { role?: string; content?: string; time_anchor?: string }
interface RawBatch { turns?: RawMsg[][]; time_anchor?: string }
/**
* Build a content→isoDate map for one conversation's chat.json.
*
* Dates propagate FORWARD: the last-seen anchor (batch-level, else message-level
* on the session's opening question) applies to every subsequent turn until the
* next anchor. The FIRST occurrence of a given content string wins, mirroring
* the ingest dedup (which keeps the first frame for duplicate content).
*/
export function buildConvDateMap(chatJsonPath: string): Map<string, string> {
const map = new Map<string, string>();
if (!fs.existsSync(chatJsonPath)) return map;
const batches = JSON.parse(fs.readFileSync(chatJsonPath, 'utf-8')) as RawBatch[];
let current: string | null = null;
for (const batch of batches) {
const batchDate = normalizeTimeAnchor(batch.time_anchor);
if (batchDate) current = batchDate;
if (!Array.isArray(batch.turns)) continue;
for (const group of batch.turns) {
if (!Array.isArray(group)) continue;
for (const msg of group) {
const role = String(msg.role ?? 'unknown').toLowerCase();
const content = String(msg.content ?? '').trim();
if (!content) continue;
const msgDate = normalizeTimeAnchor(msg.time_anchor);
if (msgDate) current = msgDate;
if (current === null) continue; // no anchor seen yet → leave undated
const key = `${role}: ${content}`;
if (!map.has(key)) map.set(key, current);
}
}
}
return map;
}
/**
* v2 memory rendering: prefix `"[date] "` when the map has the content AND it is
* not already bracketed (distilled-fact minds already embed `[YYYY-MM-DD]` — do
* not double-stamp). For v1 (or a null map) memories are returned unchanged, so
* the v1 path stays byte-identical to the original undated behaviour.
*/
export function renderMemories(
memories: string[],
dateMap: Map<string, string> | null,
prompt: 'v1' | 'v2',
): string[] {
if (prompt !== 'v2' || !dateMap) return memories;
return memories.map(m => {
if (m.startsWith('[')) return m; // already dated (e.g. distilled facts)
const d = dateMap.get(m);
return d ? `[${d}] ${m}` : m;
});
}
/**
* Coverage of a set of mind frame-contents by the date map — for the per-conv
* hit-rate log. Already-bracketed contents are excluded from the denominator
* (they are pre-dated and never stamped).
*/
export function computeDateHitRate(
contents: readonly string[],
dateMap: Map<string, string>,
): { dated: number; total: number } {
let dated = 0;
let total = 0;
for (const c of contents) {
if (c.startsWith('[')) continue;
total++;
if (dateMap.has(c)) dated++;
}
return { dated, total };
}

View File

@@ -0,0 +1,95 @@
/**
* BEAM 1M — "hybrid" retrieval merge (Option B).
*
* mem0's 0.641 comes from retrieving compact DATED FACTS presented
* CHRONOLOGICALLY (oldest-first). Our raw-turn cell wins on detail abilities but
* loses on summarization / event_ordering / instruction / preference. The hybrid
* cell combines BOTH sources into ONE context sorted chronologically:
*
* - a few RAW turns (minds-1M, content = "user: …" / "assistant: …",
* undated — dates come from the per-conv date-map sidecar)
* - many FACTS (minds-1M-obs, content already begins "[YYYY-MM-DD] fact")
*
* `mergeHybrid` is a PURE function: it takes the two already-fetched result sets
* plus the raw-turn date-map and returns a single list of entries, each reduced
* to `{ date, text }` where `text` NEVER carries a leading date bracket. Rendering
* `"[date] text"` therefore stamps every entry EXACTLY ONCE (facts are
* bracket-stripped first, so no double-bracket; raw turns get their date from the
* map). The caller passes `displayStrings` straight to the v2 answer prompt,
* bypassing the date-map re-stamp in beam-date-map.ts.
*
* SORT: chronological, oldest-first, by ISO date string. The sort is made fully
* deterministic with an explicit insertion-order tiebreaker (`order`): raw turns
* are inserted before facts, so on an equal date a raw turn precedes a fact, and
* within one kind the retrieval order is preserved. Undated entries carry date ''
* which is lexically smallest, so they sort FIRST (and are counted as date
* misses for the raw-turn hit-rate).
*/
import type { SearchResult } from '@waggle/core';
/** Leading "[YYYY-MM-DD] " on a distilled fact. */
const FACT_DATE_RE = /^\[(\d{4}-\d{2}-\d{2})\]\s*/;
export interface HybridEntry {
/** ISO 'YYYY-MM-DD', or '' when no date is known (sorts first). */
date: string;
/** Display text WITHOUT any leading date bracket. */
text: string;
kind: 'raw' | 'fact';
/** Insertion index — deterministic tiebreaker for equal dates. */
order: number;
}
export interface HybridMerge {
/** Final, already-dated display lines, oldest-first: "[date] text" (or bare
* `text` when undated). Pass straight to buildAnswerGenerationPromptV2. */
displayStrings: string[];
entries: HybridEntry[];
/** Raw turns whose content was found in the date-map (dated). */
rawDated: number;
/** Total raw turns retrieved (rawDated / rawTotal = raw-date hit-rate). */
rawTotal: number;
}
/**
* Merge raw turns + distilled facts into one chronologically-sorted, singly-dated
* list. Pure — no I/O. `dateMap` is the per-conv content→ISO-date map built by
* beam-date-map.ts (keyed by the exact raw frame content "role: content").
*/
export function mergeHybrid(
rawResults: readonly SearchResult[],
factResults: readonly SearchResult[],
dateMap: Map<string, string> | null,
): HybridMerge {
const entries: HybridEntry[] = [];
let order = 0;
let rawDated = 0;
const rawTotal = rawResults.length;
// Raw turns: content is "user: …" / "assistant: …" (no date). Date, if any,
// comes from the date-map keyed by the exact frame content.
for (const r of rawResults) {
const content = r.frame.content;
const date = dateMap?.get(content) ?? '';
if (date) rawDated++;
entries.push({ date, text: content, kind: 'raw', order: order++ });
}
// Facts: content already begins "[YYYY-MM-DD] fact" — strip the bracket so the
// single render step below re-applies exactly one "[date] " prefix.
for (const r of factResults) {
const content = r.frame.content;
const m = FACT_DATE_RE.exec(content);
const date = m ? m[1] : '';
const text = m ? content.slice(m[0].length) : content;
entries.push({ date, text, kind: 'fact', order: order++ });
}
// Stable, deterministic chronological sort (oldest-first). Equal dates keep
// insertion order → raw-before-fact, then retrieval order within a kind.
entries.sort((a, b) => (a.date === b.date ? a.order - b.order : a.date < b.date ? -1 : 1));
const displayStrings = entries.map(e => (e.date ? `[${e.date}] ${e.text}` : e.text));
return { displayStrings, entries, rawDated, rawTotal };
}

View File

@@ -0,0 +1,104 @@
/**
* BEAM metric aggregation — faithful port of mem0's `compute_beam_metrics`
* (benchmarks/beam/run.py). Produces the two headline numbers the BEAM
* leaderboard reports, plus the per-ability breakdown:
*
* - Avg Score : MICRO mean of per-question scores over all questions. This is
* the "64.1" number (mem0 1M = 0.641). Because BEAM tracks are
* ability-balanced (70/ability @ 1M), micro == macro, but we
* compute micro to match mem0 byte-for-byte.
* - Pass Rate : fraction of questions with score >= 0.5 (mem0 threshold),
* reported as a percentage. mem0 1M = 70.1%.
*
* The per-question `score` is the plain nugget-mean for EVERY ability (see
* beam-nugget-judge.ts) — event_ordering's tau-b blend is NOT aggregated here,
* matching mem0's headline.
*/
export const BEAM_PASS_THRESHOLD = 0.5;
export interface BeamQuestionResult {
instanceId: string;
memoryAbility: string;
/** Per-question headline score (nugget-mean), 0..1. */
score: number;
/** Present when the question could not be scored (e.g. no rubric). */
error?: string;
}
export interface BeamAbilityMetric {
total: number;
correct: number;
/** Pass rate as a percentage (0..100). */
accuracy: number;
/** Mean score (0..1). */
avgScore: number;
}
export interface BeamMetrics {
overall: {
total: number;
correct: number;
errors: number;
/** Pass rate as a percentage (0..100). */
accuracy: number;
/** Micro-averaged Avg Score (0..1) — the headline BEAM number. */
avgScore: number;
};
byAbility: Record<string, BeamAbilityMetric>;
}
function mean(xs: number[]): number {
return xs.length === 0 ? 0 : xs.reduce((s, x) => s + x, 0) / xs.length;
}
/** Compute overall + per-ability BEAM metrics (single retrieval cutoff). */
export function computeBeamMetrics(results: BeamQuestionResult[]): BeamMetrics {
const scores = results.map(r => r.score);
const total = scores.length;
const correct = scores.filter(s => s >= BEAM_PASS_THRESHOLD).length;
const errors = results.filter(r => r.error).length;
const byAbility: Record<string, BeamAbilityMetric> = {};
const abilities = [...new Set(results.map(r => r.memoryAbility))].sort();
for (const ability of abilities) {
const items = results.filter(r => r.memoryAbility === ability);
const abScores = items.map(r => r.score);
const abCorrect = abScores.filter(s => s >= BEAM_PASS_THRESHOLD).length;
byAbility[ability] = {
total: items.length,
correct: abCorrect,
accuracy: items.length > 0 ? (abCorrect / items.length) * 100 : 0,
avgScore: mean(abScores),
};
}
return {
overall: {
total,
correct,
errors,
accuracy: total > 0 ? (correct / total) * 100 : 0,
avgScore: mean(scores),
},
byAbility,
};
}
/** Render metrics as a compact human-readable table (for console/logs). */
export function formatBeamMetrics(m: BeamMetrics): string {
const lines: string[] = [];
lines.push(
`OVERALL avg_score=${m.overall.avgScore.toFixed(4)} ` +
`pass_rate=${m.overall.accuracy.toFixed(1)}% (${m.overall.correct}/${m.overall.total}) ` +
`errors=${m.overall.errors}`,
);
const rows = Object.entries(m.byAbility).sort((a, b) => b[1].avgScore - a[1].avgScore);
for (const [ability, v] of rows) {
lines.push(
` ${ability.padEnd(26)} avg_score=${v.avgScore.toFixed(3)} ` +
`pass=${v.correct}/${v.total} (${v.accuracy.toFixed(1)}%)`,
);
}
return lines.join('\n');
}

View File

@@ -0,0 +1,583 @@
/**
* BEAM graded "nugget" judge — a faithful TypeScript port of mem0's
* `benchmarks/beam/prompts.py` + the scoring logic in `benchmarks/beam/run.py`
* (github.com/mem0ai/memory-benchmarks). This is the judge behind the published
* BEAM SOTA (Avg Score 0.641 @ 1M), so replicating it exactly is what makes our
* numbers comparable to that leaderboard.
*
* KEY PROTOCOL FACTS (pinned from the mem0 source):
* - Each probing question carries a `rubric`: an ordered list of "nuggets".
* - Each nugget is judged INDEPENDENTLY on a 3-point scale {0.0, 0.5, 1.0}.
* - The per-question score = arithmetic MEAN of its nugget scores.
* - "Pass" = per-question score >= 0.5. "Avg Score" = mean of question scores.
* - event_ordering ALSO computes a Kendall tau-b blend into `score_with_tau`,
* but — verified against mem0's `compute_beam_metrics` — the HEADLINE Avg
* Score aggregates the plain nugget-mean `score` for EVERY ability,
* including event_ordering. `score_with_tau` is an auxiliary diagnostic and
* is NOT what 0.641 measures. We preserve that behaviour here.
*
* The judge itself is transport-agnostic: it takes a `BeamLlm` (see
* beam-openai-client.ts for the gpt-4o implementation the official protocol
* uses). Metrics aggregation lives in beam-metrics.ts.
*/
// ── LLM transport contract ─────────────────────────────────────────────────
export interface BeamLlmResult {
text: string;
inputTokens: number;
outputTokens: number;
costUsd: number;
latencyMs: number;
/** null = OK; otherwise a short failure classification. */
failureMode: string | null;
}
export interface BeamLlm {
/**
* Single-turn completion. When `jsonMode` is true the client should ask the
* provider for a JSON object (OpenAI `response_format: {type:'json_object'}`)
* so the judge's `{score, reason}` parses reliably.
*/
chat(opts: { system: string; user: string; jsonMode?: boolean; maxTokens?: number }): Promise<BeamLlmResult>;
}
// ── Prompt constants (verbatim from mem0 prompts.py) ────────────────────────
export const BEAM_JUDGE_SYSTEM_PROMPT =
'You are an expert evaluator assessing whether an AI assistant\'s response satisfies ' +
'specific rubric criteria. You must be objective, fair, and consistent. ' +
'Return ONLY valid JSON with the exact format requested.';
/** Build the single-nugget judge prompt (mem0 `get_beam_nugget_judge_prompt`). */
export function buildNuggetJudgePrompt(question: string, nugget: string, llmResponse: string): string {
return `Evaluate whether the following LLM response demonstrates compliance with the specified RUBRIC CRITERION.
QUESTION:
${question}
LLM RESPONSE:
${llmResponse}
RUBRIC CRITERION:
${nugget}
SCORING GUIDELINES:
First, determine whether the rubric criterion is a POSITIVE requirement (the response SHOULD include something) or a NEGATIVE constraint (the response SHOULD NOT include something).
**For POSITIVE requirements** (response should contain, mention, or demonstrate something):
- **1.0 (Complete Compliance)**: The required element is present, accurate, and complete. The response fully and clearly satisfies the rubric criterion.
- **0.5 (Partial Compliance)**: The required element is partially present, has minor inaccuracies, or is incomplete. The core intent is present but not fully realized.
- **0.0 (No Compliance)**: The required element is missing, incorrect, or the response is entirely off-topic / non-responsive.
**For NEGATIVE constraints** (response should NOT contain or should avoid something):
- **1.0 (Complete Compliance)**: The response is responsive to the question AND the prohibited element is absent.
- **0.5 (Partial Compliance)**: The response is responsive but contains a borderline or ambiguous reference to the prohibited element.
- **0.0 (No Compliance)**: The prohibited element is present in the response, OR the response is non-responsive (off-topic, refusal, empty).
**Compound statement handling**: If the rubric criterion contains "and" or commas connecting multiple required elements:
- All elements present and correct = 1.0
- Some (but not all) elements present and correct = 0.5
- No elements present or correct = 0.0
EVALUATION RULES:
1. **Semantic tolerance**: Paraphrases and synonyms are acceptable. The response does not need to use the exact same words as the rubric.
2. **Numeric and date equivalence**: Treat equivalent representations as identical. "$68,000" = "68k" = "sixty-eight thousand dollars". "2 years" = "24 months". Prefer normalized comparison for numbers, currencies, dates, and durations.
3. **Case / punctuation / whitespace tolerance**: Differences in capitalization, punctuation, and whitespace must be ignored when comparing content.
4. **Hedging tolerance**: Do not penalize hedging language ("I think", "probably", "it seems"), passive voice, or verbosity if the substantive content satisfies the rubric criterion.
5. **Style neutrality**: Do not penalize for tone, formatting, or length unless the rubric criterion specifically requires a particular format.
6. **Responsiveness**: If the LLM response is completely off-topic or refuses to answer, score 0.0 for all criteria.
7. **Independence**: Evaluate this criterion in isolation — do not consider other rubric items.
8. **Specificity matters**: Vague or generic answers that could apply to any question score lower than specific, detailed answers.
STEP-BY-STEP EVALUATION:
Follow these steps in order:
1. **Understand the Requirement**: Read the rubric criterion and classify it as a positive requirement or a negative constraint.
2. **Parse Compound Statements**: If the criterion contains multiple sub-requirements joined by "and" or commas, identify each element separately.
3. **Check Compliance**: Compare the LLM response against each element, applying the tolerance rules above (semantic, numeric, case, hedging).
4. **Assign Score**: Use the appropriate scoring table (positive or negative) and compound-statement rule to determine the score.
5. **Provide Reasoning**: Write a concise explanation referencing which elements were or were not satisfied.
Return your evaluation as a JSON object with exactly two fields:
{"score": <0.0 or 0.5 or 1.0>, "reason": "<one concise sentence explaining your score>"}`;
}
/** Answer-generation prompt (mem0 `get_beam_answer_generation_prompt`).
* `memories` are pre-formatted display strings, oldest-first. For the
* no-context cell pass an empty array → "(No memories available)". */
export function buildAnswerGenerationPrompt(question: string, memories: string[]): string {
const memoriesText =
memories.length === 0
? '(No memories available)'
: memories.map((m, i) => `${i + 1}. ${m}`).join('\n');
return `You are an AI assistant with access to stored memories from prior conversations with a user.
Use these memories to answer the following question as accurately and completely as possible.
IMPORTANT RULES:
1. Scan ALL provided memories before answering — do not stop after the first relevant one.
2. If multiple memories contain relevant information, combine and cross-reference them.
3. If the memories contain contradictory information, prefer the more recent one.
4. If the memories don't contain enough information to answer, say exactly: "I don't have enough information to answer this question."
5. For temporal questions: pay attention to dates and relative time references.
6. For ordering questions: present events in chronological order.
7. For preference questions: use the most recently stated preference.
8. Be specific and direct — include exact names, dates, numbers, and details from the memories.
9. Do NOT invent or assume information that isn't in the memories.
QUESTION: ${question}
RETRIEVED MEMORIES:
${memoriesText}
ANSWER:`;
}
/** Answer-generation prompt — Option A variant (v2). Same skeleton as
* `buildAnswerGenerationPrompt`; ONLY the contradiction rule is rewritten and a
* negation/"never happened" rule is added. All other rules are verbatim v1.
* Pair with date-stamped `memories` ("[YYYY-MM-DD] role: ...") so the
* contradiction rule can surface each statement with its date. */
export function buildAnswerGenerationPromptV2(question: string, memories: string[], outline?: string): string {
const memoriesText =
memories.length === 0
? '(No memories available)'
: memories.map((m, i) => `${i + 1}. ${m}`).join('\n');
// `outline` is a generic pre-labeled preamble: the CALLER builds the labeled
// block(s) (timeline, standing directives, ...) and this just inserts them.
const outlineBlock = outline ? `\n${outline}\n` : '';
return `You are an AI assistant with access to stored memories from prior conversations with a user.
Use these memories to answer the following question as accurately and completely as possible.
IMPORTANT RULES:
1. Scan ALL provided memories before answering — do not stop after the first relevant one.
2. If multiple memories contain relevant information, combine and cross-reference them.
3. If the memories contain contradictory statements relevant to the question, do NOT silently pick one: explicitly state that there is contradictory information, present each of the conflicting statements (with their dates when shown), and ask the user which statement is correct.
4. If a memory explicitly states that something never happened, was never done, or was not completed, treat that as real information: answer accordingly (e.g., "No — you mentioned that you never ..."), citing that memory. Do NOT respond that you lack information when such a statement exists.
5. If the memories don't contain enough information to answer, say exactly: "I don't have enough information to answer this question."
6. For temporal questions: pay attention to dates and relative time references.
7. For ordering questions: present events in chronological order.
8. For preference questions: use the most recently stated preference.
9. Be specific and direct — include exact names, dates, numbers, and details from the memories.
10. Do NOT invent or assume information that isn't in the memories.
QUESTION: ${question}
${outlineBlock}
RETRIEVED MEMORIES:
${memoriesText}
ANSWER:`;
}
/** v3 = v2 with a rebalanced abstention guard: Rule 4 keeps the anti-wrongful-IDK
* behaviour but drops its aggressive final clause, and Rule 5 gains an explicit
* "no relevant memory at all → abstain, never guess" instruction — gpt-5 under
* v2 over-answers questions it should decline (abstention 0.40-0.47 vs 0.60 v1).
* Takes the same optional outline block as v2. */
export function buildAnswerGenerationPromptV3(question: string, memories: string[], outline?: string): string {
const memoriesText =
memories.length === 0
? '(No memories available)'
: memories.map((m, i) => `${i + 1}. ${m}`).join('\n');
// `outline` is a generic pre-labeled preamble: the CALLER builds the labeled
// block(s) (timeline, standing directives, ...) and this just inserts them.
const outlineBlock = outline ? `\n${outline}\n` : '';
return `You are an AI assistant with access to stored memories from prior conversations with a user.
Use these memories to answer the following question as accurately and completely as possible.
IMPORTANT RULES:
1. Scan ALL provided memories before answering — do not stop after the first relevant one.
2. If multiple memories contain relevant information, combine and cross-reference them.
3. If the memories contain contradictory statements relevant to the question, do NOT silently pick one: explicitly state that there is contradictory information, present each of the conflicting statements (with their dates when shown), and ask the user which statement is correct.
4. If a memory explicitly states that something never happened, was never done, or was not completed, treat that as real information: answer accordingly (e.g., "No — you mentioned that you never ..."), citing that memory.
5. Answer ONLY what the memories support. If no memory (and nothing in the timeline) contains information about the asked fact, you MUST say exactly: "I don't have enough information to answer this question." Never guess, infer unstated facts, or answer from general knowledge.
6. For temporal questions: pay attention to dates and relative time references.
7. For ordering questions: present events in chronological order.
8. For preference questions: use the most recently stated preference.
9. Be specific and direct — include exact names, dates, numbers, and details from the memories.
10. Do NOT invent or assume information that isn't in the memories.
QUESTION: ${question}
${outlineBlock}
RETRIEVED MEMORIES:
${memoriesText}
ANSWER:`;
}
/** v4 = v2 with a uniform, conditional exhaustiveness rule. Byte-identical to
* `buildAnswerGenerationPromptV2` EXCEPT a new rule 10 (renumbering v2's rule 10
* → 11) that tells the model to be EXHAUSTIVE for summary / overview / process /
* event-list questions and to answer at normal length otherwise. Motivated by
* the BEAM forensics: on summarization / event_ordering / multi_session_reasoning
* mem0's ~1.8× longer, clause-dense answers harvest compound rubric nuggets where
* our shorter answers score 0.5. Takes the same optional outline block as v2. */
export function buildAnswerGenerationPromptV4(question: string, memories: string[], outline?: string): string {
const memoriesText =
memories.length === 0
? '(No memories available)'
: memories.map((m, i) => `${i + 1}. ${m}`).join('\n');
// `outline` is a generic pre-labeled preamble: the CALLER builds the labeled
// block(s) (timeline, standing directives, ...) and this just inserts them.
const outlineBlock = outline ? `\n${outline}\n` : '';
return `You are an AI assistant with access to stored memories from prior conversations with a user.
Use these memories to answer the following question as accurately and completely as possible.
IMPORTANT RULES:
1. Scan ALL provided memories before answering — do not stop after the first relevant one.
2. If multiple memories contain relevant information, combine and cross-reference them.
3. If the memories contain contradictory statements relevant to the question, do NOT silently pick one: explicitly state that there is contradictory information, present each of the conflicting statements (with their dates when shown), and ask the user which statement is correct.
4. If a memory explicitly states that something never happened, was never done, or was not completed, treat that as real information: answer accordingly (e.g., "No — you mentioned that you never ..."), citing that memory. Do NOT respond that you lack information when such a statement exists.
5. If the memories don't contain enough information to answer, say exactly: "I don't have enough information to answer this question."
6. For temporal questions: pay attention to dates and relative time references.
7. For ordering questions: present events in chronological order.
8. For preference questions: use the most recently stated preference.
9. Be specific and direct — include exact names, dates, numbers, and details from the memories.
10. When the question asks for a comprehensive summary, an overview, an account of a process or journey, or a list or ordering of events: be EXHAUSTIVE. Enumerate every relevant topic, project, event, and discussion found in the memories, and for each include the specific details mentioned — tools, versions, numbers, dates, causes, outcomes, and sub-steps. Prefer complete, clause-dense coverage in a structured list over brevity; do not omit minor items. For all other questions, answer at normal length.
11. Do NOT invent or assume information that isn't in the memories.
QUESTION: ${question}
${outlineBlock}
RETRIEVED MEMORIES:
${memoriesText}
ANSWER:`;
}
/** v5 = v2 with a conditional temporal-commit rule. Byte-identical to
* `buildAnswerGenerationPromptV2` EXCEPT a new rule 10 (renumbering v2's rule 10
* → 11) that forces the model to COMMIT to a single computed duration for
* time-span / days-between questions instead of over-abstaining or anchoring on
* the mention date. Motivated by the BEAM temporal_reasoning forensics: our gap
* vs mem0 (0.557 vs 0.618) is dominated by over-abstention + wrong-anchor on
* duration questions. Takes the same optional outline block as v2. */
export function buildAnswerGenerationPromptV5(question: string, memories: string[], outline?: string): string {
const memoriesText =
memories.length === 0
? '(No memories available)'
: memories.map((m, i) => `${i + 1}. ${m}`).join('\n');
// `outline` is a generic pre-labeled preamble: the CALLER builds the labeled
// block(s) (timeline, standing directives, ...) and this just inserts them.
const outlineBlock = outline ? `\n${outline}\n` : '';
return `You are an AI assistant with access to stored memories from prior conversations with a user.
Use these memories to answer the following question as accurately and completely as possible.
IMPORTANT RULES:
1. Scan ALL provided memories before answering — do not stop after the first relevant one.
2. If multiple memories contain relevant information, combine and cross-reference them.
3. If the memories contain contradictory statements relevant to the question, do NOT silently pick one: explicitly state that there is contradictory information, present each of the conflicting statements (with their dates when shown), and ask the user which statement is correct.
4. If a memory explicitly states that something never happened, was never done, or was not completed, treat that as real information: answer accordingly (e.g., "No — you mentioned that you never ..."), citing that memory. Do NOT respond that you lack information when such a statement exists.
5. If the memories don't contain enough information to answer, say exactly: "I don't have enough information to answer this question."
6. For temporal questions: pay attention to dates and relative time references.
7. For ordering questions: present events in chronological order.
8. For preference questions: use the most recently stated preference.
9. Be specific and direct — include exact names, dates, numbers, and details from the memories.
10. When the question asks for a duration, time span, or number of days/weeks/months between two events: identify the single best-supported pair of dates in the memories (use the date the event actually happened or is scheduled FOR, not the date it was merely mentioned), COMMIT to one computed answer in the form 'N days — from <Month D, YYYY> to <Month D, YYYY>', and do not hedge with ranges or multiple candidate values. Only say you lack information if the memories contain no relevant dates at all.
11. Do NOT invent or assume information that isn't in the memories.
QUESTION: ${question}
${outlineBlock}
RETRIEVED MEMORIES:
${memoriesText}
ANSWER:`;
}
/** Fact-extraction prompt for event_ordering (mem0 `get_beam_fact_extraction_prompt`). */
export function buildFactExtractionPrompt(response: string): string {
return `Extract all distinct events or facts mentioned in the following response,
in the exact order they are presented. Return ONLY a JSON array of short event descriptions.
RESPONSE:
${response}
Return format: ["event 1 description", "event 2 description", ...]`;
}
/** Event-alignment prompt for event_ordering (mem0 `get_beam_event_alignment_prompt`). */
export function buildEventAlignmentPrompt(extractedEvent: string, rubricEvents: string[]): string {
const eventsList = rubricEvents.map((e, i) => `${i}. ${e}`).join('\n');
return `Given the following extracted event from an LLM response, determine which
reference event it best corresponds to. Return ONLY a JSON object.
EXTRACTED EVENT:
${extractedEvent}
REFERENCE EVENTS:
${eventsList}
If the extracted event matches one of the reference events (even approximately or paraphrased),
return the 0-based index. If it doesn't match any, return -1.
Return format: {"index": <integer>, "reason": "<brief explanation>"}`;
}
// ── Scoring primitives ──────────────────────────────────────────────────────
/** Clamp a raw judge score to 0.0 / 0.5 / 1.0 (mem0 `_clamp_nugget_score`). */
export function clampNuggetScore(raw: number): 0 | 0.5 | 1 {
if (raw >= 0.75) return 1;
if (raw >= 0.25) return 0.5;
return 0;
}
/**
* Parse the judge's `{score, reason}` JSON. Mirrors mem0's tolerant handling:
* try JSON first (possibly wrapped in prose / fenced), then fall back to a
* text search for "1.0" / "0.5". Returns a clamped score.
*/
export function parseNuggetJudgeOutput(text: string): { score: 0 | 0.5 | 1; reason: string } {
const obj = tryExtractJsonObject(text);
if (obj && typeof obj === 'object' && 'score' in obj) {
const rawScore = Number((obj as Record<string, unknown>).score);
if (Number.isFinite(rawScore)) {
const reason = String((obj as Record<string, unknown>).reason ?? '');
return { score: clampNuggetScore(rawScore), reason };
}
}
// Fallback: look for a score token in the raw text.
if (text.includes('1.0')) return { score: 1, reason: text.slice(0, 200) };
if (text.includes('0.5')) return { score: 0.5, reason: text.slice(0, 200) };
return { score: 0, reason: `Parse error: ${text.slice(0, 200)}` };
}
/** Best-effort JSON-object extraction: whole string, then first `{...}` span. */
function tryExtractJsonObject(text: string): unknown {
const trimmed = text.trim();
try {
return JSON.parse(trimmed);
} catch {
/* fall through */
}
const start = trimmed.indexOf('{');
const end = trimmed.lastIndexOf('}');
if (start >= 0 && end > start) {
try {
return JSON.parse(trimmed.slice(start, end + 1));
} catch {
/* fall through */
}
}
return null;
}
/** Best-effort JSON-array extraction (for fact extraction). */
function tryExtractJsonArray(text: string): unknown[] | null {
const trimmed = text.trim();
try {
const v = JSON.parse(trimmed);
if (Array.isArray(v)) return v;
if (v && typeof v === 'object') {
for (const key of ['events', 'facts', 'result']) {
const arr = (v as Record<string, unknown>)[key];
if (Array.isArray(arr)) return arr;
}
}
} catch {
/* fall through */
}
const start = trimmed.indexOf('[');
const end = trimmed.lastIndexOf(']');
if (start >= 0 && end > start) {
try {
const v = JSON.parse(trimmed.slice(start, end + 1));
if (Array.isArray(v)) return v;
} catch {
/* fall through */
}
}
return null;
}
// ── Kendall tau-b (verbatim port of mem0 common/metrics.py) ──────────────────
export function computeKendallTauB(predictedOrder: number[], referenceOrder: number[]): number {
if (predictedOrder.length < 2 || referenceOrder.length < 2) return 0;
const predRank = new Map<number, number>();
predictedOrder.forEach((v, i) => predRank.set(v, i));
const refRank = new Map<number, number>();
referenceOrder.forEach((v, i) => refRank.set(v, i));
const predSet = new Set(predictedOrder);
const common = [...new Set(referenceOrder.filter(v => predSet.has(v)))].sort((a, b) => a - b);
if (common.length < 2) return 0;
let concordant = 0;
let discordant = 0;
let tiedPred = 0;
let tiedRef = 0;
for (let i = 0; i < common.length; i++) {
for (let j = i + 1; j < common.length; j++) {
const a = common[i];
const b = common[j];
const predDiff = (predRank.get(a) ?? 0) - (predRank.get(b) ?? 0);
const refDiff = (refRank.get(a) ?? 0) - (refRank.get(b) ?? 0);
if (predDiff === 0 && refDiff === 0) {
tiedPred++;
tiedRef++;
} else if (predDiff === 0) {
tiedPred++;
} else if (refDiff === 0) {
tiedRef++;
} else if ((predDiff > 0 && refDiff > 0) || (predDiff < 0 && refDiff < 0)) {
concordant++;
} else {
discordant++;
}
}
}
const n1 = concordant + discordant + tiedPred;
const n2 = concordant + discordant + tiedRef;
if (n1 === 0 || n2 === 0) return 0;
return (concordant - discordant) / Math.sqrt(n1 * n2);
}
// ── Public judging API ───────────────────────────────────────────────────────
export interface NuggetScore {
nugget: string;
score: 0 | 0.5 | 1;
reason: string;
}
export interface QuestionJudgement {
/** Headline per-question score = mean of nugget scores (0..1). */
score: number;
/** PASS when score >= 0.5, else FAIL (mem0 pass threshold). */
judgment: 'PASS' | 'FAIL' | 'ERROR';
nuggetScores: NuggetScore[];
/** event_ordering only: nugget-mean blended with normalized tau-b. Diagnostic
* — NOT used by the headline Avg Score metric. */
scoreWithTau?: number;
eventOrdering?: { tauB: number; predictedOrder: number[]; referenceOrder: number[] };
/** Number of judge/extraction LLM calls made for this question (cost trace). */
judgeCalls: number;
error?: string;
}
/** Judge one rubric nugget via the LLM. */
export async function judgeSingleNugget(
llm: BeamLlm,
question: string,
nugget: string,
answer: string,
): Promise<{ score: 0 | 0.5 | 1; reason: string; result: BeamLlmResult }> {
const result = await llm.chat({
system: BEAM_JUDGE_SYSTEM_PROMPT,
user: buildNuggetJudgePrompt(question, nugget, answer),
jsonMode: true,
maxTokens: 300,
});
const parsed = parseNuggetJudgeOutput(result.text);
return { ...parsed, result };
}
/** Compute event_ordering Kendall tau-b for a generated answer (mem0
* `compute_event_ordering_score`). Returns tau plus the LLM results used. */
export async function computeEventOrderingScore(
llm: BeamLlm,
rubricNuggets: string[],
answer: string,
): Promise<{ tauB: number; predictedOrder: number[]; referenceOrder: number[]; results: BeamLlmResult[] }> {
const results: BeamLlmResult[] = [];
const extract = await llm.chat({
system: 'Extract events as a JSON array of strings.',
user: buildFactExtractionPrompt(answer),
jsonMode: true,
maxTokens: 500,
});
results.push(extract);
const extractedEvents = (tryExtractJsonArray(extract.text) ?? []).map(e => String(e));
if (extractedEvents.length === 0 || rubricNuggets.length === 0) {
return { tauB: 0, predictedOrder: [], referenceOrder: [], results };
}
const predictedIndices: number[] = [];
for (const event of extractedEvents) {
const align = await llm.chat({
system: 'Align the event to a reference event index. Return JSON.',
user: buildEventAlignmentPrompt(event, rubricNuggets),
jsonMode: true,
maxTokens: 120,
});
results.push(align);
const obj = tryExtractJsonObject(align.text) as Record<string, unknown> | null;
let idx = -1;
if (obj && 'index' in obj) {
const n = Number(obj.index);
if (Number.isFinite(n)) idx = Math.trunc(n);
}
if (idx >= 0 && idx < rubricNuggets.length) predictedIndices.push(idx);
}
const referenceOrder = rubricNuggets.map((_, i) => i);
const tauB = computeKendallTauB(predictedIndices, referenceOrder);
return { tauB: round4(tauB), predictedOrder: predictedIndices, referenceOrder, results };
}
export interface JudgeQuestionInput {
question: string;
rubric: string[];
memoryAbility: string;
answer: string;
}
/**
* Judge a full question: score every rubric nugget, average, and (for
* event_ordering) additionally compute the tau-b blend. Mirrors the per-question
* portion of mem0's `process_question`.
*/
export async function judgeQuestion(
llm: BeamLlm,
input: JudgeQuestionInput,
opts: { computeTau?: boolean } = {},
): Promise<{ judgement: QuestionJudgement; llmResults: BeamLlmResult[] }> {
const llmResults: BeamLlmResult[] = [];
if (input.rubric.length === 0) {
return {
judgement: {
score: 0,
judgment: 'ERROR',
nuggetScores: [],
judgeCalls: 0,
error: 'No rubric nuggets found',
},
llmResults,
};
}
const nuggetScores: NuggetScore[] = [];
for (const nugget of input.rubric) {
const ns = await judgeSingleNugget(llm, input.question, nugget, input.answer);
llmResults.push(ns.result);
nuggetScores.push({ nugget, score: ns.score, reason: ns.reason });
}
const avg = nuggetScores.reduce((s, n) => s + n.score, 0) / nuggetScores.length;
const judgement: QuestionJudgement = {
score: round4(avg),
judgment: avg >= 0.5 ? 'PASS' : 'FAIL',
nuggetScores,
judgeCalls: nuggetScores.length,
};
// event_ordering: auxiliary tau-b blend (NOT part of the headline metric).
if (input.memoryAbility === 'event_ordering' && opts.computeTau) {
const eo = await computeEventOrderingScore(llm, input.rubric, input.answer);
llmResults.push(...eo.results);
judgement.judgeCalls += eo.results.length;
judgement.eventOrdering = { tauB: eo.tauB, predictedOrder: eo.predictedOrder, referenceOrder: eo.referenceOrder };
const tauNormalized = (eo.tauB + 1) / 2; // map [-1,1] -> [0,1]
judgement.scoreWithTau = round4((avg + tauNormalized) / 2);
}
return { judgement, llmResults };
}
function round4(x: number): number {
return Math.round(x * 1e4) / 1e4;
}

View File

@@ -0,0 +1,230 @@
/**
* Minimal direct-to-OpenAI client for the BEAM answerer + graded judge.
*
* Why not the existing `llm.ts` LiteLlmClient? Two reasons:
* 1. The official BEAM protocol (mem0) uses gpt-4o for both answerer and
* judge, driven directly via OPENAI_API_KEY — the team's instruction for
* this phase. This client reads that key from waggle-os/.env using the
* same "manual .env load, no dotenv dependency" convention already used in
* gepa-phase-5/scripts/cost-probe.ts and vitest.setup.ts.
* 2. The graded judge needs JSON-mode structured output
* (`response_format: {type:'json_object'}`), which the LiteLLM transport
* in llm.ts does not expose.
*
* It implements the `BeamLlm` interface from beam-nugget-judge.ts, so the judge
* and answerer are transport-agnostic and could later be pointed at the LiteLLM
* proxy or a different provider without touching the scoring logic.
*/
import fs from 'node:fs';
import path from 'node:path';
import type { BeamLlm, BeamLlmResult } from './beam-nugget-judge.js';
// ── .env loading (manual; mirrors cost-probe.ts) ────────────────────────────
/**
* Load KEY=VALUE pairs from a `.env` file into process.env without clobbering
* values already present (CLI/real env wins over the file). Searches the given
* path, then walks up from cwd looking for a `.env`. Returns the resolved path
* used, or null if none was found.
*/
export function loadDotEnv(explicitPath?: string): string | null {
const candidates: string[] = [];
if (explicitPath) candidates.push(explicitPath);
let dir = process.cwd();
for (let i = 0; i < 6; i++) {
candidates.push(path.join(dir, '.env'));
const parent = path.dirname(dir);
if (parent === dir) break;
dir = parent;
}
for (const p of candidates) {
if (!fs.existsSync(p)) continue;
const content = fs.readFileSync(p, 'utf-8');
for (const line of content.split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const eq = trimmed.indexOf('=');
if (eq <= 0) continue;
const key = trimmed.slice(0, eq).trim();
let val = trimmed.slice(eq + 1).trim();
// Strip surrounding quotes.
if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) {
val = val.slice(1, -1);
}
if (!(key in process.env) || !process.env[key]) process.env[key] = val;
}
return p;
}
return null;
}
// ── Pricing (USD per 1M tokens) ─────────────────────────────────────────────
export interface ModelPricing {
inputPerMillion: number;
outputPerMillion: number;
}
/** Default pricing table. gpt-4o = the mem0 BEAM README default; gpt-5 = the
* model behind mem0's published 0.641 (results/platform metadata). */
export const OPENAI_PRICING: Record<string, ModelPricing> = {
'gpt-4o': { inputPerMillion: 2.5, outputPerMillion: 10.0 },
'gpt-4o-mini': { inputPerMillion: 0.15, outputPerMillion: 0.6 },
'gpt-5': { inputPerMillion: 1.25, outputPerMillion: 10.0 },
'gpt-5-mini': { inputPerMillion: 0.25, outputPerMillion: 2.0 },
'gpt-5-nano': { inputPerMillion: 0.05, outputPerMillion: 0.4 },
};
/** gpt-5 / o-series reasoning models reject `max_tokens` + non-default
* temperature, and spend completion budget on hidden reasoning tokens. */
function isReasoningModel(model: string): boolean {
return /^(gpt-5|o\d)/.test(model.toLowerCase());
}
// ── Client ──────────────────────────────────────────────────────────────────
const DEFAULT_MAX_RETRIES = 3;
const DEFAULT_TIMEOUT_MS = 60_000;
export interface BeamOpenAiClientOptions {
model: string;
apiKey: string;
baseUrl?: string;
pricing?: ModelPricing;
timeoutMs?: number;
maxRetries?: number;
}
export class BeamOpenAiClient implements BeamLlm {
private readonly model: string;
private readonly apiKey: string;
private readonly baseUrl: string;
private readonly pricing: ModelPricing;
private readonly timeoutMs: number;
private readonly maxRetries: number;
constructor(opts: BeamOpenAiClientOptions) {
this.model = opts.model;
this.apiKey = opts.apiKey;
this.baseUrl = (opts.baseUrl ?? 'https://api.openai.com/v1').replace(/\/$/, '');
this.pricing = opts.pricing ?? OPENAI_PRICING[opts.model] ?? { inputPerMillion: 0, outputPerMillion: 0 };
this.timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
this.maxRetries = opts.maxRetries ?? DEFAULT_MAX_RETRIES;
}
async chat(opts: { system: string; user: string; jsonMode?: boolean; maxTokens?: number }): Promise<BeamLlmResult> {
const started = Date.now();
let lastFailure = 'unknown';
const reasoning = isReasoningModel(this.model);
// Reasoning models spend completion budget on hidden reasoning tokens before
// emitting any answer text; long-form questions can exhaust a small cap and
// return HTTP-200 with empty text. Start at a high floor and, on an empty
// completion, double the budget (capped) and retry within this loop.
let reasoningBudget = Math.max(opts.maxTokens ?? 800, 16384);
const REASONING_BUDGET_CAP = 32768;
for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
if (attempt > 0) {
await sleep(Math.min(8000, 500 * 2 ** (attempt - 1)));
}
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), this.timeoutMs);
try {
const body: Record<string, unknown> = {
model: this.model,
messages: [
{ role: 'system', content: opts.system },
{ role: 'user', content: opts.user },
],
};
if (reasoning) {
body.max_completion_tokens = reasoningBudget;
} else {
body.temperature = 0;
body.max_tokens = opts.maxTokens ?? 800;
}
if (opts.jsonMode) body.response_format = { type: 'json_object' };
const res = await fetch(`${this.baseUrl}/chat/completions`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${this.apiKey}`,
},
body: JSON.stringify(body),
signal: controller.signal,
});
if (!res.ok) {
lastFailure = `http_${res.status}`;
// Retry on rate-limit / server errors; bail on client errors.
if (res.status === 429 || res.status >= 500) continue;
const errText = await res.text().catch(() => '');
return this.fail(`http_${res.status}`, started, errText.slice(0, 200));
}
const json = (await res.json()) as {
choices?: Array<{ message?: { content?: string } }>;
usage?: { prompt_tokens?: number; completion_tokens?: number };
};
const text = json.choices?.[0]?.message?.content ?? '';
// Retry-on-empty for reasoning models: HTTP-200 but no answer text means
// the whole budget went to hidden reasoning. Count this attempt as a
// failure, double the budget (capped), and retry.
if (reasoning && text.trim() === '' && reasoningBudget < REASONING_BUDGET_CAP) {
lastFailure = 'empty_completion';
const nextBudget = Math.min(reasoningBudget * 2, REASONING_BUDGET_CAP);
console.warn(
`[beam-openai] empty completion from ${this.model} ` +
`(max_completion_tokens=${reasoningBudget}); retrying with ${nextBudget}`,
);
reasoningBudget = nextBudget;
continue;
}
const inputTokens = json.usage?.prompt_tokens ?? approxTokens(opts.system + opts.user);
const outputTokens = json.usage?.completion_tokens ?? approxTokens(text);
const costUsd =
(inputTokens / 1_000_000) * this.pricing.inputPerMillion +
(outputTokens / 1_000_000) * this.pricing.outputPerMillion;
return { text, inputTokens, outputTokens, costUsd, latencyMs: Date.now() - started, failureMode: null };
} catch (err) {
const name = (err as Error).name;
lastFailure = name === 'AbortError' ? 'timeout' : `fetch_error_${name}`;
// Loop will retry unless attempts exhausted.
} finally {
clearTimeout(timer);
}
}
return this.fail(lastFailure, started);
}
private fail(failureMode: string, started: number, _detail?: string): BeamLlmResult {
return { text: '', inputTokens: 0, outputTokens: 0, costUsd: 0, latencyMs: Date.now() - started, failureMode };
}
}
/** Build a client, resolving the API key from env/.env. Throws if absent. */
export function createBeamOpenAiClient(opts: {
model: string;
envPath?: string;
baseUrl?: string;
pricing?: ModelPricing;
}): BeamOpenAiClient {
loadDotEnv(opts.envPath);
const apiKey = process.env.OPENAI_API_KEY;
if (!apiKey) {
throw new Error(
'OPENAI_API_KEY not found in environment or .env. ' +
'Set it in waggle-os/.env or export it before running.',
);
}
return new BeamOpenAiClient({ model: opts.model, apiKey, baseUrl: opts.baseUrl, pricing: opts.pricing });
}
function approxTokens(s: string): number {
return Math.max(1, Math.ceil(s.length / 4));
}
function sleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}

View File

@@ -0,0 +1,330 @@
/**
* hive_mind_ipb cell — extends the `retrieval` cell with I/P/B frame writes.
*
* CellName extension note
* ───────────────────────
* `hive_mind_ipb` is NOT part of the existing `CellName` union defined in
* `src/types.ts`. Callers that need to dispatch this cell alongside the
* canonical 7-cell set must use the extended union:
*
* import type { CellName } from './types.js';
* export type ExtendedCellName = CellName | 'hive_mind_ipb';
*
* The runner's `isCellName()` guard in `cells.ts` will return `false` for
* `'hive_mind_ipb'`; callers must add their own dispatch branch:
*
* if (cellName === 'hive_mind_ipb') {
* result = await hiveMindIpbCell({ ... });
* } else {
* result = await cells[cellName as CellName]({ ... });
* }
*
* v8 cell-layout mapping:
* no-memory-baseline → existing `no-context` cell (no change)
* hive_mind_retrieval → existing `retrieval` cell (no change)
* hive_mind_ipb → this file (new)
* hive_mind_ipb_strong → same as hive_mind_ipb, different model (CLI-level)
*
* What this cell adds on top of `retrieval`:
*
* 1. BEFORE retrieval: write a P-frame recording the agent's query intent.
* content: `"Retrieving to answer: ${instance.question}"`
* source: 'agent_inferred'
*
* 2. Run HybridSearch retrieval — identical to `retrieval` cell (top-K=20,
* scoped to gopId when `instance.conversation_id` is present).
*
* 3. CONTRADICTION CHECK (lightweight, single LLM call):
* If ≥1 existing P-frame for this gopId exists in the substrate (i.e.
* this is not the first question in the conversation), ask the subject
* LLM whether any retrieved frame content significantly contradicts a
* prior P-frame. When the check fires and the LLM detects a conflict,
* write a B-frame recording the conflict. The check is skipped on the
* first question per conversation (no prior P-frames yet).
*
* 4. Build the answer prompt using the `# Recalled Memories` block —
* identical to the `retrieval` cell.
*
* 5. Call the LLM for the answer — identical to `retrieval`.
*
* 6. AFTER answering: write an I-frame for this Q-A turn:
* content: `"Q: ${instance.question} A: ${answer}"`
* source: 'import'
*
* The P-frame and I-frame written here enrich future retrievals for later
* questions in the same conversation (the same substrate is reused across
* instances within a run).
*
* Cost profile: 1–2 extra LLM calls per instance when contradiction check
* fires (P-frames exist). The check call uses the cell's `LlmClient` with
* a short prompt; it does NOT route through the agentic agent-loop.
*
* Style: matches `cells.ts` exactly — same imports, same error patterns,
* same `formatRecalledMemories` block format, same system-prompt builder.
*/
import type { DatasetInstance, ModelSpec, CellName } from './types.js';
import type { LlmClient, LlmCallResult } from './llm.js';
import type { Substrate } from './substrate.js';
import type { SearchResult, MemoryFrame } from '@waggle/core';
// ── CellName extension ────────────────────────────────────────────────────────
/** Extended cell-name union that includes the IPB cell.
* Callers must use this type when building dispatch tables that include
* `hive_mind_ipb` alongside the canonical `CellName` set. */
export type ExtendedCellName = CellName | 'hive_mind_ipb';
// ── CellFn re-export (for standalone use without cells.ts import) ─────────────
/** Identical to `CellFn` in `cells.ts` — repeated here so `cells-ipb.ts` is
* importable without depending on the full `cells.ts` module. */
export interface CellInput {
instance: DatasetInstance;
model: ModelSpec;
llm: LlmClient;
turnId: string;
/** Memory substrate — required by this cell; throws loudly when absent. */
substrate?: Substrate;
/** LiteLLM routing — used by agentic cell; accepted here for interface parity
* but not used (IPB routes all LLM calls through `llm: LlmClient`). */
litellm?: { url: string; apiKey: string };
/** Retrieval top-K. Default 20 (Stage 2-Retry §1.2). */
retrievalTopK?: number;
/** Unused by this cell — present for interface symmetry. */
agenticMaxTurns?: number;
/** Unused by this cell — present for interface symmetry. */
agenticTimeoutMs?: number;
}
export type CellFn = (input: CellInput) => Promise<LlmCallResult>;
// ── Shared helpers (mirrors cells.ts) ────────────────────────────────────────
const FACTOID_BASELINE_PERSONA = 'short-answer factoid QA agent';
/** Inline selectShape — avoids importing cells.ts to keep this file self-contained.
* Falls back to a simple system-prompt wrapper that works with any model shape. */
function buildSystemPrompt(persona: string): string {
return `You are a ${persona}. Answer questions concisely. Output the answer span only — no sentences, no preamble.`;
}
/** Format a HybridSearch result list into the `# Recalled Memories` block —
* identical to `formatRecalledMemories` in `cells.ts`. */
function formatRecalledMemories(results: readonly SearchResult[]): string {
if (results.length === 0) {
return '# Recalled Memories\n(none)';
}
const lines = results.map(r => {
const score = r.finalScore.toFixed(3);
const source = r.frame.source ?? 'user_stated';
return `- [memory:${r.frame.gop_id}:${r.frame.id} score=${score} src=${source}] ${r.frame.content}`;
});
return `# Recalled Memories\n${lines.join('\n')}`;
}
function assertSubstrate(cellName: string, substrate: Substrate | undefined): asserts substrate is Substrate {
if (!substrate) {
throw new Error(
`cells.${cellName} requires a Substrate dependency. Construct it via ` +
`createSubstrate({embedder}) and pass it in CellInput.substrate. See ` +
`benchmarks/harness/src/substrate.ts.`,
);
}
}
// ── Contradiction-check prompt ────────────────────────────────────────────────
/** System prompt for the lightweight contradiction-check call (step 3).
* Instructs the model to return a structured one-line verdict — keeps
* post-processing trivial. */
const SYSTEM_CONTRADICTION_CHECK =
'You are a consistency checker. You are given a list of prior prediction ' +
'statements and a list of retrieved memory excerpts. ' +
'Respond with EXACTLY one line in the format: ' +
'CONFLICT: <short description> OR NO_CONFLICT — no other text. ' +
'A conflict exists only when a retrieved excerpt directly contradicts ' +
'a specific factual claim in a prior prediction (same entity, incompatible values). ' +
'Superficial overlap or topic similarity is NOT a conflict.';
function buildContradictionCheckPrompt(
priorPFrames: readonly MemoryFrame[],
retrievedResults: readonly SearchResult[],
): string {
const priors = priorPFrames
.map((f, i) => `[prior_${i + 1}] ${f.content}`)
.join('\n');
const retrieved = retrievedResults
.slice(0, 10) // limit to first 10 retrieved frames to keep the prompt short
.map((r, i) => `[retrieved_${i + 1}] ${r.frame.content}`)
.join('\n');
return (
'## Prior predictions\n' + priors +
'\n\n## Retrieved memories\n' + retrieved +
'\n\nDo any retrieved memories directly contradict any prior prediction? ' +
'Respond with CONFLICT: <description> or NO_CONFLICT.'
);
}
// ── hive_mind_ipb cell ────────────────────────────────────────────────────────
/**
* `hive_mind_ipb` — retrieval cell extended with I/P/B frame writes.
*
* Step-by-step (see module-level doc for full rationale):
*
* 1. Assert substrate present.
* 2. Write a P-frame: "Retrieving to answer: <question>".
* 3. Run HybridSearch (top-K=20, gopId-scoped).
* 4. Contradiction check (if prior P-frames exist):
* - Single LLM call with SYSTEM_CONTRADICTION_CHECK.
* - If response starts with "CONFLICT:", write a B-frame.
* 5. Build answer prompt (`# Recalled Memories` block + question).
* 6. LLM answer call.
* 7. Write I-frame: "Q: <question> A: <answer>".
* 8. Return `LlmCallResult` (tokens and cost aggregated across all LLM calls).
*/
export const hiveMindIpbCell: CellFn = async ({
instance,
model,
llm,
turnId: _turnId,
substrate,
retrievalTopK,
}: CellInput): Promise<LlmCallResult> => {
assertSubstrate('hive_mind_ipb', substrate);
const gopId = instance.conversation_id;
const limit = retrievalTopK ?? 20;
const searchOpts: { limit: number; gopId?: string } = { limit };
if (gopId) searchOpts.gopId = gopId;
const started = Date.now();
let totalInputTokens = 0;
let totalOutputTokens = 0;
let totalCostUsd = 0;
// ── Step 1: Write P-frame (query intent) ─────────────────────────────────
// Source 'agent_inferred' — this is a predicted/planned retrieval, not
// imported data. The P-frame enriches future retrievals: other questions
// in the same conversation will see this frame when their retrieval
// scans the gopId.
//
// `createPFrame` requires a `baseFrameId` (the I-frame this P-frame
// predicts an update to). We use the latest I-frame for this gopId as
// the base; if none exists yet (very first ingest of this conversation),
// we fall back to `createIFrame` with source='agent_inferred' so the
// frame still lands in the substrate.
let pFrame: MemoryFrame;
const latestI = substrate.frames.getLatestIFrame(gopId ?? '_global');
if (latestI) {
pFrame = substrate.frames.createPFrame(
gopId ?? '_global',
`Retrieving to answer: ${instance.question}`,
latestI.id,
'normal',
'agent_inferred',
);
} else {
// No I-frame yet — use createIFrame as a fallback so the P-frame intent
// still lands in the substrate. source='agent_inferred' marks it as a
// predicted frame even though the frame_type will be 'I'.
pFrame = substrate.frames.createIFrame(
gopId ?? '_global',
`Retrieving to answer: ${instance.question}`,
'normal',
'agent_inferred',
);
}
// ── Step 2: HybridSearch retrieval ───────────────────────────────────────
const results = await substrate.search.search(instance.question, searchOpts);
// ── Step 3: Contradiction check ──────────────────────────────────────────
// Only fires when ≥1 prior P-frame exists for this gopId. That means: skip
// the check on the very first question in a conversation (the P-frame we
// just wrote is the only P-frame; checking for conflicts with itself would
// be noise). We look for P-frames that predate the one we just created.
let bFrame: MemoryFrame | null = null;
if (gopId) {
// Fetch all P-type frames for this conversation that existed BEFORE
// the one we just wrote (t < pFrame.t). getPFramesSinceLastI returns
// P-frames since the latest I-frame; we filter to those older than pFrame.
const allGopFrames = substrate.frames.getGopFrames(gopId);
const priorPFrames = allGopFrames.filter(
f => f.frame_type === 'P' && f.id !== pFrame.id && f.t < pFrame.t,
);
if (priorPFrames.length > 0 && results.length > 0) {
const checkPrompt = buildContradictionCheckPrompt(priorPFrames, results);
let checkResult: LlmCallResult;
try {
checkResult = await llm.call({
model,
systemPrompt: SYSTEM_CONTRADICTION_CHECK,
userPrompt: checkPrompt,
});
totalInputTokens += checkResult.inputTokens;
totalOutputTokens += checkResult.outputTokens;
totalCostUsd += checkResult.costUsd;
const verdict = checkResult.text.trim();
if (verdict.startsWith('CONFLICT:')) {
// Write B-frame: references the P-frame we wrote (base) and the
// retrieved frame ids that triggered the conflict.
const conflictDescription = verdict.slice('CONFLICT:'.length).trim();
const referencedIds = results.slice(0, 5).map(r => r.frame.id);
bFrame = substrate.frames.createBFrame(
gopId,
conflictDescription,
pFrame.id,
referencedIds,
);
}
} catch {
// Contradiction check is best-effort — a failure here must not abort
// the answer generation. Swallow silently (consistent with the agentic
// cell's error-swallow pattern for non-fatal sub-steps).
}
}
}
// bFrame is written to substrate; no further action required unless callers
// want to surface it. Suppressing unused-variable warning:
void bFrame;
// ── Step 4: Build answer prompt and call LLM ─────────────────────────────
const memoryBlock = formatRecalledMemories(results);
const userPrompt = `${memoryBlock}\n\nQuestion: ${instance.question}`;
const answerResult = await llm.call({
model,
systemPrompt: buildSystemPrompt(FACTOID_BASELINE_PERSONA),
userPrompt,
});
totalInputTokens += answerResult.inputTokens;
totalOutputTokens += answerResult.outputTokens;
totalCostUsd += answerResult.costUsd;
const answer = answerResult.text;
// ── Step 5: Write I-frame for this Q-A turn ───────────────────────────────
// Stores the question + answer so future questions in the same conversation
// can retrieve it. source='import' matches the bulk-ingest frames so
// retrieval ranking treats Q-A frames at parity with original turn frames.
substrate.frames.createIFrame(
gopId ?? '_global',
`Q: ${instance.question} A: ${answer}`,
'normal',
'import',
);
// ── Aggregate and return ──────────────────────────────────────────────────
const latencyMs = Date.now() - started;
return {
text: answer,
inputTokens: totalInputTokens,
outputTokens: totalOutputTokens,
latencyMs,
costUsd: totalCostUsd,
failureMode: answerResult.failureMode,
};
};

View File

@@ -0,0 +1,504 @@
/**
* Four + two cell implementations — the causal isolation grid.
*
* Each cell is a pure(-ish) function from a `CellInput` to an `LlmCallResult`.
* Cells differ only in HOW they assemble the model's context:
*
* raw: no memory injection, no prompt evolution.
* filtered: inject the instance's own context as "retrieved memory"
* (Sprint 9 scaffold proxy — retained for back-compat; new
* `retrieval` cell below is the real-substrate replacement).
* compressed: raw prompt wrapped in an evolved scaffold (Sprint 9 GEPA
* prompt-evolution proxy — retained for back-compat).
* full-context: memory + evolve (both Sprint 9 treatments).
* retrieval: real `@waggle/core::HybridSearch` recall → top-K turn
* frames → "# Recalled Memories" block → baseline system
* prompt. Sprint 12 Task 2.5 Stage 1.
* agentic: `@waggle/agent::agent-loop` with a `search_memory` tool
* allowlist + 3-turn cap. The agent decides for itself when
* to search, how to query, and when to stop. Sprint 12
* Task 2.5 Stage 1.
*
* Controls live in `controls.ts`.
*/
import type { DatasetInstance, ModelSpec, CellName, ControlName } from './types.js';
import type { LlmClient, LlmCallResult } from './llm.js';
import type { Substrate } from './substrate.js';
import type { SearchResult } from '@waggle/core';
import {
runAgentLoop,
selectShape,
type AgentLoopConfig,
type ToolDefinition,
} from '@waggle/agent';
/**
* Phase 2.3 (Option A) refactor — the two ChatGPT-audit-flagged
* Claude-shaped artefacts have been deleted from this file:
* - the strict-extraction baseline + evolved system-prompt constants
* (replaced by cell-local personas routed through Phase 1.2 prompt-shapes
* via `systemPromptForCell` below)
* - the synthetic-memory scaffold-proxy user-prompt builder (replaced by
* `buildUserPromptRetrieved` with neutral framing)
*
* Cell semantics are preserved via cell-specific personas + cell-specific
* user-prompt builders. SYSTEM_AGENTIC and SYSTEM_AGENTIC_FORCED_FALLBACK
* (declared further below) remain on their pre-existing methodology lineage
* and are explicitly excluded from this refactor's scope.
*
* For the original audit-flagged literal strings, see the pre-refactor revision
* of this file at git tag/commit prior to commit 5699677.
*/
const FACTOID_BASELINE_PERSONA = 'short-answer factoid QA agent';
const FACTOID_STRICT_PERSONA =
'short-answer factoid QA agent that extracts the exact answer span ' +
'from supplied context, or replies "unknown" when the context does ' +
'not support an answer';
/**
* Build a model-aware system prompt via Phase 1.2 prompt-shapes. Falls back
* to generic-simple when the model alias is not registered.
*/
function systemPromptForCell(model: ModelSpec, persona: string): string {
const shape = selectShape(model.litellmModel);
return shape.systemPrompt({ persona, question: '', isMultiStep: false });
}
/**
* SYSTEM_AGENTIC — SOFTENED by PM 2026-04-24 (Stage 2-Retry Gate A ratification).
* Sprint 12 Task 2.5 Stage 2-Retry source of record. Supersedes the Stage 1
* Gate 1 text ratified in commit `c80a4a3`.
*
* Changes vs Stage 1 text (Stage 2 N=20 FAIL drove these):
* - §1 protocol verb: MUST → SHOULD. Stage 2 showed the MUST-call-first-
* turn rule was too rigid — 2/20 instances answered correctly from
* general knowledge, hitting the 95% floor by one instance. Softening
* lets the agent skip the tool on clearly-non-conversational factoids.
* - §1 explicit exception: general-knowledge lookups where the answer
* does not require conversation-specific context may skip the tool call.
* - §3 phrasing: "directly contain" → "contain" (tolerate inference from
* retrieved memories rather than requiring exact-span match).
* - §5 cap description: dropped the "SHOULD finish in 2" prescriptive
* language in favour of "use your turns wisely" — diagnostic data from
* Stage 2 showed 14/20 finished in 2 organically, and the prescriptive
* phrasing was not carrying behavioural weight.
* - §6 fallback threshold: "do not contain the answer" → "after reasonable
* search you believe the memory does not contain a supported answer"
* (nominalized, gives the agent latitude before abstaining).
* - §7 NEW: explicit tool-exhaustion fallback clause — if turn 3 arrives
* without a clear answer, commit to a best-supported answer. Backs the
* runtime-side forced-answer fallback (§1.4 in the brief).
* - Closing paragraph: "content returned by search_memory … and general
* knowledge" instead of "ONLY … search_memory tool" — matches §1
* softening.
*
* Previous Stage 1 text archive: commit c80a4a3.
*
* PM gate reference: `PM-Waggle-OS/briefs/2026-04-24-cc-task25-stage2-retry-kickoff.md` §1.3.
*/
export const SYSTEM_AGENTIC = [
'You are a memory-grounded answering agent. Your job: answer a short',
'factoid question using content returned by the search_memory tool and',
'your reasoning over it.',
'',
'Protocol (you SHOULD follow):',
'1. First turn: call search_memory with a focused query derived from the',
' question, UNLESS the question is a simple factual lookup you can',
' answer with high confidence from general knowledge and the answer',
' does not require conversation-specific context. When uncertain,',
' prefer the search_memory call.',
'2. After the tool returns, read the retrieved memories carefully.',
'3. If the retrieved memories contain the answer, respond with the',
' shortest possible answer span — no sentences, no hedging, no preamble.',
'4. If the retrieved memories are ambiguous or incomplete, you MAY call',
' search_memory ONE more time with a refined query (different wording,',
' different entity, different time window). Then answer.',
'5. You have a hard cap of 3 total turns. Use your turns wisely.',
'6. If after reasonable search you believe the memory does not contain a',
' supported answer, reply with exactly: unknown',
'7. If turn 3 arrives without a clear answer, commit to your best',
' supported answer span using the context you have gathered across',
' search calls. Do NOT leave the response empty.',
'',
'Output format: plain answer span only. No JSON, no markdown, no',
'explanation. Never invent facts. Ground every factual claim in retrieved',
'context or clearly-established general knowledge.',
].join('\n');
/**
* SYSTEM_AGENTIC_FORCED_FALLBACK — Sprint 12 Task 2.5 Stage 2-Retry §1.4.
*
* The runtime fallback prompt fires when agent-loop exhausts `maxTurns`
* with empty `resp.content` but non-empty `toolsUsed`. The agentic cell
* wrapper calls the subject LLM directly (no tools, no agent-loop) with
* this system prompt and the accumulated search_memory tool results in
* the user message. Prevents the Stage 2 "2/20 empty-answer" tail.
*/
export const SYSTEM_AGENTIC_FORCED_FALLBACK =
'You must commit to your best supported answer span or reply `unknown`. ' +
'Do not call tools. Use only the retrieved memory context below. ' +
'Respond with ONLY the answer span — no sentences, no hedging, no preamble.';
export interface CellInput {
instance: DatasetInstance;
model: ModelSpec;
llm: LlmClient;
turnId: string;
/** Memory substrate — required by `retrieval` + `agentic` cells, ignored
* by the other four. A clear error is thrown when a substrate-requiring
* cell fires without one so the misconfiguration is loud. */
substrate?: Substrate;
/** LiteLLM routing used by the agentic cell's inner agent-loop. Ignored by
* every other cell (they route through `llm: LlmClient`). */
litellm?: { url: string; apiKey: string };
/** Retrieval top-K. Default 10 per Stage 1 GATE-S0 decision. */
retrievalTopK?: number;
/** Agentic hard turn cap. Default 3 per Stage 1 GATE-S0 decision. */
agenticMaxTurns?: number;
/** Agentic AbortController timeout in ms. Default 180_000 (matches the
* LiteLLM client's thinking=on timeout). */
agenticTimeoutMs?: number;
/** Testability hook — injects a mock `runAgentLoop` impl. Unit tests pass
* an in-memory stub; production runs leave undefined and use the real
* `@waggle/agent::runAgentLoop`. */
runAgentLoopFn?: typeof runAgentLoop;
}
export type CellFn = (input: CellInput) => Promise<LlmCallResult>;
function buildUserPromptRaw(instance: DatasetInstance): string {
return `Context: ${instance.context}\n\nQuestion: ${instance.question}`;
}
/**
* Phase 2.3: replaces the deleted scaffold-proxy user-prompt builder. Same
* semantic — context framed AS retrieved memory — with neutral phrasing.
* The retrieval cell still uses `formatRecalledMemories` (which produces
* real frame-metadata-tagged output from HybridSearch results, not the
* deleted scaffold proxy).
*/
function buildUserPromptRetrieved(instance: DatasetInstance): string {
return (
'Retrieved context (from session memory):\n' +
`${instance.context}\n\n` +
`Question: ${instance.question}`
);
}
/** Format a `HybridSearch.search()` result list into the "# Recalled
* Memories" block the real Waggle orchestrator emits. The retrieval cell's
* causal contract is: "this is what HybridSearch would return at inference
* time, in the shape the agent would see." */
function formatRecalledMemories(results: readonly SearchResult[]): string {
if (results.length === 0) {
return '# Recalled Memories\n(none)';
}
const lines = results.map((r, idx) => {
const score = r.finalScore.toFixed(3);
const source = r.frame.source ?? 'user_stated';
return `- [memory:${r.frame.gop_id}:${r.frame.id} score=${score} src=${source}] ${r.frame.content}`;
});
return `# Recalled Memories\n${lines.join('\n')}`;
}
/** Build a `search_memory`-only `ToolDefinition` bound to the substrate's
* HybridSearch instance. The tool returns a plain-text memory block the
* LLM can parse inline — matches the shape the real Waggle orchestrator
* emits for `search_memory` calls.
*
* Sprint 12 Task 2.5 Stage 2-Retry §1.2: when `boundToGopId` is provided,
* every tool invocation scopes its underlying `HybridSearch.search` call
* to that `gopId` (conversation). This matches the LoCoMo QA-pair locality
* — relevant evidence for a question lives inside its own conversation.
* The agent CANNOT override the binding at call time (the tool does not
* expose a `gopId` param); this is intentional — per-conversation scope
* is a benchmark invariant, not an agent decision.
*
* Default `defaultLimit` bumped from Stage 1's 10 → 20 per Stage 2-Retry
* brief §1.2 tail ("Top-K moves from 10 to 20"). The upper clamp also
* moves from 20 to 50 so agents can request wider recall when genuinely
* needed without hitting a surprise cap.
*/
export function makeSearchMemoryTool(
substrate: Substrate,
defaultLimit: number = 20,
boundToGopId?: string,
): ToolDefinition {
return {
name: 'search_memory',
description:
'Search the conversation memory corpus for turns relevant to a query. ' +
'Call this BEFORE answering so you can ground your answer in retrieved content. ' +
'Results are ranked turn frames with speaker, text, and relevance score.' +
(boundToGopId
? ' (Scope is auto-restricted to the current conversation.)'
: ''),
offlineCapable: true,
parameters: {
type: 'object',
properties: {
query: {
type: 'string',
description: 'Natural-language search query. Focus on entities + topic.',
},
limit: {
type: 'number',
description: `Max results to return. Default ${defaultLimit}. Cap 50.`,
},
},
required: ['query'],
},
execute: async (args: Record<string, unknown>): Promise<string> => {
const query = typeof args.query === 'string' ? args.query.trim() : '';
if (!query) return 'ERROR: query is required and must be a non-empty string.';
const limitRaw = typeof args.limit === 'number' ? args.limit : defaultLimit;
const limit = Math.max(1, Math.min(50, Math.floor(limitRaw)));
const searchOpts: { limit: number; gopId?: string } = { limit };
if (boundToGopId) searchOpts.gopId = boundToGopId;
const results = await substrate.search.search(query, searchOpts);
if (results.length === 0) return '(no memories found)';
return results.map((r, idx) => {
const score = r.finalScore.toFixed(3);
return `[${idx + 1}] (${r.frame.gop_id}:${r.frame.id} score ${score}) ${r.frame.content}`;
}).join('\n');
},
};
}
function assertSubstrate(cellName: string, substrate: Substrate | undefined): asserts substrate is Substrate {
if (!substrate) {
throw new Error(
`cells.${cellName} requires a Substrate dependency. Construct it via ` +
`createSubstrate({embedder}) and pass it in CellInput.substrate. See ` +
`benchmarks/harness/src/substrate.ts.`,
);
}
}
function assertLitellm(
cellName: string,
litellm: CellInput['litellm'],
): asserts litellm is { url: string; apiKey: string } {
if (!litellm?.url || !litellm?.apiKey) {
throw new Error(
`cells.${cellName} requires litellm={url, apiKey} in CellInput ` +
`(agent-loop talks to LiteLLM directly, not via the cell's LlmClient).`,
);
}
}
export const cells: Record<CellName, CellFn> = {
raw: async ({ instance, model, llm, turnId: _turnId }: CellInput) => {
return llm.call({
model,
systemPrompt: systemPromptForCell(model, FACTOID_BASELINE_PERSONA),
userPrompt: buildUserPromptRaw(instance),
});
},
filtered: async ({ instance, model, llm, turnId: _turnId }: CellInput) => {
return llm.call({
model,
systemPrompt: systemPromptForCell(model, FACTOID_BASELINE_PERSONA),
userPrompt: buildUserPromptRetrieved(instance),
});
},
compressed: async ({ instance, model, llm, turnId: _turnId }: CellInput) => {
return llm.call({
model,
systemPrompt: systemPromptForCell(model, FACTOID_STRICT_PERSONA),
userPrompt: buildUserPromptRaw(instance),
});
},
'full-context': async ({ instance, model, llm, turnId: _turnId }: CellInput) => {
return llm.call({
model,
systemPrompt: systemPromptForCell(model, FACTOID_STRICT_PERSONA),
userPrompt: buildUserPromptRetrieved(instance),
});
},
/**
* retrieval — Sprint 12 Task 2.5 Stage 1, updated by Stage 2-Retry.
*
* Real `@waggle/core::HybridSearch` RRF-fused FTS5 + vec0 recall. Stage
* 1 ran whole-corpus top-K=10. Stage 2-Retry §1.2 scopes search to the
* instance's conversation via `gopId` filter (plumbed through
* `HybridSearch.SearchOptions.gopId`) and bumps the top-K default to 20.
* Matches LoCoMo QA-pair locality.
*
* When `instance.conversation_id` is unset (synthetic datasets, pre-
* Stage-2-Retry fixtures), falls back to whole-corpus search — preserves
* backward compatibility with existing unit tests.
*/
retrieval: async ({ instance, model, llm, turnId: _turnId, substrate, retrievalTopK }: CellInput) => {
assertSubstrate('retrieval', substrate);
const limit = retrievalTopK ?? 20;
const searchOpts: { limit: number; gopId?: string } = { limit };
if (instance.conversation_id) searchOpts.gopId = instance.conversation_id;
const results = await substrate.search.search(instance.question, searchOpts);
const memoryBlock = formatRecalledMemories(results);
const userPrompt = `${memoryBlock}\n\nQuestion: ${instance.question}`;
return llm.call({
model,
systemPrompt: systemPromptForCell(model, FACTOID_BASELINE_PERSONA),
userPrompt,
});
},
/**
* agentic — Sprint 12 Task 2.5 Stage 1 (2026-04-23).
*
* Inner `runAgentLoop` call with:
* - tools: [search_memory] — single-tool allowlist per GATE-S0 decision.
* - maxTurns: 3 — hard cap per GATE-S0 decision.
* - signal: AbortController → setTimeout(timeoutMs).
*
* Returns an `LlmCallResult` whose `text` is the agent's final answer,
* `usage` tokens come from the agent loop's aggregate, and `costUsd` is
* computed from model pricing × tokens. `failureMode` is set when the
* agent aborts or throws; null on clean completion.
*
* The agent loop talks to LiteLLM directly — it does NOT go through the
* cell's `LlmClient`. Callers must supply `litellm={url, apiKey}` in
* CellInput. (The cell's `llm: LlmClient` is kept in the signature for
* interface symmetry but is not used by this cell.)
*/
agentic: async ({ instance, model, llm, turnId, substrate, litellm, agenticMaxTurns, agenticTimeoutMs, runAgentLoopFn }: CellInput) => {
assertSubstrate('agentic', substrate);
assertLitellm('agentic', litellm);
const maxTurns = agenticMaxTurns ?? 3;
const timeoutMs = agenticTimeoutMs ?? 180_000;
const runFn = runAgentLoopFn ?? runAgentLoop;
// Stage 2-Retry §1.2: bind search_memory to this instance's conversation
// via gopId, so agent's search calls are automatically scoped. Cannot be
// overridden by the agent — conversation scope is a benchmark invariant.
const searchMemoryTool = makeSearchMemoryTool(substrate, 20, instance.conversation_id);
// Stage 2-Retry §1.4: capture tool-result text as it streams through so
// the exhaustion-fallback pass has the accumulated context available.
// Each entry is the string returned by makeSearchMemoryTool.execute.
const capturedToolResults: string[] = [];
const captureToolResult = (name: string, _input: Record<string, unknown>, result: string): void => {
if (name === 'search_memory') capturedToolResults.push(result);
};
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
const started = Date.now();
const cfg: AgentLoopConfig = {
litellmUrl: litellm.url,
litellmApiKey: litellm.apiKey,
model: model.litellmModel,
systemPrompt: SYSTEM_AGENTIC,
tools: [searchMemoryTool],
messages: [{ role: 'user', content: instance.question }],
maxTurns,
signal: controller.signal,
turnId,
onToolResult: captureToolResult,
};
try {
const resp = await runFn(cfg);
let finalContent = resp.content;
let fallbackInput = 0;
let fallbackOutput = 0;
// Stage 2-Retry §1.4 tool-exhaustion fallback: agent exhausted maxTurns
// without producing content but DID use tools → synthesize one forced-
// answer call with accumulated search context + SYSTEM_AGENTIC_FORCED_
// FALLBACK. SYSTEM_AGENTIC §7 addresses this prompt-side; this is the
// runtime guarantee. Fires only on empty-content-with-tool-use; normal
// empty-abstain ("unknown") paths don't trigger.
if (!finalContent.trim() && capturedToolResults.length > 0) {
const ctxBlock = capturedToolResults
.map((r, i) => `## search_memory call ${i + 1}\n${r}`)
.join('\n\n');
const forcedResp = await llm.call({
model,
systemPrompt: SYSTEM_AGENTIC_FORCED_FALLBACK,
userPrompt: `Question: ${instance.question}\n\n# Retrieved memory context\n${ctxBlock}`,
});
finalContent = forcedResp.text;
fallbackInput = forcedResp.inputTokens;
fallbackOutput = forcedResp.outputTokens;
}
const latencyMs = Date.now() - started;
const inputTokens = resp.usage.inputTokens + fallbackInput;
const outputTokens = resp.usage.outputTokens + fallbackOutput;
const costUsd =
(inputTokens / 1_000_000) * model.pricePerMillionInput +
(outputTokens / 1_000_000) * model.pricePerMillionOutput;
return {
text: finalContent,
inputTokens,
outputTokens,
latencyMs,
costUsd,
failureMode: null,
};
} catch (err: unknown) {
const latencyMs = Date.now() - started;
const name = err instanceof Error ? err.name : 'unknown';
const failureMode = name === 'AbortError' ? 'timeout' : `agentic_error_${name}`;
return {
text: '',
inputTokens: 0,
outputTokens: 0,
latencyMs,
costUsd: 0,
failureMode,
};
} finally {
clearTimeout(timer);
}
},
/**
* no-context — Sprint 12 Task 2.5 Stage 2-Retry §1.1 (2026-04-24).
*
* True zero-memory baseline: question-only user prompt, no
* `instance.context`, no retrieval, no memory injection. The factoid-
* baseline persona (via `systemPromptForCell`) keeps the model's output
* format consistent with raw / filtered / retrieval cells. This is
* the honest comparator for the retrieval memory-lift success criterion
* (brief §4 criterion 2): `retrieval >= no-context + 5pp`.
*
* Rationale (per Stage 2 N=20 FAIL exit §6.1): Sprint 9 `raw` embeds
* LoCoMo's oracle-selected `instance.context` in its prompt, so `raw` is
* NOT a zero-memory baseline on LoCoMo — it's an oracle-fed diagnostic
* (now exposed as v3 `oracle-context` alias). `no-context` is the
* zero-memory ground truth.
*/
'no-context': async ({ instance, model, llm, turnId: _turnId }: CellInput) => {
return llm.call({
model,
systemPrompt: systemPromptForCell(model, FACTOID_BASELINE_PERSONA),
userPrompt: `Question: ${instance.question}`,
});
},
};
/** Type-narrowing helper for the runner's cell-or-control dispatch. */
export function isCellName(name: string): name is CellName {
return (
name === 'raw' ||
name === 'filtered' ||
name === 'compressed' ||
name === 'full-context' ||
name === 'retrieval' ||
name === 'agentic' ||
name === 'no-context'
);
}
export function isControlName(name: string): name is ControlName {
return name === 'verbose-fixed';
}

View File

@@ -0,0 +1,38 @@
/**
* Control runs — sanity checks that the harness is not broken.
*
* Day 1 control: `verbose-fixed`. A prompt that deliberately tells the model
* to answer in long form. On a short-factoid accuracy metric (substring
* match), this should UNDERPERFORM the `raw` cell. If verbose-fixed scores
* equal-to or better than raw on the synthetic or real LoCoMo dataset, the
* harness scoring is suspect and must be audited before any scored run.
*
* Controls are intentionally NOT in the cells grid — they're diagnostic,
* not ablation data.
*/
import type { DatasetInstance, ModelSpec, ControlName } from './types.js';
import type { LlmClient, LlmCallResult } from './llm.js';
const SYSTEM_VERBOSE_FIXED =
'You are a careful assistant. Think step by step. Explain your reasoning in full sentences. ' +
'Provide context for your answer. Do not give short or terse responses.';
export interface ControlInput {
instance: DatasetInstance;
model: ModelSpec;
llm: LlmClient;
turnId: string;
}
export type ControlFn = (input: ControlInput) => Promise<LlmCallResult>;
export const controls: Record<ControlName, ControlFn> = {
'verbose-fixed': async ({ instance, model, llm, turnId: _turnId }: ControlInput) => {
return llm.call({
model,
systemPrompt: SYSTEM_VERBOSE_FIXED,
userPrompt: `Context: ${instance.context}\n\nQuestion: ${instance.question}`,
});
},
};

View File

@@ -0,0 +1,237 @@
/**
* Dataset loader + version hash + opt-in synthetic fallback.
*
* Canonical archives live under `benchmarks/data/<dataset>/`:
* locomo/locomo-1540.jsonl (built by scripts/build-locomo-canonical.ts)
*
* Production path: if the canonical archive is absent, the loader throws
* `DatasetMissingError`. No silent fallback — absent data used to masquerade
* as a 60-instance synthetic run, which is the substrate gap Sprint 12 Task 1
* Blocker #1 eliminates.
*
* Development convenience: `BENCH_SYNTHETIC_DATASET=1` re-enables the
* synthetic fallback with a prominent console.warn. Never use that path for
* publishable runs — its output is scaffold-only.
*
* Every external dataset carries a `dataset_version` hash (SHA-256 of the
* archive bytes). The hash is the audit anchor A3 LOCK §H-AUDIT-2 requires
* for pre-registration-conformant benchmark runs. The runner attaches the
* hash to every emitted JSONL record (types.ts §dataset_version).
*/
import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import type { DatasetInstance, DatasetSpec } from './types.js';
/** Version string for the built-in synthetic scaffold. Static because the
* instances are hard-coded in this file — any change will bump the source
* revision and therefore the git SHA, so a user looking for drift has a
* single obvious needle. */
export const SYNTHETIC_DATASET_VERSION = 'synthetic-scaffold-v1';
/** Thrown when a non-synthetic dataset archive is absent from the expected
* path and the `BENCH_SYNTHETIC_DATASET` escape hatch is not set. */
export class DatasetMissingError extends Error {
constructor(
public readonly resolvedPath: string,
public readonly datasetId: string,
) {
super(
`Dataset '${datasetId}' canonical archive missing at ${resolvedPath}. ` +
`Build it via \`npx tsx benchmarks/harness/scripts/build-locomo-canonical.ts\` ` +
`or set BENCH_SYNTHETIC_DATASET=1 for the synthetic dev fallback ` +
`(scaffold-only — do NOT use for publishable runs).`,
);
this.name = 'DatasetMissingError';
}
}
/** SHA-256 hex of the dataset archive bytes, or the static version string
* for synthetic. Throws `DatasetMissingError` when the archive is absent
* and the env escape hatch is not set. */
export function getDatasetVersion(spec: DatasetSpec, dataRoot: string): string {
if (spec.source === 'synthetic') return SYNTHETIC_DATASET_VERSION;
const resolved = path.resolve(dataRoot, spec.dataPath);
if (!fs.existsSync(resolved)) {
if (process.env.BENCH_SYNTHETIC_DATASET === '1') {
return SYNTHETIC_DATASET_VERSION;
}
throw new DatasetMissingError(resolved, spec.id);
}
const buf = fs.readFileSync(resolved);
return crypto.createHash('sha256').update(buf).digest('hex');
}
/** Preflight sample-lock schema written by scripts/build-preflight-samples.ts. */
export interface PreflightSampleInstance {
id: string;
category: 'single-hop' | 'multi-hop' | 'temporal' | 'open-ended';
context: string;
question: string;
ground_truth_answer: string;
locomo_metadata?: unknown;
}
export interface PreflightSampleFile {
_meta?: {
distribution?: Record<string, number>;
seed?: number;
[k: string]: unknown;
};
instances: PreflightSampleInstance[];
}
/** The 4-cell Stage 2 preflight gate requires exactly this distribution
* per `decisions/2026-04-20-preflight-oq-resolutions-locked.md` §OQ-PF-1. */
export const PREFLIGHT_LOCOMO_50_DISTRIBUTION = {
'single-hop': 13,
'multi-hop': 13,
'temporal': 12,
'open-ended': 12,
} as const;
function distributionOf(instances: PreflightSampleInstance[]): Record<string, number> {
const out: Record<string, number> = {};
for (const i of instances) out[i.category] = (out[i.category] ?? 0) + 1;
return out;
}
/** Loads a committed sample-lock JSON and asserts the 13/13/12/12 distribution.
*
* This is the enforcement point required by Task 1 of the CC preflight
* sprint brief. Any deviation — whether from tampering, an incomplete
* rebuild, or an accidental schema drift — must fail loudly so that no
* Stage 2 run proceeds against a silently-broken sample. */
export function loadPreflightSampleLock(lockPath: string): DatasetInstance[] {
if (!fs.existsSync(lockPath)) {
throw new Error(`Pre-flight sample lock not found at ${lockPath}`);
}
const raw = fs.readFileSync(lockPath, 'utf-8');
const parsed = JSON.parse(raw) as PreflightSampleFile;
if (!parsed || !Array.isArray(parsed.instances)) {
throw new Error(`Pre-flight sample lock at ${lockPath} is missing the "instances" array`);
}
const actual = distributionOf(parsed.instances);
const expected = PREFLIGHT_LOCOMO_50_DISTRIBUTION;
const keys: (keyof typeof expected)[] = ['single-hop', 'multi-hop', 'temporal', 'open-ended'];
const mismatch =
parsed.instances.length !== 50 ||
keys.some(k => (actual[k] ?? 0) !== expected[k]) ||
Object.keys(actual).some(k => !(k in expected));
if (mismatch) {
const actualStr = keys.map(k => `${k}=${actual[k] ?? 0}`).join('/');
const expectedStr = keys.map(k => `${k}=${expected[k]}`).join('/');
throw new Error(
`Pre-flight sample distribution mismatch: expected 13/13/12/12, got ${actualStr} ` +
`(expected breakdown: ${expectedStr}; total ${parsed.instances.length}, expected 50)`,
);
}
return parsed.instances.map(inst => {
// Stage 2-Retry §1.2: LoCoMo instance_id format is
// `locomo_<conversation-id>_q<index>`. Derive conversation_id from that
// pattern so preflight-lock instances (which don't carry the field
// directly) still scope correctly. Safe because the preflight builder
// already enforces LoCoMo-only rows in the lock file.
const convMatch = inst.id.match(/^locomo_(conv-\d+)_q\d+$/);
return {
instance_id: inst.id,
question: inst.question,
context: inst.context,
expected: [inst.ground_truth_answer],
...(convMatch ? { conversation_id: convMatch[1] } : {}),
};
});
}
/**
* Built-in 60-instance synthetic dataset — enough for the `--limit 50`
* verbose-fixed acceptance test plus 10 slack. Questions probe short-context
* recall, entity tracking, and one-hop reasoning so the scaffold exercises
* a realistic-ish prompt shape.
*/
const SYNTHETIC_INSTANCES: DatasetInstance[] = Array.from({ length: 60 }, (_, i) => {
const n = i + 1;
const topics = [
{ subj: 'Marko', verb: 'works at', obj: 'Egzakta Advisory', q: 'Where does Marko work?', a: 'Egzakta Advisory' },
{ subj: 'Ana', verb: 'leads', obj: 'the KVARK platform', q: 'Who leads the KVARK platform?', a: 'Ana' },
{ subj: 'The Waggle release', verb: 'shipped on', obj: '2026-04-20', q: 'When did Waggle ship?', a: '2026-04-20' },
{ subj: 'The benchmark', verb: 'uses model', obj: 'Qwen3.6-35B-A3B', q: 'Which model does the benchmark use?', a: 'Qwen3.6-35B-A3B' },
{ subj: 'The harness', verb: 'runs', obj: 'four cells', q: 'How many cells does the harness run?', a: 'four' },
];
const t = topics[i % topics.length];
return {
instance_id: `synthetic_${String(n).padStart(3, '0')}`,
question: t.q,
context: `${t.subj} ${t.verb} ${t.obj}.`,
expected: [t.a],
};
});
export function loadDataset(spec: DatasetSpec, dataRoot: string): DatasetInstance[] {
if (spec.source === 'synthetic') {
return SYNTHETIC_INSTANCES;
}
// External (LoCoMo / LongMemEval). Sprint 12 Task 1 Blocker #1: no silent
// fallback. Missing archive throws `DatasetMissingError`, unless the
// `BENCH_SYNTHETIC_DATASET=1` escape hatch is set (dev convenience only —
// never use for publishable runs).
const resolved = path.resolve(dataRoot, spec.dataPath);
if (!fs.existsSync(resolved)) {
if (process.env.BENCH_SYNTHETIC_DATASET === '1') {
console.warn(
`[harness] ${spec.id} archive missing at ${resolved} — ` +
`BENCH_SYNTHETIC_DATASET=1 set, falling back to synthetic scaffold. ` +
`Dev-only path; do NOT use for publishable runs.`,
);
return SYNTHETIC_INSTANCES;
}
throw new DatasetMissingError(resolved, spec.id);
}
const raw = fs.readFileSync(resolved, 'utf-8');
const out: DatasetInstance[] = [];
for (const line of raw.split('\n')) {
const trimmed = line.trim();
if (!trimmed) continue;
try {
const parsed = JSON.parse(trimmed) as Partial<DatasetInstance> & { conversation_id?: string };
if (parsed.instance_id && parsed.question && parsed.expected) {
out.push({
instance_id: parsed.instance_id,
question: parsed.question,
context: parsed.context ?? '',
expected: Array.isArray(parsed.expected) ? parsed.expected : [String(parsed.expected)],
// Stage 2-Retry §1.2: preserve conversation_id so retrieval +
// agentic cells can scope HybridSearch to the instance's
// conversation via the existing `gopId` filter at search.ts:14.
...(parsed.conversation_id ? { conversation_id: parsed.conversation_id } : {}),
});
}
} catch {
// Tolerate malformed lines (common in exported benchmark dumps) —
// skip + surface the count at the end.
}
}
return out;
}
/** Deterministic shuffle + sampling so `--seed N --limit M` is reproducible. */
export function sampleInstances(all: DatasetInstance[], seed: number, limit: number): DatasetInstance[] {
if (!Number.isFinite(limit) || limit >= all.length) return all.slice();
// xorshift32 — cheap deterministic PRNG, good enough for sampling.
let state = (seed || 1) >>> 0;
const rand = (): number => {
state ^= state << 13;
state ^= state >>> 17;
state ^= state << 5;
return (state >>> 0) / 0x100000000;
};
const shuffled = all.slice();
for (let i = shuffled.length - 1; i > 0; i--) {
const j = Math.floor(rand() * (i + 1));
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
}
return shuffled.slice(0, limit);
}

View File

@@ -0,0 +1,98 @@
/**
* Sprint 12 Task 1 Blocker #6 — failure distribution aggregator.
*
* Counts failure codes across a run's emitted rows, computes F_other rate,
* flags taxonomy-review triggers per A3 LOCK § 6 (strict >10% threshold),
* and captures up to 10 F_other rationales for manual PM review in the
* exit ping.
*/
import {
FAILURE_CODES,
F_OTHER_REVIEW_THRESHOLD,
type FailureCode,
} from './codes.js';
export interface FailureRow {
failure_code: FailureCode;
rationale?: string | null;
}
export interface FailureDistribution {
/** Counts keyed by code. `null` is the correct-verdict bucket. */
counts: Record<'null' | 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | 'F6' | 'F_other', number>;
/** Total rows counted (sum of `counts`). */
total: number;
/** F_other rate ∈ [0, 1]. Zero when total=0. */
f_other_rate: number;
/** True when f_other_rate > 10% (strict greater-than per A3 LOCK § 6). */
f_other_review_flag: boolean;
/** First 10 F_other rationales in input order, for PM manual inspection. */
f_other_rationales_sample: string[];
}
const MAX_F_OTHER_SAMPLE = 10;
function emptyCounts(): FailureDistribution['counts'] {
return {
null: 0,
F1: 0,
F2: 0,
F3: 0,
F4: 0,
F5: 0,
F6: 0,
F_other: 0,
};
}
export function computeFailureDistribution(
rows: readonly FailureRow[],
): FailureDistribution {
const counts = emptyCounts();
const f_other_rationales_sample: string[] = [];
for (const row of rows) {
const code = row.failure_code;
if (code === null) {
counts.null += 1;
continue;
}
if (!(FAILURE_CODES as readonly string[]).includes(code)) {
// Defensive skip — aggregate is lenient vs. validator. Unknown codes
// don't crash the report but also don't pollute the known-buckets.
continue;
}
counts[code] += 1;
if (code === 'F_other') {
if (
typeof row.rationale === 'string' &&
row.rationale.length > 0 &&
f_other_rationales_sample.length < MAX_F_OTHER_SAMPLE
) {
f_other_rationales_sample.push(row.rationale);
}
}
}
const total =
counts.null +
counts.F1 +
counts.F2 +
counts.F3 +
counts.F4 +
counts.F5 +
counts.F6 +
counts.F_other;
const f_other_rate = total === 0 ? 0 : counts.F_other / total;
const f_other_review_flag = f_other_rate > F_OTHER_REVIEW_THRESHOLD;
return {
counts,
total,
f_other_rate,
f_other_review_flag,
f_other_rationales_sample,
};
}

View File

@@ -0,0 +1,52 @@
/**
* Sprint 12 Task 1 Blocker #6 — failure taxonomy codes.
*
* A3 LOCK § 6 hybrid taxonomy: 6 categorical failure modes (F1–F6) +
* `null` (correct) + `F_other` escape category with mandatory rationale.
*
* Taxonomy version tag: `F1-F6+other v1` (surfaces into A3 LOCK § 7 field 14
* `failure_taxonomy_version` of the per-run manifest).
*/
/**
* 8-value failure code space:
* - `null` — correct verdict (no failure classification)
* - `'F1'`..`'F6'` — LOCKED categorical failure modes per A3 LOCK § 6
* - `'F_other'` — escape hatch; requires ≥10-word rationale per validator
*/
export type FailureCode = null | 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | 'F6' | 'F_other';
/**
* Ordered list of non-null failure codes. Used by aggregators and rubric
* renderers that need stable iteration order.
*/
export const FAILURE_CODES = ['F1', 'F2', 'F3', 'F4', 'F5', 'F6', 'F_other'] as const;
/**
* Verbatim definitions per A3 LOCK § 6. Plain-English short form suitable
* for inclusion in the judge rubric prompt block and for per-run exit-ping
* narration. Do NOT paraphrase — any edit requires PM ratification and a
* taxonomy version bump (`F1-F6+other v2`).
*/
export const FAILURE_CODE_DEFINITIONS: Record<Exclude<FailureCode, null>, string> = {
F1: 'contradicts-ground-truth — Model output asserts a fact that directly contradicts the LoCoMo reference answer. Most severe failure class.',
F2: 'partial-answer — Model output contains correct information but is incomplete against the reference\'s required components.',
F3: 'off-topic — Model output is tangentially related or addresses a different question than asked.',
F4: 'refusal — Model declines to answer (safety response, capability disclaimer, "I don\'t know").',
F5: 'tool-use-error — Model attempted a tool call but the harness returned an error, a malformed response, or an infinite loop; applies only in cells where tool use is permitted.',
F6: 'format-violation — Model output is correct in content but violates the required output format (JSON schema mismatch, wrong key names, escape errors).',
F_other: 'F-other — Judge identifies a failure that does not fit F1–F6. Mandatory ≥10-word rationale explaining the failure.',
};
/**
* Taxonomy version tag per A3 LOCK § 6 / § 7 field 14. Emitted verbatim
* into the pre-registration manifest and any aggregate report.
*/
export const FAILURE_TAXONOMY_VERSION = 'F1-F6+other v1';
/**
* Threshold at which `F_other` rate triggers a taxonomy-review flag per
* A3 LOCK § 6 ("F-other rate on any run > 10% triggers taxonomy review and
* potential v2 amendment"). Strict greater-than.
*/
export const F_OTHER_REVIEW_THRESHOLD = 0.10;

View File

@@ -0,0 +1,34 @@
/**
* Sprint 12 Task 1 Blocker #6 — failure taxonomy module barrel.
*
* A3 LOCK § 6 surface consumed by:
* - Task 2 judge rubric splicer (buildJudgeRubricBlock)
* - Task 2 judge response parser (validateFailureCodeEntry)
* - Aggregate JSON writer (computeFailureDistribution)
* - Pre-registration manifest (FAILURE_TAXONOMY_VERSION field)
*/
export {
FAILURE_CODES,
FAILURE_CODE_DEFINITIONS,
FAILURE_TAXONOMY_VERSION,
F_OTHER_REVIEW_THRESHOLD,
} from './codes.js';
export type { FailureCode } from './codes.js';
export { buildJudgeRubricBlock } from './rubric.js';
export {
validateFailureCodeEntry,
F_OTHER_RATIONALE_MIN_TOKENS,
} from './validator.js';
export type {
FailureCodeEntryInput,
ValidationErrorCode,
ValidationFailure,
ValidationResult,
ValidationSuccess,
} from './validator.js';
export { computeFailureDistribution } from './aggregate.js';
export type { FailureRow, FailureDistribution } from './aggregate.js';

View File

@@ -0,0 +1,47 @@
/**
* Sprint 12 Task 1 Blocker #6 — judge rubric block builder.
*
* Deterministic multi-line string that embeds the A3 LOCK § 6 failure
* taxonomy verbatim into the judge prompt. Task 2 (Stage 2 mini C3
* execution) will splice this block into the judge system prompt; Session
* 3 only guarantees the block exists, renders deterministically, and
* carries the LOCKED taxonomy version tag.
*
* Determinism contract: identical output on every call. No parameters, no
* time / cwd / env dependency. Same bytes every invocation.
*/
import {
FAILURE_CODE_DEFINITIONS,
FAILURE_TAXONOMY_VERSION,
} from './codes.js';
/**
* Returns the A3 LOCK § 6 failure taxonomy rubric block for inclusion in
* the judge prompt. The block is appended to the judge system prompt by
* Task 2 runtime; Session 3 ships only the renderer.
*
* Structure:
* - Header with taxonomy version tag
* - One line per non-null failure code (F1–F6 + F_other)
* - Trailing F-other escape-clause instruction matching A3 LOCK § 6
*
* The block MUST be deterministic — two successive calls produce
* byte-identical strings. Tests pin the presence of key sentinel phrases.
*/
export function buildJudgeRubricBlock(): string {
const lines: string[] = [];
lines.push(`Failure taxonomy (${FAILURE_TAXONOMY_VERSION}):`);
lines.push('');
lines.push(`F1 — ${FAILURE_CODE_DEFINITIONS.F1}`);
lines.push(`F2 — ${FAILURE_CODE_DEFINITIONS.F2}`);
lines.push(`F3 — ${FAILURE_CODE_DEFINITIONS.F3}`);
lines.push(`F4 — ${FAILURE_CODE_DEFINITIONS.F4}`);
lines.push(`F5 — ${FAILURE_CODE_DEFINITIONS.F5}`);
lines.push(`F6 — ${FAILURE_CODE_DEFINITIONS.F6}`);
lines.push('');
lines.push(
'If no category fits, select F-other and provide ≥10-word rationale explaining the failure.',
);
return lines.join('\n');
}

View File

@@ -0,0 +1,111 @@
/**
* Sprint 12 Task 1 Blocker #6 — failure-code entry validator.
*
* Enforces A3 LOCK § 6 invariants on judge-emitted {failure_code, rationale}
* pairs:
*
* - `failure_code === null` → rationale must be null/undefined
* - `failure_code` in F1..F6 → rationale optional, no length constraint
* - `failure_code === 'F_other'` → rationale mandatory, ≥10 whitespace-
* separated tokens, non-empty strings only
*
* Error codes surface the exact failure mode for downstream routing (Task 2
* judge-response parser will map these onto its own error class).
*/
import { FAILURE_CODES, type FailureCode } from './codes.js';
export type ValidationErrorCode =
| 'invalid_failure_code'
| 'null_code_with_rationale'
| 'F_other_rationale_missing'
| 'F_other_rationale_too_short';
export interface ValidationSuccess {
ok: true;
}
export interface ValidationFailure {
ok: false;
code: ValidationErrorCode;
message: string;
}
export type ValidationResult = ValidationSuccess | ValidationFailure;
export interface FailureCodeEntryInput {
failure_code: FailureCode | string;
rationale?: string | null;
}
/** Minimum token count required for F_other rationale per A3 LOCK § 6. */
export const F_OTHER_RATIONALE_MIN_TOKENS = 10;
/** Tokenise on any whitespace run; drop empty strings. */
function countRationaleTokens(rationale: string): number {
return rationale.split(/\s+/).filter(tok => tok.length > 0).length;
}
function isRecognisedCode(code: unknown): code is FailureCode {
if (code === null) return true;
if (typeof code !== 'string') return false;
return (FAILURE_CODES as readonly string[]).includes(code);
}
export function validateFailureCodeEntry(entry: FailureCodeEntryInput): ValidationResult {
const { failure_code, rationale } = entry;
if (!isRecognisedCode(failure_code)) {
return {
ok: false,
code: 'invalid_failure_code',
message: `failure_code must be null | F1..F6 | F_other; got ${JSON.stringify(failure_code)}`,
};
}
const rationaleProvided =
rationale !== undefined &&
rationale !== null &&
typeof rationale === 'string' &&
rationale.length > 0;
if (failure_code === null) {
if (rationaleProvided) {
return {
ok: false,
code: 'null_code_with_rationale',
message: 'failure_code=null (correct) must not carry a rationale',
};
}
return { ok: true };
}
if (failure_code === 'F_other') {
if (rationale === undefined || rationale === null) {
return {
ok: false,
code: 'F_other_rationale_missing',
message: 'F_other requires a non-null rationale string',
};
}
if (typeof rationale !== 'string' || rationale.trim().length === 0) {
return {
ok: false,
code: 'F_other_rationale_missing',
message: 'F_other rationale must be a non-empty string',
};
}
const tokens = countRationaleTokens(rationale);
if (tokens < F_OTHER_RATIONALE_MIN_TOKENS) {
return {
ok: false,
code: 'F_other_rationale_too_short',
message: `F_other rationale must have ≥${F_OTHER_RATIONALE_MIN_TOKENS} whitespace-separated tokens; got ${tokens}`,
};
}
return { ok: true };
}
// F1..F6: rationale is optional and has no length constraint.
return { ok: true };
}

View File

@@ -0,0 +1,140 @@
/**
* Sprint 12 Task 2.5 Stage 1.5 §7.3 — pre-cell health check.
*
* Before the runner burns ~20 instance evaluations on a cell, verify every
* upstream route the run depends on is actually reachable. Catches:
* - Judge API-key expiry (Anthropic/OpenAI/Gemini rotation)
* - LiteLLM route rename (provider changed slug between runs)
* - DashScope authentication drift
* - LiteLLM proxy container down (Docker daemon crash since last run)
*
* Two levels of probe:
* 1. GET `/health/liveliness` — is the proxy itself alive?
* 2. POST `/v1/chat/completions` with a 5-token "ping" payload for each
* required model (subject + judge ensemble). Verifies the route is
* wired AND the upstream provider is responsive.
*
* Any 5xx response or network error accumulates into `result.failures`.
* `ok` is true only when every probe returns 2xx. Caller (main) throws a
* clear error on `ok: false` so nothing starts against a broken upstream.
*
* `fetchFn` is injectable so tests can stub it without touching globals.
* `timeoutMs` defaults to 15 s per probe — thinking=on reasoning calls can
* push latency; shorter timeouts cause false negatives on slow judges.
*/
const DEFAULT_PROBE_TIMEOUT_MS = 15_000;
export interface HealthCheckOptions {
litellmUrl: string;
litellmApiKey: string;
subjectModel: string;
judgeModels?: string[];
fetchFn?: typeof globalThis.fetch;
timeoutMs?: number;
/** When true (default), also GETs `/health/liveliness`. Some LiteLLM
* configs don't expose that endpoint; set false to skip it. */
includeLivenessProbe?: boolean;
}
export interface HealthCheckFailure {
endpoint: string;
error: string;
}
export interface HealthCheckResult {
ok: boolean;
failures: HealthCheckFailure[];
probedAt: string; // ISO timestamp
durationMs: number;
}
/** Short message used in the ping body. Minimum tokens that still elicits a
* response; any live model will say "Pong" or similar. */
const PING_MESSAGE = 'Respond with just: pong';
async function probeOnce(
label: string,
fn: () => Promise<Response>,
failures: HealthCheckFailure[],
): Promise<void> {
try {
const res = await fn();
if (!res.ok) {
failures.push({ endpoint: label, error: `http_${res.status}` });
}
} catch (err: unknown) {
const name = err instanceof Error ? err.name : 'unknown';
failures.push({ endpoint: label, error: `fetch_error_${name}` });
}
}
export async function preCellHealthCheck(opts: HealthCheckOptions): Promise<HealthCheckResult> {
const fetchFn = opts.fetchFn ?? globalThis.fetch;
const timeoutMs = opts.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS;
const includeLiveness = opts.includeLivenessProbe ?? true;
const failures: HealthCheckFailure[] = [];
const started = Date.now();
const baseUrl = opts.litellmUrl.replace(/\/$/, '');
// 1. Liveness probe (skipped when includeLivenessProbe=false).
if (includeLiveness) {
await probeOnce('GET /health/liveliness', async () => {
const ctrl = new AbortController();
const t = setTimeout(() => ctrl.abort(), timeoutMs);
try {
return await fetchFn(`${baseUrl}/health/liveliness`, {
method: 'GET',
headers: { Authorization: `Bearer ${opts.litellmApiKey}` },
signal: ctrl.signal,
});
} finally {
clearTimeout(t);
}
}, failures);
}
// 2. Model ping probes (subject + each judge).
//
// Opus 4.7 + some newer reasoning-model families reject the `temperature`
// param with HTTP 400 (`temperature is deprecated for this model`). Mirror
// the regex-based detection judge-client.ts:88 already ships so the ping
// payload survives across provider generations. Also bumps max_tokens to
// match the judge-client default (1024) — 5 was too tight for providers
// that burn tokens on reasoning before content, producing empty-body
// responses that aren't technically 5xx but also aren't useful.
const probeModels: string[] = [opts.subjectModel, ...(opts.judgeModels ?? [])];
for (const model of probeModels) {
await probeOnce(`POST /v1/chat/completions model=${model}`, async () => {
const ctrl = new AbortController();
const t = setTimeout(() => ctrl.abort(), timeoutMs);
try {
const rejectsTemperature = /opus-4-7|gpt-5|o3|o4/i.test(model);
const reqBody: Record<string, unknown> = {
model,
messages: [{ role: 'user', content: PING_MESSAGE }],
max_tokens: 1024,
};
if (!rejectsTemperature) reqBody.temperature = 0.0;
return await fetchFn(`${baseUrl}/v1/chat/completions`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${opts.litellmApiKey}`,
},
signal: ctrl.signal,
body: JSON.stringify(reqBody),
});
} finally {
clearTimeout(t);
}
}, failures);
}
return {
ok: failures.length === 0,
failures,
probedAt: new Date().toISOString(),
durationMs: Date.now() - started,
};
}

View File

@@ -0,0 +1,274 @@
/**
* BEAM turn-level ingest — mirrors the LoCoMo ingest pattern in ingest.ts.
*
* Reads the BEAM canonical archive (`benchmarks/data/beam/beam.jsonl`, built
* by `build-beam-canonical.ts`) and produces an array of atomic turn frames,
* one per message in each unique conversation.
*
* BEAM canonical source shape per instance (DatasetInstance with extras):
* {
* instance_id: string, // e.g. "beam_0_q3"
* conversation_id: string, // e.g. "beam_0" (= `beam_${conversationIndex}`)
* question: string,
* expected: string[],
* context: string, // formatted full conversation text:
* // "user: ...\nassistant: ...\n..."
* chat_size?: string, // '128K' | '500K' | '1M' | '10M'
* }
*
* The conversation turns are reconstructed by parsing the `context` field of
* the FIRST instance encountered for each unique `conversation_id`. All
* instances that share a `conversation_id` reference the same conversation, so
* only one parse per conversation is needed; subsequent instances are skipped.
*
* gopId mapping: `turn.gopId = instance.conversation_id` (e.g. `"beam_0"`).
* The retrieval and agentic cells use `instance.conversation_id` as their
* gopId filter — so turns stored under `gopId = conversation_id` align
* without any cell-level changes.
*
* GATE-S0 decision (frame-per-turn granularity) applies here too. See
* ingest.ts §GATE-S0.
*/
import fs from 'node:fs';
import type { MindDB, HybridSearch, FrameStore, SessionStore } from '@waggle/core';
// ── Public types ─────────────────────────────────────────────────────────────
/** One BEAM conversation turn, ready to be written as a frame. */
export interface BeamTurn {
/** Conversation identifier — becomes `memory_frames.gop_id`.
* Format: `beam_${conversationIndex}`, matching the canonical builder. */
gopId: string;
/** Zero-based index of this message within the conversation. */
messageIndex: number;
role: 'user' | 'assistant';
content: string;
/** Formatted `"${role}: ${content}"` — the string that lands in
* `memory_frames.content` and gets FTS5 / vec-indexed. */
formattedContent: string;
/** BEAM context-window size bucket this conversation belongs to.
* Preserved from the source instance for observability; not used
* by the ingest or retrieval logic. */
chatSize: string;
}
export interface IngestStats {
/** Number of frames successfully created (after dedup). */
count: number;
/** Wall-clock ms spent on `createIFrame` loop (includes FTS5 auto-index). */
ingestMs: number;
/** Wall-clock ms spent on `indexFramesBatch` (embedder + vec0 insert). */
indexMs: number;
}
// ── Internal raw-schema types ─────────────────────────────────────────────────
interface BeamRawInstance {
instance_id?: string;
conversation_id?: string;
question?: string;
expected?: unknown;
context?: string;
chat_size?: string;
}
// ── Context parser ────────────────────────────────────────────────────────────
/**
* Parse a BEAM `context` string back into individual (role, content) pairs.
*
* The canonical builder formats the conversation as:
* "user: <text>\nassistant: <text>\nuser: <text>\n..."
*
* Lines that start with `"user: "` or `"assistant: "` begin a new turn;
* any subsequent lines that do NOT start with one of those prefixes are
* treated as continuation lines of the current turn (i.e. multi-line content
* is preserved). This matches the round-trip produced by:
* `turns.map(t => \`${t.role}: ${t.content}\`).join('\n')`
*
* Returns an empty array when `context` is empty or cannot be parsed.
*/
function parseContextToTurns(context: string): Array<{ role: 'user' | 'assistant'; content: string }> {
if (!context || !context.trim()) return [];
const result: Array<{ role: 'user' | 'assistant'; content: string }> = [];
let currentRole: 'user' | 'assistant' | null = null;
const currentLines: string[] = [];
const flush = (): void => {
if (currentRole === null || currentLines.length === 0) return;
const content = currentLines.join('\n').trim();
if (content) result.push({ role: currentRole, content });
currentLines.length = 0;
currentRole = null;
};
for (const line of context.split('\n')) {
if (line.startsWith('user: ')) {
flush();
currentRole = 'user';
currentLines.push(line.slice('user: '.length));
} else if (line.startsWith('assistant: ')) {
flush();
currentRole = 'assistant';
currentLines.push(line.slice('assistant: '.length));
} else if (currentRole !== null) {
// Continuation line for the current turn.
currentLines.push(line);
}
// Lines before the first recognisable prefix are silently skipped —
// BEAM contexts always start with a "user: " line.
}
flush();
return result;
}
// ── Turn extractor ────────────────────────────────────────────────────────────
/**
* Flatten a BEAM canonical JSONL archive into an array of atomic turn records.
*
* One record per conversation message. Per-conversation dedup applies: only
* the FIRST instance encountered for each `conversation_id` drives the context
* parse. Subsequent instances for the same conversation are skipped (they
* carry identical context).
*
* @param jsonlPath Absolute or CWD-relative path to `beam.jsonl`.
*/
export function extractTurnsFromBeam(jsonlPath: string): BeamTurn[] {
if (!fs.existsSync(jsonlPath)) {
throw new Error(
`BEAM canonical archive not found at ${jsonlPath}. ` +
`Build it via: npx tsx benchmarks/harness/scripts/build-beam-canonical.ts`,
);
}
const raw = fs.readFileSync(jsonlPath, 'utf-8');
const lines = raw.split('\n');
const out: BeamTurn[] = [];
/** Tracks which conversation_ids we've already parsed context from. */
const seenConversations = new Set<string>();
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed) continue;
let instance: BeamRawInstance;
try {
instance = JSON.parse(trimmed) as BeamRawInstance;
} catch {
// Tolerate malformed lines.
continue;
}
const conversationId = instance.conversation_id;
if (!conversationId) continue;
// Per-conversation dedup: only the first instance for each conversation_id
// has its context parsed (all instances for a conversation share the same
// context — parsing more than once just creates duplicate frames).
if (seenConversations.has(conversationId)) continue;
seenConversations.add(conversationId);
const context = instance.context ?? '';
const chatSize = instance.chat_size ?? 'unknown';
const parsedTurns = parseContextToTurns(context);
if (parsedTurns.length === 0) continue;
for (let i = 0; i < parsedTurns.length; i++) {
const { role, content } = parsedTurns[i];
out.push({
gopId: conversationId,
messageIndex: i,
role,
content,
formattedContent: `${role}: ${content}`,
chatSize,
});
}
}
return out;
}
// ── Ingest options ────────────────────────────────────────────────────────────
/**
* Default batch size for vector indexing. Matches `ingest.ts`.
* See ingest.ts for the ollama-embedder timeout rationale.
*/
const DEFAULT_INDEX_BATCH_SIZE = 200;
export interface IngestOptions {
/** Vector-index batch size. Default 200. Callers with fast/parallel
* embedders can raise this; callers hitting timeouts should lower it. */
batchSize?: number;
}
// ── Corpus ingest ─────────────────────────────────────────────────────────────
/**
* Ingest a BEAM turn stream into an ephemeral MindDB + HybridSearch pair.
*
* Signature is intentionally identical to `ingestLoCoMoCorpus` in `ingest.ts`
* so callers can swap the three ingest functions without changing their
* substrate wiring. The only semantic difference is the session label written
* to `SessionStore.ensure()` (`'beam-benchmark'`), which is cosmetic.
*
* Each turn becomes one I-frame with:
* - `gop_id = turn.gopId` (= instance.conversation_id, e.g. "beam_0")
* - `content = turn.formattedContent` (= `"${role}: ${content}"`)
* - `source = 'import'`
* - `importance = 'normal'`
*
* The `gopId` value matches the `conversation_id` stored on every
* `DatasetInstance` for BEAM — so the retrieval and agentic cells' existing
* `gopId` filter in `HybridSearch.search()` scopes to the right conversation
* without any cell-level changes.
*
* Caller owns the MindDB + HybridSearch lifecycle (see `createSubstrate` in
* `substrate.ts`). Call `substrate.close()` in your `finally` block.
*/
export async function ingestBeamCorpus(
db: MindDB,
search: HybridSearch,
frames: FrameStore,
sessions: SessionStore,
turns: BeamTurn[],
options: IngestOptions = {},
): Promise<IngestStats> {
void db; // reserved for future per-db hooks; kept for signature symmetry with ingestLoCoMoCorpus
const batchSize = Math.max(1, options.batchSize ?? DEFAULT_INDEX_BATCH_SIZE);
const ingestStart = Date.now();
const toIndex: Array<{ id: number; content: string }> = [];
const seen = new Set<number>();
// memory_frames.gop_id → sessions.gop_id is a FOREIGN KEY. Ensure one
// session row per conversation exists BEFORE any createIFrame call fires.
const ensuredGops = new Set<string>();
for (const turn of turns) {
if (!ensuredGops.has(turn.gopId)) {
sessions.ensure(turn.gopId, 'beam-benchmark', `BEAM conversation ${turn.gopId}`);
ensuredGops.add(turn.gopId);
}
const frame = frames.createIFrame(turn.gopId, turn.formattedContent, 'normal', 'import');
if (seen.has(frame.id)) continue; // dedup-collapsed duplicate
seen.add(frame.id);
toIndex.push({ id: frame.id, content: turn.formattedContent });
}
const ingestMs = Date.now() - ingestStart;
// Chunk the vector-index batch to avoid timeout issues on slow embedders.
const indexStart = Date.now();
for (let i = 0; i < toIndex.length; i += batchSize) {
const slice = toIndex.slice(i, i + batchSize);
await search.indexFramesBatch(slice);
}
const indexMs = Date.now() - indexStart;
return { count: toIndex.length, ingestMs, indexMs };
}

View File

@@ -0,0 +1,255 @@
/**
* LME V1 turn-level ingest — mirrors the LoCoMo ingest pattern in ingest.ts.
*
* Reads the LongMemEval canonical archive (`benchmarks/data/longmemeval/
* longmemeval.jsonl`, built by build-longmemeval-canonical.ts) and produces
* an array of atomic turn frames, one per message per session.
*
* LME V1 source shape per instance:
* {
* instance_id: string, // unique QA-pair id
* conversation_id: string, // equals question_id; conversation scope
* question: string,
* expected: string[],
* context: string, // formatted conversation (unused by ingest)
* sessions: Array<{
* session_id: string,
* date?: string, // optional ISO date for the session
* messages: Array<{
* role: 'user' | 'assistant',
* content: string,
* }>,
* }>,
* }
*
* The `retrieval` and `agentic` cells already use `instance.conversation_id`
* as a gopId filter in HybridSearch. This module stores each turn under
* `gopId = instance.conversation_id` so the filter scope aligns correctly.
* Because multiple instances share the same conversation_id, turns are
* deduplicated per conversation: only the FIRST instance encountered for
* each conversation_id drives the session extraction. Later instances for
* the same conversation carry the same turns — ingesting them twice would
* pollute the vector index with byte-identical duplicates.
*
* GATE-S0 decision (frame-per-turn granularity) applies here too: one frame
* per message, not one frame per session. See ingest.ts §GATE-S0.
*/
import fs from 'node:fs';
import type { MindDB, HybridSearch, FrameStore, SessionStore } from '@waggle/core';
// ── Public types ─────────────────────────────────────────────────────────────
/** One LME V1 session message after parsing, ready to be written as a frame. */
export interface LongMemEvalTurn {
/** Conversation identifier — becomes `memory_frames.gop_id`.
* Equals `instance.conversation_id` from the canonical builder. */
gopId: string;
/** Zero-based index within the flattened session-message sequence. */
messageIndex: number;
role: 'user' | 'assistant';
/** Session identifier from `sessions[N].session_id`. */
sessionId: string;
/** Optional ISO date from `sessions[N].date`. */
sessionDate?: string;
content: string;
/** Formatted `"${role}: ${content}"` — the string that lands in
* `memory_frames.content` and gets FTS5 / vec-indexed. Mirrors the
* LoCoMo `"${speaker}: ${text}"` pattern from ingest.ts. */
formattedContent: string;
}
export interface IngestStats {
/** Number of frames successfully created (after dedup). */
count: number;
/** Wall-clock ms spent on `createIFrame` loop (includes FTS5 auto-index). */
ingestMs: number;
/** Wall-clock ms spent on `indexFramesBatch` (embedder + vec0 insert). */
indexMs: number;
}
// ── Internal raw-schema types ─────────────────────────────────────────────────
interface LmeRawMessage {
role: string;
content: string;
}
interface LmeRawSession {
session_id?: string;
date?: string;
messages?: LmeRawMessage[];
}
interface LmeRawInstance {
instance_id?: string;
conversation_id?: string;
question?: string;
expected?: unknown;
context?: string;
sessions?: LmeRawSession[];
}
// ── Turn extractor ────────────────────────────────────────────────────────────
/**
* Flatten a LongMemEval canonical JSONL archive into an array of atomic turn
* records.
*
* One record per `sessions[].messages[]` entry across ALL instances in the
* file, with per-conversation dedup: if two instances share the same
* `conversation_id`, only the FIRST encountered instance's sessions are
* extracted. Downstream `ingestLongMemEvalCorpus` will hit `FrameStore`'s
* own content-hash dedup, but this pre-dedup keeps the returned array lean
* and avoids redundant embedder calls.
*
* Session order is preserved verbatim (array index order in the source).
* Message order within each session is also preserved verbatim.
*
* @param jsonlPath Absolute or CWD-relative path to `longmemeval.jsonl`.
*/
export function extractTurnsFromLongMemEval(jsonlPath: string): LongMemEvalTurn[] {
if (!fs.existsSync(jsonlPath)) {
throw new Error(
`LongMemEval canonical archive not found at ${jsonlPath}. ` +
`Build it via: npx tsx benchmarks/harness/scripts/build-longmemeval-canonical.ts`,
);
}
const raw = fs.readFileSync(jsonlPath, 'utf-8');
const lines = raw.split('\n');
const out: LongMemEvalTurn[] = [];
/** Tracks which conversation_ids we've already extracted sessions from. */
const seenConversations = new Set<string>();
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed) continue;
let instance: LmeRawInstance;
try {
instance = JSON.parse(trimmed) as LmeRawInstance;
} catch {
// Tolerate malformed lines — consistent with datasets.ts loadDataset behaviour.
continue;
}
const conversationId = instance.conversation_id;
if (!conversationId) continue;
// Per-conversation dedup: only the first instance for each conversation_id
// drives the session extraction (all instances for a conversation share the
// same session history — re-ingesting would create duplicate frames).
if (seenConversations.has(conversationId)) continue;
seenConversations.add(conversationId);
if (!Array.isArray(instance.sessions)) continue;
let messageIndex = 0;
for (const session of instance.sessions) {
if (!session || !Array.isArray(session.messages)) continue;
const sessionId = session.session_id ?? `session_${messageIndex}`;
const sessionDate = typeof session.date === 'string' ? session.date : undefined;
for (const msg of session.messages) {
if (!msg || typeof msg.content !== 'string' || !msg.content.trim()) continue;
const role = msg.role === 'assistant' ? 'assistant' : 'user';
out.push({
gopId: conversationId,
messageIndex,
role,
sessionId,
...(sessionDate !== undefined ? { sessionDate } : {}),
content: msg.content,
formattedContent: `${role}: ${msg.content}`,
});
messageIndex++;
}
}
}
return out;
}
// ── Ingest options ────────────────────────────────────────────────────────────
/**
* Default batch size for vector indexing. Matches `ingest.ts` DEFAULT_INDEX_BATCH_SIZE.
* See ingest.ts for the rationale (ollama-embedder 30 s per-request timeout).
*/
const DEFAULT_INDEX_BATCH_SIZE = 200;
export interface IngestOptions {
/** Vector-index batch size. Default 200. Callers with fast/parallel
* embedders can raise this; callers hitting timeouts should lower it. */
batchSize?: number;
}
// ── Corpus ingest ─────────────────────────────────────────────────────────────
/**
* Ingest a LongMemEval turn stream into an ephemeral MindDB + HybridSearch pair.
*
* Signature is intentionally identical to `ingestLoCoMoCorpus` in `ingest.ts`
* so callers can swap the two without changing their substrate wiring.
*
* Each turn becomes one I-frame with:
* - `gop_id = turn.gopId` (= instance.conversation_id)
* - `content = turn.formattedContent` (= `"${role}: ${content}"`)
* - `source = 'import'`
* - `importance = 'normal'`
*
* The `gopId` value matches the `conversation_id` stored on every
* `DatasetInstance` for LME V1 — so the retrieval and agentic cells'
* existing `gopId` filter in `HybridSearch.search()` will correctly scope
* to the right conversation without any cell-level changes.
*
* FTS5 indexing fires automatically inside `createIFrame`. Vector indexing is
* batched via `indexFramesBatch` in chunks of `batchSize` so slow/rate-limited
* embedders don't hit request timeouts on large corpora.
*
* Caller owns the MindDB + HybridSearch lifecycle (see `createSubstrate` in
* `substrate.ts`). Call `substrate.close()` in your `finally` block.
*/
export async function ingestLongMemEvalCorpus(
db: MindDB,
search: HybridSearch,
frames: FrameStore,
sessions: SessionStore,
turns: LongMemEvalTurn[],
options: IngestOptions = {},
): Promise<IngestStats> {
void db; // reserved for future per-db hooks; kept for signature symmetry with ingestLoCoMoCorpus
const batchSize = Math.max(1, options.batchSize ?? DEFAULT_INDEX_BATCH_SIZE);
const ingestStart = Date.now();
const toIndex: Array<{ id: number; content: string }> = [];
const seen = new Set<number>();
// memory_frames.gop_id → sessions.gop_id is a FOREIGN KEY. Ensure one
// session row per conversation exists BEFORE any createIFrame call fires.
const ensuredGops = new Set<string>();
for (const turn of turns) {
if (!ensuredGops.has(turn.gopId)) {
sessions.ensure(turn.gopId, 'longmemeval-benchmark', `LME V1 conversation ${turn.gopId}`);
ensuredGops.add(turn.gopId);
}
const frame = frames.createIFrame(turn.gopId, turn.formattedContent, 'normal', 'import');
if (seen.has(frame.id)) continue; // dedup-collapsed duplicate
seen.add(frame.id);
toIndex.push({ id: frame.id, content: turn.formattedContent });
}
const ingestMs = Date.now() - ingestStart;
// Chunk the vector-index batch so a slow embedder can't blow the per-request
// timeout on a large corpus. Each chunk is one sqlite-vec transaction.
const indexStart = Date.now();
for (let i = 0; i < toIndex.length; i += batchSize) {
const slice = toIndex.slice(i, i + batchSize);
await search.indexFramesBatch(slice);
}
const indexMs = Date.now() - indexStart;
return { count: toIndex.length, ingestMs, indexMs };
}

View File

@@ -0,0 +1,209 @@
/**
* Task 2.5 Stage 1 — LoCoMo turn-level ingest.
*
* Reads the raw LoCoMo archive (`benchmarks/data/locomo10.json` from the
* snap-research/locomo repo) and produces a stream of atomic turn frames.
* Each LoCoMo conversation is a `{speaker_a, speaker_b, session_N_date_time,
* session_N: LocomoTurn[]}` object; we enumerate every `session_N` array,
* flatten its turns, and emit one `{gopId, diaId, speaker, text, content}`
* record per turn.
*
* GATE-S0 decision (2026-04-23, PM Marko Marković): granularity = frame-per-turn.
* Rationale: frame-per-conversation (10 frames, via dedup collapse) degenerates
* the retrieval cell into the full-context cell and kills the 4-cell ablation
* signal. See sessions/2026-04-23-task25-s0-readiness.md §0.3 and the Stage 1
* brief for the adjudication record.
*
* The ingest wrapper composes two primitives that already exist in
* `@waggle/core`:
* - `FrameStore.createIFrame(gopId, content, importance, source)` — inserts
* one memory_frames row + auto-indexes FTS5.
* - `HybridSearch.indexFramesBatch([{id, content}])` — atomic batch vector
* index via `embedder.embedBatch(contents)`.
*
* No LLM calls. Ingest is offline and costs nothing (the embedder is local:
* ollama-embedder for production runs, a deterministic fake for unit tests).
*/
import fs from 'node:fs';
import type { MindDB, HybridSearch, FrameStore, SessionStore } from '@waggle/core';
/** Raw LoCoMo turn shape from `benchmarks/data/locomo10.json`. Matches the
* snap-research/locomo schema. Optional fields (`img_url`, `blip_caption`,
* `query`) are ignored by the ingest — they don't carry text-level memory. */
export interface LocomoRawTurn {
speaker: string;
dia_id: string;
text: string;
img_url?: string[];
blip_caption?: string;
query?: string;
}
export interface LocomoRawConversation {
speaker_a: string;
speaker_b: string;
[sessionKey: string]: string | LocomoRawTurn[];
}
export interface LocomoRawSample {
sample_id: string;
conversation: LocomoRawConversation;
qa: unknown[];
}
/** Extracted turn, ready to be written as a frame. `content` is the string
* that lands in `memory_frames.content` and gets FTS5/vec-indexed. */
export interface LocomoTurn {
/** Conversation identifier — becomes `memory_frames.gop_id`. */
gopId: string;
/** LoCoMo evidence id (`D<session>:<turn>`). Preserved for traceability. */
diaId: string;
speaker: string;
text: string;
/** Formatted `"{speaker}: {text}"` — the content that gets embedded. */
content: string;
}
export interface IngestStats {
/** Number of frames successfully created. */
count: number;
/** Wall-clock ms spent on `createIFrame` loop (includes FTS5 auto-index). */
ingestMs: number;
/** Wall-clock ms spent on `indexFramesBatch` (embedder + vec0 insert). */
indexMs: number;
}
/**
* Flatten a raw LoCoMo archive into an array of atomic turn records.
*
* One record per turn across every `session_N` array in every conversation.
* Session order within a conversation is numeric ascending; turn order within
* a session is preserved verbatim from the source. This matches the paper's
* "1540 atomic frames across 10 LoCoMo conversations (~154 turns each)" claim
* — the actual count may differ slightly from 1540 because the source file
* can have variable turn counts per conversation.
*/
export function extractTurnsFromLocomoRaw(rawPath: string): LocomoTurn[] {
if (!fs.existsSync(rawPath)) {
throw new Error(
`LoCoMo raw archive not found at ${rawPath}. Download with: ` +
`curl -sL -o benchmarks/data/locomo10.json ` +
`https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json`,
);
}
const raw = fs.readFileSync(rawPath, 'utf-8');
let samples: LocomoRawSample[];
try {
samples = JSON.parse(raw) as LocomoRawSample[];
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
throw new Error(`LoCoMo archive at ${rawPath} is not valid JSON: ${msg}`);
}
if (!Array.isArray(samples)) {
throw new Error(`LoCoMo archive at ${rawPath} must be a JSON array of samples`);
}
const out: LocomoTurn[] = [];
for (const sample of samples) {
if (!sample?.sample_id || !sample.conversation) continue;
const sessionKeys = Object.keys(sample.conversation)
.filter(k => /^session_\d+$/.test(k))
.sort((a, b) => parseSessionNumber(a) - parseSessionNumber(b));
for (const key of sessionKeys) {
const turns = sample.conversation[key];
if (!Array.isArray(turns)) continue;
for (const turn of turns) {
if (!turn?.dia_id || typeof turn.text !== 'string' || !turn.speaker) continue;
out.push({
gopId: sample.sample_id,
diaId: turn.dia_id,
speaker: turn.speaker,
text: turn.text,
content: `${turn.speaker}: ${turn.text}`,
});
}
}
}
return out;
}
function parseSessionNumber(key: string): number {
const m = key.match(/^session_(\d+)$/);
return m ? Number(m[1]) : Number.POSITIVE_INFINITY;
}
/**
* Ingest a turn stream into an ephemeral MindDB + HybridSearch pair.
*
* Each turn becomes one I-frame with `gop_id = turn.gopId`, `content =
* turn.content`, `source = 'import'`, `importance = 'normal'`. After all
* frames are created, a single `indexFramesBatch` call embeds them in one
* sqlite-vec transaction. FTS5 indexing is handled automatically by
* `createIFrame` via `indexFts`.
*
* Caller owns the MindDB + HybridSearch lifecycle (see `createSubstrate` in
* `substrate.ts`). Dedup (via `FrameStore.findDuplicate(content)`) is
* expected to be a no-op at turn granularity — two turns with byte-identical
* `"{speaker}: {text}"` across conversations are vanishingly rare in LoCoMo.
* When it does fire, the returned `count` reflects the deduplicated total so
* the caller's vector-index batch stays in sync with the frame table.
*/
/**
* Default batch size for vector indexing. ollama-embedder has a hardcoded 30s
* per-request timeout and nomic-embed-text handles ~200 short turns/request
* comfortably; larger batches can hit the timeout on slower machines or
* larger embedding models. 200 is a conservative default that works on a
* dev workstation; callers can override via the `batchSize` option for
* faster embedders or tighter memory budgets.
*/
const DEFAULT_INDEX_BATCH_SIZE = 200;
export interface IngestOptions {
/** Vector-index batch size. Default 200. Callers with fast/parallel
* embedders can raise this; callers hitting timeouts should lower it. */
batchSize?: number;
}
export async function ingestLoCoMoCorpus(
db: MindDB,
search: HybridSearch,
frames: FrameStore,
sessions: SessionStore,
turns: LocomoTurn[],
options: IngestOptions = {},
): Promise<IngestStats> {
void db; // reserved for future per-db hooks; kept for signature symmetry
const batchSize = Math.max(1, options.batchSize ?? DEFAULT_INDEX_BATCH_SIZE);
const ingestStart = Date.now();
const toIndex: Array<{ id: number; content: string }> = [];
const seen = new Set<number>();
// memory_frames.gop_id → sessions.gop_id is a FOREIGN KEY. Ensure one
// session row per conversation exists BEFORE any createIFrame call fires.
const ensuredGops = new Set<string>();
for (const turn of turns) {
if (!ensuredGops.has(turn.gopId)) {
sessions.ensure(turn.gopId, 'locomo-benchmark', `LoCoMo conversation ${turn.gopId}`);
ensuredGops.add(turn.gopId);
}
const frame = frames.createIFrame(turn.gopId, turn.content, 'normal', 'import');
if (seen.has(frame.id)) continue; // dedup-collapsed duplicate
seen.add(frame.id);
toIndex.push({ id: frame.id, content: turn.content });
}
const ingestMs = Date.now() - ingestStart;
// Chunk the vector-index batch so a slow embedder (ollama, API with
// rate limits) can't blow the per-request timeout on a large corpus.
// sqlite-vec's vec0 insert is already transactional per `indexFramesBatch`
// call, so chunking preserves atomicity per batch (just with multiple
// transactions end-to-end — the same data lands either way).
const indexStart = Date.now();
for (let i = 0; i < toIndex.length; i += batchSize) {
const slice = toIndex.slice(i, i + batchSize);
await search.indexFramesBatch(slice);
}
const indexMs = Date.now() - indexStart;
return { count: toIndex.length, ingestMs, indexMs };
}

View File

@@ -0,0 +1,176 @@
/**
* Judge LLM client — thin wrapper around LiteLLM chat completions that
* implements the `LlmClient` interface the failure-mode-judge module
* expects (`complete(prompt: string): Promise<string>`).
*
* Sprint 9 Task 2. Two retries with exponential backoff (1s, 3s) per
* brief §Failure-handling. Parse-level retry (the reminder-and-retry
* for malformed JSON) is handled INSIDE the judge module itself —
* this client only retries transport-level failures (HTTP non-2xx,
* fetch errors, timeouts). Keeping the two concerns separated stops
* a single flaky network hop from eating both retry budgets at once.
*
* Cost tracking: the underlying LiteLLM response carries `usage.*` and
* sometimes `cost` on the message envelope; the caller that constructs
* this client passes a cost table so the runner can aggregate per-cell
* judge spend in the Task-3 rollup.
*/
import type { LlmClient } from './judge-types.js';
export type { LlmClient } from './judge-types.js';
export interface JudgeClientCostEntry {
/** ISO-8601 timestamp of the call. */
timestamp: string;
/** Model id used for the judge call. */
model: string;
promptTokens: number;
completionTokens: number;
usd: number;
latencyMs: number;
/** `true` when the call succeeded, `false` when all retries exhausted. */
ok: boolean;
}
export interface JudgeLlmClientConfig {
litellmUrl: string;
litellmApiKey: string;
model: string;
/** USD per 1M tokens — [input, output]. Defaults to Sonnet tier rates. */
pricePerMillionInput?: number;
pricePerMillionOutput?: number;
/** Called once per completed attempt (success or final failure). The
* runner's cost aggregator reads this to populate the Task-3 cost
* summary in aggregate.ts. Optional — tests can omit. */
onCall?: (entry: JudgeClientCostEntry) => void;
/** Injection seam for tests: replaces `fetch` so unit tests never
* reach the network. Production path leaves this undefined and uses
* the global fetch. */
fetchImpl?: typeof fetch;
/** Abort individual attempt after N ms. Default 30_000. */
timeoutMs?: number;
/** Override the default backoff schedule ([1000, 3000] per brief).
* Exposed so tests can collapse sleeps to near-zero and stay fast. */
backoffMs?: number[];
}
/** Default Sonnet-4.6 pricing (current 2026-04 list rate). */
const DEFAULT_PRICE_PER_MILLION_INPUT = 3.0;
const DEFAULT_PRICE_PER_MILLION_OUTPUT = 15.0;
const DEFAULT_BACKOFF_MS = [1000, 3000];
export function createJudgeLlmClient(config: JudgeLlmClientConfig): LlmClient {
const url = config.litellmUrl.replace(/\/$/, '');
const priceIn = config.pricePerMillionInput ?? DEFAULT_PRICE_PER_MILLION_INPUT;
const priceOut = config.pricePerMillionOutput ?? DEFAULT_PRICE_PER_MILLION_OUTPUT;
const fetchFn = config.fetchImpl ?? fetch;
const timeoutMs = config.timeoutMs ?? 30_000;
const backoff = config.backoffMs ?? DEFAULT_BACKOFF_MS;
async function sleep(ms: number): Promise<void> {
return new Promise(resolve => { setTimeout(resolve, ms); });
}
async function attempt(prompt: string): Promise<{
text: string;
promptTokens: number;
completionTokens: number;
latencyMs: number;
}> {
const started = Date.now();
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
// Opus 4.7 and some newer reasoning-model families reject the
// `temperature` param with HTTP 400. Omit it for those; they're
// effectively deterministic at their provider defaults. Keep T=0
// for the rest for reproducibility.
const rejectsTemperature = /opus-4-7|gpt-5|o3|o4/i.test(config.model);
const reqBody: Record<string, unknown> = {
model: config.model,
messages: [{ role: 'user', content: prompt }],
max_tokens: 1024,
};
if (!rejectsTemperature) reqBody.temperature = 0.0;
const res = await fetchFn(`${url}/v1/chat/completions`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${config.litellmApiKey}`,
},
signal: controller.signal,
body: JSON.stringify(reqBody),
});
const latencyMs = Date.now() - started;
if (!res.ok) {
const body = await res.text();
throw new Error(`judge-client ${config.model} HTTP ${res.status}: ${body.slice(0, 240)}`);
}
const body = (await res.json()) as {
choices?: Array<{ message?: { content?: string; reasoning_content?: string } }>;
usage?: { prompt_tokens?: number; completion_tokens?: number };
};
const message = body.choices?.[0]?.message ?? {};
const content = typeof message.content === 'string' ? message.content : '';
// Thinking-mode providers occasionally return empty `content` with
// the parsed JSON hiding in `reasoning_content`. Fall back so the
// judge module's JSON extractor still has a chance to find the
// payload — its `extractJsonBody` strips prose and fence wrappers.
const reasoning = typeof message.reasoning_content === 'string' ? message.reasoning_content : '';
const text = content || reasoning;
return {
text,
promptTokens: body.usage?.prompt_tokens ?? 0,
completionTokens: body.usage?.completion_tokens ?? 0,
latencyMs,
};
} finally {
clearTimeout(timer);
}
}
return {
async complete(prompt: string): Promise<string> {
// Up to `backoff.length + 1` total attempts: one initial + len(backoff) retries.
const totalAttempts = backoff.length + 1;
let lastErr: unknown = null;
for (let i = 0; i < totalAttempts; i++) {
try {
const result = await attempt(prompt);
config.onCall?.({
timestamp: new Date().toISOString(),
model: config.model,
promptTokens: result.promptTokens,
completionTokens: result.completionTokens,
usd:
(result.promptTokens / 1_000_000) * priceIn +
(result.completionTokens / 1_000_000) * priceOut,
latencyMs: result.latencyMs,
ok: true,
});
return result.text;
} catch (err) {
lastErr = err;
if (i < backoff.length) {
await sleep(backoff[i]);
}
}
}
// All attempts exhausted — surface one cost log entry marked failed
// so the aggregator can account for consumed budget even on total
// loss, then throw for the caller's retry-vs-skip decision.
config.onCall?.({
timestamp: new Date().toISOString(),
model: config.model,
promptTokens: 0,
completionTokens: 0,
usd: 0,
latencyMs: 0,
ok: false,
});
throw lastErr instanceof Error
? lastErr
: new Error(`judge-client ${config.model} failed after ${totalAttempts} attempts`);
},
};
}

View File

@@ -0,0 +1,401 @@
/**
* Judge runner adapter — wraps the failure-mode-judge module for use
* inside the benchmark runner loop.
*
* Sprint 9 Task 2. The cells produce raw LLM answers; this module takes
* (question, ground_truth, context_excerpt, model_answer) from each
* instance + cell result and returns a JSONL-ready verdict payload.
*
* Single-judge path is the production default. `judgeEnsemble` path is
* exposed for the calibration Task 5 (Fleiss' kappa probe) and for
* future Week-1 ensemble runs per taxonomy §6.
*
* Parse failures are already handled inside the judge module (one
* reminder-retry, then JudgeParseError). Transport failures are handled
* inside the JudgeLlmClient (two retries with exponential backoff).
* Both failure surfaces are caught here and converted into an
* `unjudged` result so the run continues — losing one judge call never
* aborts a whole Stage-2 batch.
*/
import type { LlmClient } from './judge-types.js';
import type { FailureCode, FailureMode, JudgeEnsembleEntry, JudgeVerdict } from './types.js';
/**
* Map a Sprint 9 5-value `FailureMode` onto the A3 LOCK § 6 8-value
* `FailureCode` space per decisions/2026-04-23-jsonl-record-taxonomy-split-locked.md.
*
* Sprint 9 F1..F5 semantics align 1:1 with A3 § 6 F1..F5 (refusal / partial /
* off-topic / hallucination / incorrect). F6 (format-violation) and F_other
* (≥10-word rationale escape) are A3-only surfaces — they appear on the
* harness output only after the judge rubric splice upgrade (follow-on of
* §2.1 in the Task 2 runtime; this mapper passes them through as-is on the
* judge-emit path).
*/
function mapLegacyToA3(legacy: FailureMode | null | undefined): FailureCode {
if (legacy === undefined || legacy === null) return null;
// FailureMode ⊂ FailureCode at the string level; cast is semantically safe.
return legacy as FailureCode;
}
// The judge module lives in a sibling workspace; tsc's `rootDir: "src"`
// refuses a direct typed import (TS6059). We resolve the runtime
// module at call time via dynamic import + ambient shape typing, which
// keeps the compile-time contract local to the harness (see
// judge-types.ts) while the real implementation ships from server/.
//
// This is deliberately narrow — only the symbols the runner consumes
// are typed here, mirroring the canonical declarations in the server
// judge module. Any shape drift between the two will surface as a
// runtime TypeError at the first call site, not a silent downgrade.
interface JudgeModule {
judgeAnswer(params: {
question: string;
groundTruth: string;
contextExcerpt: string;
modelAnswer: string;
judgeModel: string;
llmClient: LlmClient;
}): Promise<{
verdict: JudgeVerdict;
failure_mode: null | FailureMode;
rationale: string;
judge_model: string;
}>;
judgeEnsemble(params: {
question: string;
groundTruth: string;
contextExcerpt: string;
modelAnswer: string;
judgeModels: string[];
llmClients: Map<string, LlmClient>;
}): Promise<{
ensemble: Array<{
verdict: JudgeVerdict;
failure_mode: null | FailureMode;
rationale: string;
judge_model: string;
}>;
majority: {
verdict: JudgeVerdict;
failure_mode: null | FailureMode;
rationale: string;
judge_model: string;
};
fleissKappa: number;
}>;
JudgeParseError: new (...args: unknown[]) => Error;
}
let cachedModule: JudgeModule | null = null;
/** Compute the runtime path to the judge module from the harness dist/src
* location at call time. Building the string from `import.meta.url` keeps
* tsc from chasing it during rootDir resolution (TS6059) and defers the
* path to vite-node / tsx / runtime ESM loader. */
async function loadJudgeModule(): Promise<JudgeModule> {
if (cachedModule) return cachedModule;
const { fileURLToPath, pathToFileURL } = await import('node:url');
const nodePath = await import('node:path');
const here = fileURLToPath(import.meta.url);
// `harness/src/judge-runner.ts` OR `harness/dist/judge-runner.js` → up to repo root.
const repoRoot = nodePath.resolve(nodePath.dirname(here), '..', '..', '..');
// Prefer src (for tsx/vitest) and fall back to compiled dist (for `node`).
const candidates = [
nodePath.resolve(repoRoot, 'packages/server/src/benchmarks/judge/failure-mode-judge.ts'),
nodePath.resolve(repoRoot, 'packages/server/src/benchmarks/judge/failure-mode-judge.js'),
nodePath.resolve(repoRoot, 'packages/server/dist/benchmarks/judge/failure-mode-judge.js'),
];
const fs = await import('node:fs');
const target = candidates.find(p => fs.existsSync(p));
if (!target) {
throw new Error(
`judge module not found — looked in:\n ${candidates.join('\n ')}`,
);
}
cachedModule = (await import(pathToFileURL(target).href)) as JudgeModule;
return cachedModule;
}
export interface JudgeTriple {
question: string;
groundTruth: string;
contextExcerpt: string;
modelAnswer: string;
}
/** Subset of JsonlRecord fields the runner copies from this payload.
* Fields are `undefined` when judging was skipped, disabled, or failed
* irrecoverably — the runner writes only the fields this payload
* supplied (consumers treat `undefined` as "not judged yet"). */
export interface JudgePayload {
model_answer: string;
judge_verdict?: JudgeVerdict;
judge_failure_mode?: FailureMode | null;
judge_rationale?: string;
judge_model?: string;
judge_timestamp?: string;
judge_ensemble?: JudgeEnsembleEntry[];
/** Non-null when the judge call failed after all retries. The runner
* logs this but does NOT surface it as a cell-level failure_mode —
* the cell succeeded, only the post-hoc grading failed. */
judge_error?: string;
/**
* Sprint 11 B2 fold-in (2026-04-22): path the ensemble resolver took.
* - `undefined` on single-judge runs or when the tie-break module
* was never consulted (e.g. 3-0 consensus, 2-1 majority — handled
* by legacy `computeMajority` inside `judgeEnsemble`).
* - `'none'` / `'majority'` when resolveTieBreak short-circuited.
* - `'quadri-vendor'` when 1-1-1 was escalated to the fourth vendor
* and resolved.
* - `'pm-escalation'` when the four votes produced 1-1-1-1 → runtime
* surfaces this as `judge_error: 'PM_ESCALATION'` so the aggregator
* treats the instance as skipped (no silent coin-flip verdict).
*/
tie_break_path?: 'none' | 'majority' | 'quadri-vendor' | 'pm-escalation';
/** Model slug that cast the fourth vote when `tie_break_path === 'quadri-vendor'`
* or `'pm-escalation'`. e.g. `'xai/grok-4.20'`. */
tie_break_fourth_vendor?: string;
// ── Sprint 12 Task 2 §2.1 A3 namespace split (LOCKED 2026-04-23) ──────
/**
* A3 LOCK § 6 failure-code column. Populated alongside the legacy
* `judge_failure_mode` per namespace-split decision doc — legacy field
* is preserved verbatim for backward compat with pre-A3 consumers, a3
* field is the authoritative A3 exit-criterion column.
*/
a3_failure_code?: FailureCode;
/** A3 rationale. Non-null when a3_failure_code === 'F_other' (validator-
* enforced). Null for correct verdicts and optional for F1..F6. */
a3_rationale?: string | null;
}
export interface SingleJudgeConfig {
kind: 'single';
model: string;
client: LlmClient;
}
export interface EnsembleJudgeConfig {
kind: 'ensemble';
/** Ordered list — index 0 is the tie-breaker (Sonnet by convention
* per taxonomy §6). */
models: string[];
clients: Map<string, LlmClient>;
/**
* Sprint 11 B2 fold-in (2026-04-22) per decisions/2026-04-22-tie-break-policy-locked.md:
* when provided AND a 1-1-1 three-way split emerges from the primary
* ensemble (only meaningful when `models.length === 3`), judge-runner
* calls `resolveTieBreak` with this client as the fourth vendor.
* Absent → keep the legacy `computeMajority` behavior (tie-breaker =
* first model in `models` list).
*/
tieBreakerModel?: string;
tieBreakerClient?: LlmClient;
}
export type JudgeConfig = SingleJudgeConfig | EnsembleJudgeConfig;
/** Module shape mirror for resolveTieBreak — same dynamic-import pattern
* as loadJudgeModule to keep tsc happy under `rootDir: "src"`. */
interface TieBreakModule {
resolveTieBreak(
votes: Array<{ verdict: JudgeVerdict; failure_mode: FailureMode | null; rationale: string; judge_model: string }>,
options: {
callFourthVendor?: (payload: { primaryVotes: Array<{ verdict: JudgeVerdict; failure_mode: FailureMode | null; rationale: string; judge_model: string }>; model: string }) => Promise<{ verdict: JudgeVerdict; failure_mode: FailureMode | null; rationale: string; judge_model: string }>;
fourthVendorModel?: string;
logger?: { info(event: string, fields: Record<string, unknown>): void; warn?(event: string, fields: Record<string, unknown>): void };
},
): Promise<{
verdict: string;
path: 'none' | 'majority' | 'quadri-vendor' | 'pm-escalation';
votes: Array<{ verdict: JudgeVerdict; failure_mode: FailureMode | null; rationale: string; judge_model: string }>;
fourthVendorVote?: { verdict: JudgeVerdict; failure_mode: FailureMode | null; rationale: string; judge_model: string };
fourthVendorSlug?: string;
}>;
PM_ESCALATION_VERDICT: string;
DEFAULT_FOURTH_VENDOR: string;
}
let cachedTieBreakModule: TieBreakModule | null = null;
async function loadTieBreakModule(): Promise<TieBreakModule> {
if (cachedTieBreakModule) return cachedTieBreakModule;
const { fileURLToPath, pathToFileURL } = await import('node:url');
const nodePath = await import('node:path');
const here = fileURLToPath(import.meta.url);
const repoRoot = nodePath.resolve(nodePath.dirname(here), '..', '..', '..');
const candidates = [
nodePath.resolve(repoRoot, 'packages/server/src/benchmarks/judge/ensemble-tiebreak.ts'),
nodePath.resolve(repoRoot, 'packages/server/src/benchmarks/judge/ensemble-tiebreak.js'),
nodePath.resolve(repoRoot, 'packages/server/dist/benchmarks/judge/ensemble-tiebreak.js'),
];
const fs = await import('node:fs');
const target = candidates.find(p => fs.existsSync(p));
if (!target) {
throw new Error(
`ensemble-tiebreak module not found — looked in:\n ${candidates.join('\n ')}`,
);
}
cachedTieBreakModule = (await import(pathToFileURL(target).href)) as TieBreakModule;
return cachedTieBreakModule;
}
export async function runJudge(
triple: JudgeTriple,
config: JudgeConfig,
): Promise<JudgePayload> {
const base: JudgePayload = { model_answer: triple.modelAnswer };
const mod = await loadJudgeModule();
try {
if (config.kind === 'single') {
const result = await mod.judgeAnswer({
question: triple.question,
groundTruth: triple.groundTruth,
contextExcerpt: triple.contextExcerpt,
modelAnswer: triple.modelAnswer,
judgeModel: config.model,
llmClient: config.client,
});
return {
...base,
judge_verdict: result.verdict,
judge_failure_mode: result.failure_mode,
judge_rationale: result.rationale,
judge_model: result.judge_model,
judge_timestamp: new Date().toISOString(),
// A3 namespace split (LOCKED 2026-04-23 §2.1): mirror the legacy
// 5-value code into the 8-value column. Judge rubric upgrade will
// extend emission to F6 / F_other in a follow-on commit.
a3_failure_code: mapLegacyToA3(result.failure_mode),
a3_rationale: null,
};
}
const result = await mod.judgeEnsemble({
question: triple.question,
groundTruth: triple.groundTruth,
contextExcerpt: triple.contextExcerpt,
modelAnswer: triple.modelAnswer,
judgeModels: config.models,
llmClients: config.clients,
});
// Sprint 11 B2 fold-in (2026-04-22): 3-primary ensemble + tie-break
// client supplied + 1-1-1 three-way split observed → escalate via
// resolveTieBreak. Preserves judgeEnsemble's internal contract (it
// still returns its computeMajority-derived `majority` field);
// judge-runner post-processes to override the majority when the
// escalation triggers.
const isThreePrimary = config.models.length === 3;
const hasTieBreaker = Boolean(config.tieBreakerClient);
if (isThreePrimary && hasTieBreaker) {
const distinctVoteKeys = new Set(
result.ensemble.map(r => `${r.verdict}|${r.failure_mode ?? 'NA'}`),
);
if (distinctVoteKeys.size === 3) {
// 1-1-1 split confirmed. Dispatch to resolveTieBreak.
const tb = await loadTieBreakModule();
const fourthVendorModel = config.tieBreakerModel ?? tb.DEFAULT_FOURTH_VENDOR;
const tbResult = await tb.resolveTieBreak(result.ensemble, {
fourthVendorModel,
callFourthVendor: async ({ model: tbModel }) => {
const grokJudge = await mod.judgeAnswer({
question: triple.question,
groundTruth: triple.groundTruth,
contextExcerpt: triple.contextExcerpt,
modelAnswer: triple.modelAnswer,
judgeModel: tbModel,
llmClient: config.tieBreakerClient!,
});
return {
verdict: grokJudge.verdict,
failure_mode: grokJudge.failure_mode,
rationale: grokJudge.rationale,
judge_model: grokJudge.judge_model,
};
},
});
const ensembleVotes = tbResult.votes.map(r => ({
model: r.judge_model,
verdict: r.verdict,
failure_mode: r.failure_mode,
rationale: r.rationale,
}));
if (tbResult.path === 'pm-escalation') {
// 1-1-1-1 four-way — surface as judge_error so aggregator treats
// it as skipped (no silent coin-flip verdict). Preserves the
// Fleiss' κ=0.8784 methodology lock by NEVER fabricating a
// verdict when the ensemble + tie-break cannot reach plurality.
return {
...base,
judge_timestamp: new Date().toISOString(),
judge_ensemble: ensembleVotes,
judge_error: 'PM_ESCALATION',
tie_break_path: 'pm-escalation',
tie_break_fourth_vendor: tbResult.fourthVendorSlug,
// A3 namespace split: skipped instance carries no failure_code —
// `undefined` so the aggregator excludes it from failure_distribution
// counts (mirrors `judge_verdict === undefined` semantics).
a3_failure_code: undefined,
a3_rationale: null,
};
}
// path === 'quadri-vendor' — decode back to structured verdict.
const [verdictStr, failureModeRaw] = tbResult.verdict.split('|');
const resolvedVerdict = (verdictStr as JudgeVerdict);
const resolvedFailureMode: FailureMode | null =
failureModeRaw === 'NA' ? null : (failureModeRaw as FailureMode);
return {
...base,
judge_verdict: resolvedVerdict,
judge_failure_mode: resolvedFailureMode,
judge_rationale: `tie-break path=${tbResult.path} via ${tbResult.fourthVendorSlug ?? 'unknown'}`,
judge_model: 'ensemble_with_tiebreak',
judge_timestamp: new Date().toISOString(),
judge_ensemble: ensembleVotes,
tie_break_path: tbResult.path,
tie_break_fourth_vendor: tbResult.fourthVendorSlug,
// A3 namespace split: mirror resolved code into 8-value column.
a3_failure_code: mapLegacyToA3(resolvedFailureMode),
a3_rationale: null,
};
}
// Not a 1-1-1 split — fall through to legacy majority below.
}
return {
...base,
judge_verdict: result.majority.verdict,
judge_failure_mode: result.majority.failure_mode,
judge_rationale: result.majority.rationale,
judge_model: result.majority.judge_model,
judge_timestamp: new Date().toISOString(),
judge_ensemble: result.ensemble.map(r => ({
model: r.judge_model,
verdict: r.verdict,
failure_mode: r.failure_mode,
rationale: r.rationale,
})),
// A3 namespace split: majority legacy code mirrors into 8-value column.
a3_failure_code: mapLegacyToA3(result.majority.failure_mode),
a3_rationale: null,
};
} catch (err) {
// Two failure classes both land here:
// - JudgeParseError: judge LLM returned garbage twice in a row
// - Transport / HTTP / timeout after all JudgeLlmClient retries
// Either way, we keep the run going. A Stage-2 batch that loses one
// judge call out of 200 should not abort; the aggregator treats
// `judge_verdict === undefined` as a skipped-judge instance and
// downgrades confidence in the per-cell rollup accordingly.
const message = err instanceof Error ? err.message : String(err);
const kind = err instanceof mod.JudgeParseError ? 'parse' : 'transport';
console.warn(
`[judge-runner] ${kind} failure — instance left unjudged (${message.slice(0, 200)})`,
);
return { ...base, judge_error: `${kind}: ${message.slice(0, 200)}` };
}
}

View File

@@ -0,0 +1,32 @@
/**
* Harness-local mirror of the public surface of
* `packages/server/src/benchmarks/judge/failure-mode-judge.ts`.
*
* Why a mirror: the harness tsconfig has `rootDir: "src"` and the judge
* module lives in a sibling workspace, so direct `import from '../../../…'`
* trips TS6059 ("File is not under rootDir"). Declaring the narrow
* interface surface here keeps TypeScript honest at build time while
* runtime imports still resolve against the real server module via the
* `.js` extension convention (vite-node / tsx follow the symlink to the
* .ts source).
*
* Keep this file in sync with the canonical definitions — if a new field
* or method is added there, mirror it here. The mirror carries only the
* types the harness imports; it intentionally does not re-export the
* concrete classes (JudgeParseError) or functions (judgeAnswer,
* judgeEnsemble) — those come from the real module at runtime.
*/
export type FailureMode = 'F1' | 'F2' | 'F3' | 'F4' | 'F5';
export type Verdict = 'correct' | 'incorrect';
export interface JudgeResult {
verdict: Verdict;
failure_mode: null | FailureMode;
rationale: string;
judge_model: string;
}
export interface LlmClient {
complete(prompt: string): Promise<string>;
}

View File

@@ -0,0 +1,296 @@
/**
* LLM client — routes through LiteLLM when configured, stubs deterministically
* when dryRun is true.
*
* The client deliberately accepts no tools. Each cell owns its own
* prompt-assembly logic (memory injection vs. not, evolved prompt vs. not)
* and hands the assembled prompt to this client as a single user turn. That
* keeps the cell logic unit-testable and the LLM client a thin transport.
*/
import type { ModelSpec } from './types.js';
export interface LlmCallResult {
text: string;
inputTokens: number;
outputTokens: number;
latencyMs: number;
/** Dollar cost of this single call, computed from model pricing + tokens. */
costUsd: number;
/** null = OK, otherwise a short classification of the failure. */
failureMode: string | null;
/**
* Sprint 11 Task B1 (2026-04-22): captured chain-of-thought when the
* provider emits it under `thinking=on`. Parsed per H-AUDIT-1 ratification
* §Q3 precedence:
* 1. `body.choices[0].message.reasoning_content` (DashScope native, primary)
* 2. `body.choices[0].message.reasoning` (OpenRouter unified, current bridge)
* 3. `body.reasoning_content` (legacy top-level fallback)
* `undefined` when thinking is off or the provider omits the field.
* Per H-AUDIT-1 §2.4 exclusion rules, NEVER persisted to frames / memory /
* judge inputs — captured at the transport layer for JSONL + B1 smoke logs
* only.
*/
reasoningContent?: string;
/**
* Sprint 11 Task A2 (2026-04-22): which shape yielded the reasoning. Enum
* values per ratification §Q3 — `'unknown'` signals thinking=on was
* requested but no reasoning field was present; undefined when thinking
* was off (no expectation). Consumed by the runner to emit a
* `reasoning_content_shape_unknown` observability event when drift is
* detected.
*/
reasoningShape?: 'message.reasoning_content' | 'message.reasoning' | 'body.reasoning_content' | 'unknown';
}
export interface LlmCallInput {
model: ModelSpec;
systemPrompt: string;
userPrompt: string;
/** Abort the fetch after N ms. Default 30_000. */
timeoutMs?: number;
/**
* Sprint 11 Task B1 (2026-04-22): enable provider reasoning/thinking mode.
* Takes precedence over `model.stage2Config?.thinking`. Request body gets
* `reasoning: { enabled: true }` (OpenRouter unified shape) when true.
*/
thinking?: boolean;
/**
* Sprint 11 Task B1 (2026-04-22): override request `max_tokens`. Takes
* precedence over `model.stage2Config?.maxTokens`. Default (no override)
* keeps the pre-existing 600 value — back-compat for non-Stage-2 cells.
*/
maxTokensOverride?: number;
}
export interface LlmClient {
call(input: LlmCallInput): Promise<LlmCallResult>;
}
export function createLlmClient(opts: {
dryRun: boolean;
litellmUrl: string;
litellmApiKey: string;
}): LlmClient {
if (opts.dryRun) return new DryRunClient();
return new LiteLlmClient(opts.litellmUrl, opts.litellmApiKey);
}
// ── Dry-run (deterministic echo) ───────────────────────────────────────────
class DryRunClient implements LlmClient {
async call(input: LlmCallInput): Promise<LlmCallResult> {
// Return the expected span from the user prompt if present, else echo.
// The synthetic dataset embeds the answer in the context, so a smart
// "model" can extract it — our stub uses a trivial rule that's enough
// for the harness scaffold to verify end-to-end flow including accuracy
// scoring against the synthetic set.
const match = input.userPrompt.match(/Context:\s*([^\n]+)/);
const firstLine = match ? match[1].trim() : input.userPrompt.slice(0, 120);
const text = `DRY_RUN: ${firstLine}`;
const inputTokens = approximateTokenCount(input.systemPrompt) + approximateTokenCount(input.userPrompt);
const outputTokens = approximateTokenCount(text);
// Even in dry-run we record a "cost" so downstream aggregators
// exercise the cost path. Price comes from the model spec — in dry-run
// it's book-value, not wire-actual.
const costUsd =
(inputTokens / 1_000_000) * input.model.pricePerMillionInput +
(outputTokens / 1_000_000) * input.model.pricePerMillionOutput;
return {
text,
inputTokens,
outputTokens,
latencyMs: 1, // dry-run is instant
costUsd,
failureMode: null,
};
}
}
// ── LiteLLM proxy ──────────────────────────────────────────────────────────
/**
* Sprint 12 Task 2.5 Stage 1.5 §7.1 — fetch-retry on TypeError.
*
* The v2 full-context cell exhibited 100% `fetch_error_TypeError` at ~1 ms
* latency per instance (see sessions/2026-04-23-task25-s0-v2-fullcontext-
* forensic.md). Root cause: concurrent runner processes saturating the
* OpenRouter bridge / libuv thread pool. A single retry with a 1 s backoff
* absorbs transient saturation on normal ops (~1% of rows per PM estimate).
*
* Retry ONLY on `fetch_error_TypeError`. All other failure modes (`timeout`
* via AbortError, `http_5xx`, other error classes) return immediately — those
* aren't bridge-saturation patterns and retry can't help.
*
* Retry count is a module constant (default 1) and backoff is a module const
* (default 1000 ms). Tuning is intentional: more retries add per-row worst-
* case latency; more aggressive backoff adds wall-clock to the whole run.
*/
const FETCH_RETRY_MAX = 1;
const FETCH_RETRY_BACKOFF_MS = 1000;
class LiteLlmClient implements LlmClient {
constructor(private url: string, private apiKey: string) {}
async call(input: LlmCallInput): Promise<LlmCallResult> {
const overallStarted = Date.now();
let lastResult: LlmCallResult | undefined;
for (let attempt = 0; attempt <= FETCH_RETRY_MAX; attempt++) {
if (attempt > 0) {
await new Promise<void>(resolve => setTimeout(resolve, FETCH_RETRY_BACKOFF_MS));
}
const result = await this.attemptOnce(input);
lastResult = result;
if (result.failureMode !== 'fetch_error_TypeError') {
// Success or non-retryable failure — return with total wall-clock
// latency (including any backoff + prior attempts). Retrying a
// non-TypeError would both waste budget and invalidate the latency
// metric's meaning as "time to first clean signal."
if (attempt > 0) {
return { ...result, latencyMs: Date.now() - overallStarted };
}
return result;
}
}
// All retries exhausted. Return last result with total wall-clock.
return lastResult
? { ...lastResult, latencyMs: Date.now() - overallStarted }
: {
text: '',
inputTokens: 0,
outputTokens: 0,
latencyMs: Date.now() - overallStarted,
costUsd: 0,
failureMode: 'fetch_error_TypeError',
};
}
/** One attempt — the pre-Stage-1.5 `call` body unchanged. Returns an
* LlmCallResult (success or failure) rather than throwing so the outer
* retry loop can read `failureMode` to decide whether to retry. */
private async attemptOnce(input: LlmCallInput): Promise<LlmCallResult> {
const started = Date.now();
const controller = new AbortController();
// Sprint 11 B1: thinking=on on Stage 2 config pushes avg latency up to
// ~18s (Task 1.1 measured). Widen default timeout to 180s so a single
// reasoning-heavy call doesn't abort mid-response. Callers can still
// pass a tighter timeoutMs when needed.
const timer = setTimeout(() => controller.abort(), input.timeoutMs ?? 180_000);
// Resolve thinking + max_tokens: per-call input > model.stage2Config > defaults.
const stage2 = input.model.stage2Config;
const thinkingEnabled = input.thinking ?? stage2?.thinking ?? false;
const maxTokens = input.maxTokensOverride ?? stage2?.maxTokens ?? 600;
const requestBody: Record<string, unknown> = {
model: input.model.litellmModel,
messages: [
{ role: 'system', content: input.systemPrompt },
{ role: 'user', content: input.userPrompt },
],
max_tokens: maxTokens,
temperature: 0.0,
};
if (thinkingEnabled) {
// OpenRouter unified reasoning API — LiteLLM with drop_params=true will
// pass this through to OpenRouter unchanged. DashScope-intl native
// accepts a different shape (enable_thinking); LiteLLM normalizes
// either way when routed through its provider adapter. If the provider
// is one that doesn't support reasoning, drop_params strips silently.
requestBody.reasoning = { enabled: true };
}
try {
const res = await fetch(`${this.url.replace(/\/$/, '')}/v1/chat/completions`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${this.apiKey}`,
},
signal: controller.signal,
body: JSON.stringify(requestBody),
});
const latencyMs = Date.now() - started;
if (!res.ok) {
return {
text: '',
inputTokens: 0,
outputTokens: 0,
latencyMs,
costUsd: 0,
failureMode: `http_${res.status}`,
};
}
const body = await res.json() as {
choices?: Array<{
message?: {
content?: string;
reasoning?: string; // OpenRouter unified shape
reasoning_content?: string; // DashScope native (message-level)
};
}>;
reasoning_content?: string; // DashScope legacy top-level
usage?: { prompt_tokens?: number; completion_tokens?: number };
};
const msg = body.choices?.[0]?.message;
const text = msg?.content ?? '';
// Sprint 11 A2: parser precedence per H-AUDIT-1 ratification §Q3.
// Primary = DashScope native `message.reasoning_content`.
// Secondary = OpenRouter unified `message.reasoning`.
// Tertiary = legacy top-level `body.reasoning_content`.
// Unknown = thinking was requested but no field was present — emit
// observability signal so provider schema drift becomes visible.
let reasoningContent: string | undefined;
let reasoningShape: LlmCallResult['reasoningShape'];
if (msg?.reasoning_content !== undefined) {
reasoningContent = msg.reasoning_content;
reasoningShape = 'message.reasoning_content';
} else if (msg?.reasoning !== undefined) {
reasoningContent = msg.reasoning;
reasoningShape = 'message.reasoning';
} else if (body.reasoning_content !== undefined) {
reasoningContent = body.reasoning_content;
reasoningShape = 'body.reasoning_content';
} else if (thinkingEnabled) {
// thinking=on was requested but no reasoning surface present.
// `reasoningContent` stays undefined; `reasoningShape` = 'unknown'
// surfaces the drift for the runner to log.
reasoningShape = 'unknown';
}
const inputTokens = body.usage?.prompt_tokens ?? approximateTokenCount(input.systemPrompt + input.userPrompt);
const outputTokens = body.usage?.completion_tokens ?? approximateTokenCount(text);
const costUsd =
(inputTokens / 1_000_000) * input.model.pricePerMillionInput +
(outputTokens / 1_000_000) * input.model.pricePerMillionOutput;
return {
text,
inputTokens,
outputTokens,
latencyMs,
costUsd,
failureMode: null,
...(reasoningContent !== undefined && { reasoningContent }),
...(reasoningShape !== undefined && { reasoningShape }),
};
} catch (err) {
const latencyMs = Date.now() - started;
const name = (err as Error).name;
const failureMode = name === 'AbortError' ? 'timeout' : `fetch_error_${name}`;
return {
text: '',
inputTokens: 0,
outputTokens: 0,
latencyMs,
costUsd: 0,
failureMode,
};
} finally {
clearTimeout(timer);
}
}
}
// ── Helpers ────────────────────────────────────────────────────────────────
/** Rough token estimate (chars / 4). Used when the LLM response doesn't
* include usage info (e.g. in dry-run or certain proxy setups). */
export function approximateTokenCount(s: string): number {
return Math.max(1, Math.ceil(s.length / 4));
}

View File

@@ -0,0 +1,188 @@
/**
* Metrics + JSONL writer.
*
* Each instance run emits one JSONL record. The record shape is intentionally
* flat so downstream tools (jq, DuckDB, pandas) don't need unnesting. At the
* end of a run we also write an aggregate summary JSON alongside the JSONL.
*/
import fs from 'node:fs';
import path from 'node:path';
import type { AggregateSummary, JsonlRecord, RunConfig } from './types.js';
import {
computeFailureDistribution,
type FailureRow,
} from './failure-taxonomy/index.js';
/**
* Sprint 11 Task A2 — read-path pruning per H-AUDIT-1 ratification §Q4.
*
* Write-path always persists full records (incl. `reasoning_content`). The
* exclusion contract (design doc §2.4) is enforced on the READ side: any
* caller that might surface reasoning to a judge, UI, MCP payload, or
* summary brief must pass `{ includeReasoning: false }` so the field is
* stripped at the boundary.
*
* Default is `includeReasoning: false` — callers opt in explicitly when
* they need the raw trace (e.g. for archival gzip, audit replay).
*/
export interface ReadJsonlOptions {
/** When false (default), strips `reasoning_content` from each record.
* `reasoning_content_chars` and `reasoning_shape` are lightweight
* observability fields and are retained either way. */
includeReasoning?: boolean;
}
export function readJsonl(filePath: string, options: ReadJsonlOptions = {}): JsonlRecord[] {
const includeReasoning = options.includeReasoning ?? false;
if (!fs.existsSync(filePath)) return [];
const raw = fs.readFileSync(filePath, 'utf-8');
const records: JsonlRecord[] = [];
for (const line of raw.split('\n')) {
const trimmed = line.trim();
if (trimmed.length === 0) continue;
const record = JSON.parse(trimmed) as JsonlRecord;
if (!includeReasoning && record.reasoning_content !== undefined) {
// Strip the content; keep the chars + shape observability fields.
const { reasoning_content: _stripped, ...rest } = record;
records.push(rest as JsonlRecord);
} else {
records.push(record);
}
}
return records;
}
/** Scores a model output against expected substrings (any-match = full credit). */
export function scoreAccuracy(output: string, expected: string[]): number {
if (expected.length === 0) return 0;
const lower = output.toLowerCase();
for (const exp of expected) {
if (lower.includes(exp.toLowerCase())) return 1;
}
return 0;
}
/** p50 / p95 helpers. Returns 0 on empty input rather than NaN so JSONL
* consumers don't have to special-case an empty batch. */
export function percentile(values: number[], p: number): number {
if (values.length === 0) return 0;
const sorted = values.slice().sort((a, b) => a - b);
const idx = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length));
return sorted[idx];
}
export class JsonlWriter {
private stream: fs.WriteStream;
private records: JsonlRecord[] = [];
constructor(private outputPath: string) {
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
this.stream = fs.createWriteStream(outputPath, { flags: 'a' });
}
write(record: JsonlRecord): void {
this.stream.write(JSON.stringify(record) + '\n');
this.records.push(record);
}
all(): JsonlRecord[] {
return this.records.slice();
}
async close(): Promise<void> {
// Wait for BOTH finish (all writes flushed to kernel) AND close (file
// descriptor released). end(callback) only guarantees finish — on
// Windows the fd release lags, which races test cleanup rmSync.
await new Promise<void>((resolve, reject) => {
this.stream.once('close', () => resolve());
this.stream.once('error', reject);
this.stream.end();
});
}
}
export function buildAggregate(config: RunConfig, records: JsonlRecord[], startedAt: string, finishedAt: string, budgetStoppedAt: number | null): AggregateSummary {
const completed = records.filter(r => r.failure_mode === null);
const failed = records.filter(r => r.failure_mode !== null);
const latencies = records.map(r => r.p50_latency_ms).filter(v => v > 0);
const p95Latencies = records.map(r => r.p95_latency_ms).filter(v => v > 0);
const totalUsd = records.reduce((s, r) => s + r.usd_per_query, 0);
const meanAccuracy = records.length === 0 ? 0 : completed.reduce((s, r) => s + r.accuracy, 0) / records.length;
const failureModes: Record<string, number> = {};
for (const r of failed) {
const key = r.failure_mode ?? 'unknown';
failureModes[key] = (failureModes[key] ?? 0) + 1;
}
// Sprint 11 A2: reasoning_content aggregates when any record carries it.
// `undefined` when no records had reasoning — lets consumers distinguish
// "thinking was off" from "zero chars observed". Chars only, never content
// (design doc §2.4 exclusion rule).
const reasoningChars = records
.filter(r => r.reasoning_content_chars !== undefined && r.reasoning_content_chars > 0)
.map(r => r.reasoning_content_chars as number);
const shapeDistribution: Record<string, number> = {};
for (const r of records) {
if (r.reasoning_shape !== undefined) {
shapeDistribution[r.reasoning_shape] = (shapeDistribution[r.reasoning_shape] ?? 0) + 1;
}
}
const reasoningAggregate = reasoningChars.length === 0 && Object.keys(shapeDistribution).length === 0
? undefined
: {
count: reasoningChars.length,
sumChars: reasoningChars.reduce((s, n) => s + n, 0),
p50Chars: Math.round(percentile(reasoningChars, 50)),
p95Chars: Math.round(percentile(reasoningChars, 95)),
shapeDistribution,
};
// Sprint 12 Task 2 §2.1 A3 namespace split (LOCKED 2026-04-23): compute
// the A3 LOCK § 6 failure distribution from the `a3_failure_code` /
// `a3_rationale` columns. Only rows that carry the A3 column are
// included (pre-Sprint-12 rows and skipped-judge rows are excluded). The
// aggregate section stays `undefined` when no A3 rows exist so
// pre-A3 runs continue emitting the legacy shape verbatim.
const a3Rows: FailureRow[] = records
.filter(r => r.a3_failure_code !== undefined)
.map(r => ({
failure_code: r.a3_failure_code!,
rationale: r.a3_rationale ?? null,
}));
const failureDistribution =
a3Rows.length === 0 ? undefined : computeFailureDistribution(a3Rows);
return {
run: {
kind: config.run.kind,
name: config.run.name,
dataset: config.dataset.id,
model: config.model.id,
seed: config.seed,
startedAt,
finishedAt,
},
counts: {
total: records.length,
completed: completed.length,
failed: failed.length,
budgetStoppedAt,
},
metrics: {
meanAccuracy: round(meanAccuracy, 4),
p50LatencyMs: round(percentile(latencies, 50), 2),
p95LatencyMs: round(percentile(p95Latencies, 95), 2),
totalUsd: round(totalUsd, 6),
meanUsdPerQuery: records.length === 0 ? 0 : round(totalUsd / records.length, 6),
},
failureModes,
...(reasoningAggregate && { reasoningContent: reasoningAggregate }),
...(failureDistribution && { failure_distribution: failureDistribution }),
};
}
function round(n: number, decimals: number): number {
const f = Math.pow(10, decimals);
return Math.round(n * f) / f;
}

View File

@@ -0,0 +1,244 @@
/**
* Sprint 12 Task 1 Blocker #3 — Pre-registration event emitter.
*
* Emits `bench.preregistration.manifest_hash` once per benchmark run, carrying
* the SHA-256 of the frozen A3 LOCK manifest YAML, the canonical dataset
* version (from Session 1 Blocker #1), the CLI-resolved cell + judge-tiebreak
* choices, and the full judge-model roster with per-model pinning surface
* annotations (B3 LOCK addendum § 4).
*
* H-AUDIT-2 integration: the emitted event IS the audit anchor that ties a
* benchmark run to the v1 pre-registration doc. Any downstream replication
* claim verifies that the emitted `manifest_hash` matches the YAML committed
* to PM-Waggle-OS at the time of the run.
*
* Logger choice: uses `createCoreLogger` from `@waggle/core` (the established
* Waggle structured-log surface; thin console.* wrapper tagged with scope).
* Brief § 3.1 said "pino event signature" but no `pino` dep exists in the
* waggle-os workspace; the existing Waggle logger matches the structural
* contract (scope + message + payload). Surprise flagged in exit ping.
*
* YAML parsing: this module reads the manifest YAML file as bytes (SHA-256)
* and regex-extracts `locked_date:` to populate `manifest_locked_at`. No
* YAML parser dependency introduced per R4 verification (no `js-yaml` or
* `yaml` package available).
*/
import { execFileSync } from 'node:child_process';
import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { createCoreLogger } from '@waggle/core';
// ── Scope / constants ─────────────────────────────────────────────────────
/** Canonical manifest path (PM-Waggle-OS-relative) emitted on the event
* payload. This is a reference string, not a runtime lookup path — the
* actual filesystem path used to compute the hash is resolved separately
* via `resolveManifestPath()`. */
export const CANONICAL_MANIFEST_PATH = 'decisions/2026-04-22-bench-spec-locked.manifest.yaml';
/** Event name emitted on the pre-registration pino-style event. */
export const PREREGISTRATION_EVENT_NAME = 'bench.preregistration.manifest_hash';
/** Fallback runner version when git is unavailable. */
export const RUNNER_VERSION_FALLBACK = 'unknown';
/** Scoped logger. One per module load; cheap to re-use. */
const log = createCoreLogger('bench.preregistration');
// ── Types ─────────────────────────────────────────────────────────────────
export type PinningSurface = 'anthropic_immutable' | 'floating_alias' | 'revision_hash_pinned';
export interface JudgeModelManifestEntry {
model_id: string;
provider: string;
/** Kept in sync with `JudgeRole` in types.ts (2026-04-22 B2 LOCK remap
* added `'reserve'` for Grok tie-break). Duplicated inline here to
* avoid a types.ts → preregistration.ts → types.ts type cycle per
* existing `PinningSurface` dual-declaration precedent. */
judge_role: 'primary' | 'secondary' | 'tertiary' | 'reserve';
pinning_surface: PinningSurface;
pinning_surface_carve_out_reason: string | null;
}
export interface PreregistrationManifestPayload {
// Bench-spec lock
manifest_hash: string;
manifest_path: string;
manifest_locked_at: string;
// Canonical dataset
dataset_version: string;
dataset_path: string;
dataset_instance_count: number;
// CLI choices
per_cell: string[];
judge_tiebreak: string;
judge_models: JudgeModelManifestEntry[];
// Provenance
emitted_at: string;
runner_version: string;
runner_invocation: {
argv: string[];
cwd: string;
};
}
/** Thrown when the manifest YAML can't be located and no explicit hash
* was provided via CLI. */
export class ManifestNotFoundError extends Error {
constructor(public readonly attemptedPaths: string[]) {
super(
`Pre-registration manifest YAML not found. Tried:\n` +
attemptedPaths.map(p => ` - ${p}`).join('\n') +
`\nSet BENCH_SPEC_MANIFEST_PATH or pass --manifest-hash <sha> explicitly.`,
);
this.name = 'ManifestNotFoundError';
}
}
// ── Path resolution ───────────────────────────────────────────────────────
/**
* Default lookup order for the manifest YAML:
* 1. `BENCH_SPEC_MANIFEST_PATH` env var (absolute or cwd-relative)
* 2. Sibling-repo default `../PM-Waggle-OS/decisions/2026-04-22-bench-spec-locked.manifest.yaml`
* 3. Throw `ManifestNotFoundError`
*
* Exposed for tests — pass `override` to force a specific path (tests use
* tmp fixtures).
*/
export function resolveManifestPath(override?: string): string {
const tried: string[] = [];
if (override) {
const resolved = path.isAbsolute(override) ? override : path.resolve(process.cwd(), override);
tried.push(resolved);
if (fs.existsSync(resolved)) return resolved;
throw new ManifestNotFoundError(tried);
}
const envPath = process.env.BENCH_SPEC_MANIFEST_PATH;
if (envPath) {
const resolved = path.isAbsolute(envPath) ? envPath : path.resolve(process.cwd(), envPath);
tried.push(resolved);
if (fs.existsSync(resolved)) return resolved;
}
const siblingDefault = path.resolve(
process.cwd(),
'..',
'PM-Waggle-OS',
'decisions',
'2026-04-22-bench-spec-locked.manifest.yaml',
);
tried.push(siblingDefault);
if (fs.existsSync(siblingDefault)) return siblingDefault;
throw new ManifestNotFoundError(tried);
}
// ── Hash + metadata extraction ────────────────────────────────────────────
/**
* SHA-256 hex of the bench-spec manifest YAML bytes. Pure byte hash — no
* YAML parsing. The hash stability invariant mirrors the dataset archive
* invariant from Session 1 Blocker #1.
*/
export function computeBenchSpecManifestHash(manifestPath?: string): string {
const resolved = resolveManifestPath(manifestPath);
const buf = fs.readFileSync(resolved);
return crypto.createHash('sha256').update(buf).digest('hex');
}
/**
* Regex-extract the `locked_date:` field from the manifest YAML and
* normalize to ISO-8601. A3 LOCK YAML writes `locked_date: 2026-04-22`
* (date-only); we pad to `2026-04-22T00:00:00Z` for payload conformance.
*
* No YAML parser dep — regex is safe for this stable single-line field
* per A3 LOCK v1 format. If the format changes (YAML parser reshuffles
* keys, block scalar, etc.), the regex falls back to 'unknown' rather
* than throwing.
*/
export function readManifestLockedDate(manifestPath?: string): string {
const resolved = resolveManifestPath(manifestPath);
const content = fs.readFileSync(resolved, 'utf-8');
const match = content.match(/^locked_date:\s*(\S+)/m);
if (!match) return 'unknown';
const dateStr = match[1].trim();
// If already ISO-8601 (YYYY-MM-DDTHH:MM:SSZ), pass through.
if (/^\d{4}-\d{2}-\d{2}T/.test(dateStr)) return dateStr;
// YYYY-MM-DD → ISO-8601 midnight UTC.
if (/^\d{4}-\d{2}-\d{2}$/.test(dateStr)) return `${dateStr}T00:00:00Z`;
return dateStr;
}
// ── Runner version ────────────────────────────────────────────────────────
/**
* Git short SHA of the runner's working tree, or the fallback when git
* is unavailable. Uses `execFileSync` (no shell) — arguments are hardcoded,
* no injection surface.
*/
export function getRunnerVersion(): string {
try {
const out = execFileSync('git', ['rev-parse', '--short', 'HEAD'], {
cwd: process.cwd(),
stdio: ['ignore', 'pipe', 'ignore'],
timeout: 2000,
});
return out.toString().trim() || RUNNER_VERSION_FALLBACK;
} catch {
return RUNNER_VERSION_FALLBACK;
}
}
// ── Emitter ───────────────────────────────────────────────────────────────
/**
* Emit the pre-registration manifest hash event once per benchmark run.
* Expected call site: `runner.ts::runOne()`, before the first cell
* iteration. Zero-side-effects beyond the structured log line.
*
* No throw path — logging failure is swallowed (logger is a console wrapper,
* so the failure mode is effectively never hit). Payload construction is
* the caller's responsibility; this function trusts its input.
*/
export function emitPreregistrationManifest(payload: PreregistrationManifestPayload): void {
log.info(PREREGISTRATION_EVENT_NAME, { ...payload, event: PREREGISTRATION_EVENT_NAME });
}
// ── Argv sanitization (export-only for tests) ─────────────────────────────
/**
* Return a copy of `process.argv` with any argument that looks like an
* API key or bearer token replaced by '[REDACTED]'. The preregistration
* payload carries the full argv for provenance — keys and tokens cannot
* leak into the audit trail.
*
* Matches: values after `--api-key`, `--bearer`, `--token`, `--key`, or
* containing substrings `sk-*`, `Bearer *`.
*/
export function sanitizeArgv(argv: readonly string[]): string[] {
const REDACT_AFTER = new Set(['--api-key', '--bearer', '--token', '--key']);
const out: string[] = [];
for (let i = 0; i < argv.length; i++) {
const arg = argv[i];
if (REDACT_AFTER.has(arg) && i + 1 < argv.length) {
out.push(arg, '[REDACTED]');
i++;
continue;
}
if (/^sk-[A-Za-z0-9_-]+/.test(arg) || /^Bearer\s+/i.test(arg)) {
out.push('[REDACTED]');
continue;
}
out.push(arg);
}
return out;
}

View File

@@ -0,0 +1,152 @@
/**
* Sprint 12 Task 2.5 Stage 1.5 §7.4 — single-runner PID + heartbeat lock.
*
* Encodes `concurrent_runners: FORBIDDEN` at the process level. The v2 full-
* context 100%-fail incident had concurrent benchmark processes saturating
* the OpenRouter bridge (see §0.4 forensic). Manifest v4 will declare the
* policy in Field 7; this module is the enforcement.
*
* Cross-platform strategy: mtime-heartbeat is PRIMARY; `process.kill(pid, 0)`
* is secondary/audit-only because Windows reports success even for dead
* PIDs that were recently in use (per PM §6 note). Lock ownership is
* determined by whether the lock file's mtime is within the heartbeat-
* staleness window — the running process `utimesSync`s the lock every
* `heartbeatIntervalMs` (default 15 s), so a stale mtime (> 60 s) means
* no live owner regardless of platform-specific PID check quirks.
*
* Failure modes:
* - Lock exists, mtime fresh → refuse with clear error (owner alive)
* - Lock exists, mtime stale → take over (warn), previous owner is dead
* - Lock exists, content corrupt → treat as stale, take over
* - Lock dir doesn't exist → mkdir -p then write
* - Fs write fails → propagate error up
*
* On clean exit / SIGINT / SIGTERM the lock is deleted. On hard crash
* (kill -9, OOM) the lock persists until its heartbeat expires + the next
* runner takes over.
*/
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
/** How long without a heartbeat update before a lock is considered stale. */
const DEFAULT_STALE_MS = 60_000;
/** How often the holder refreshes the lock file's mtime. */
const DEFAULT_HEARTBEAT_INTERVAL_MS = 15_000;
export interface LockPayload {
pid: number;
hostname: string;
startedAt: string;
}
export interface LockHandle {
readonly lockPath: string;
/** Release the lock, stop the heartbeat, and remove signal handlers.
* Safe to call multiple times. */
release(): void;
}
export interface AcquireOptions {
/** Override staleness window (ms). Default 60_000. */
staleMs?: number;
/** Override heartbeat refresh interval (ms). Default 15_000. */
heartbeatIntervalMs?: number;
/** Injectable clock for deterministic tests. */
nowFn?: () => number;
/** Skip signal-handler registration (tests that don't want to hook the
* test runner's SIGINT). Default false — register handlers in production. */
skipSignalHandlers?: boolean;
}
/** Read the existing lock file if it exists; returns null on any error. */
function readExistingLock(lockPath: string): { payload: LockPayload | null; ageMs: number | null } {
if (!fs.existsSync(lockPath)) return { payload: null, ageMs: null };
let payload: LockPayload | null = null;
try {
payload = JSON.parse(fs.readFileSync(lockPath, 'utf-8')) as LockPayload;
} catch {
// Corrupt content — treat as payload null; caller still computes ageMs
// from stat mtime below.
}
const stat = fs.statSync(lockPath);
return { payload, ageMs: Date.now() - stat.mtimeMs };
}
export function acquireRunnerLock(outputPath: string, opts: AcquireOptions = {}): LockHandle {
const lockPath = `${outputPath}.lock`;
const staleMs = opts.staleMs ?? DEFAULT_STALE_MS;
const heartbeatMs = opts.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
const nowFn = opts.nowFn ?? Date.now;
// 1. Check for existing lock.
const existing = readExistingLock(lockPath);
if (existing.ageMs !== null && existing.ageMs < staleMs) {
// Fresh heartbeat → active owner. Refuse.
const pidStr = existing.payload?.pid ?? '?';
const hostStr = existing.payload?.hostname ?? '?';
throw new Error(
`[bench:lock] active runner lock at ${lockPath} ` +
`(pid=${pidStr}, host=${hostStr}, age_ms=${Math.round(existing.ageMs)}). ` +
`Another waggle-bench process is running against this output path. ` +
`Wait for it to finish, or if you're certain it's dead, delete the lock file manually.`,
);
}
if (existing.ageMs !== null) {
// Stale lock — note and take over.
console.warn(
`[bench:lock] taking stale lock at ${lockPath} ` +
`(pid=${existing.payload?.pid ?? '?'}, age_ms=${Math.round(existing.ageMs)}, ` +
`staleMs=${staleMs}). Previous owner is dead.`,
);
}
// 2. Write our lock.
const payload: LockPayload = {
pid: process.pid,
hostname: os.hostname(),
startedAt: new Date(nowFn()).toISOString(),
};
fs.mkdirSync(path.dirname(lockPath), { recursive: true });
fs.writeFileSync(lockPath, JSON.stringify(payload, null, 2), 'utf-8');
// 3. Start heartbeat. .unref() so a forgotten release() doesn't block
// node's event loop from exiting normally.
const heartbeat = setInterval(() => {
try {
const now = new Date();
fs.utimesSync(lockPath, now, now);
} catch {
// Disk full, lock deleted, etc. Continue best-effort — the lock may
// be stolen by another runner, but we can't do much about it.
}
}, heartbeatMs);
heartbeat.unref();
// 4. Signal handlers for clean release on Ctrl+C / kill <PID>.
const signalHandler = (): void => {
clearInterval(heartbeat);
try { fs.unlinkSync(lockPath); } catch { /* already gone */ }
process.exit(130);
};
if (!opts.skipSignalHandlers) {
process.once('SIGINT', signalHandler);
process.once('SIGTERM', signalHandler);
}
let released = false;
return {
lockPath,
release(): void {
if (released) return;
released = true;
clearInterval(heartbeat);
if (!opts.skipSignalHandlers) {
process.removeListener('SIGINT', signalHandler);
process.removeListener('SIGTERM', signalHandler);
}
try { fs.unlinkSync(lockPath); } catch { /* already gone */ }
},
};
}

View File

@@ -0,0 +1,979 @@
#!/usr/bin/env tsx
/**
* Four-cell ablation harness — CLI entry.
*
* Usage (via `npm run bench -- ...` from repo root, or direct `tsx`):
*
* --cell raw Single cell
* --cell filtered
* --cell compressed
* --cell full-context
* --all-cells Run all four sequentially (same dataset/seed)
* --control verbose-fixed Diagnostic control (not a cell)
*
* --dataset locomo locomo | longmemeval | synthetic
* --limit N N instances. --full sets Infinity.
* --full Alias for --limit Infinity.
* --model qwen3.6-35b-a3b Must match an id in config/models.json
* --seed N Default: 42 (reproducibility artifact req).
* --budget USD Hard USD cap. Default: Infinity.
* --output path.jsonl Default: ../results/<cell>-<dataset>-<ts>.jsonl
* --dry-run Stub LLM calls. Default: true when
* LITELLM_URL is not set.
*
* Env:
* LITELLM_URL default http://localhost:4000
* LITELLM_API_KEY default sk-waggle-dev
*/
import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import url from 'node:url';
import { generateTurnId, logTurnEvent } from '@waggle/agent';
import type {
CellName, ControlName, DatasetSpec, JsonlRecord, ModelSpec, RunConfig, RunKind,
} from './types.js';
import { getDatasetVersion, loadDataset, loadPreflightSampleLock, sampleInstances } from './datasets.js';
import { createLlmClient } from './llm.js';
import {
CANONICAL_MANIFEST_PATH,
computeBenchSpecManifestHash,
emitPreregistrationManifest,
getRunnerVersion,
readManifestLockedDate,
sanitizeArgv,
type JudgeModelManifestEntry,
type PreregistrationManifestPayload,
} from './preregistration.js';
import { JsonlWriter, buildAggregate, scoreAccuracy, percentile } from './metrics.js';
import { cells, isCellName, isControlName } from './cells.js';
import { controls } from './controls.js';
import { createJudgeLlmClient, type JudgeClientCostEntry } from './judge-client.js';
import { runJudge, type JudgeConfig } from './judge-runner.js';
// Sprint 12 Task 2.5 Stage 1 (2026-04-23) — substrate hookup for retrieval +
// agentic cells. Substrate is created once in main() when the cell roster
// includes `retrieval` or `agentic`, pre-ingested from raw LoCoMo, then
// threaded through RunConfig → CellInput to each iteration.
import { createSubstrate, type Substrate } from './substrate.js';
import { extractTurnsFromLocomoRaw, ingestLoCoMoCorpus } from './ingest.js';
// Sprint 12 Task 2.5 Stage 1.5 §7.2 — consecutive fetch-transport failure
// halt. Prevents the v2 full-context saturation cascade from burning the
// full instance budget on dead calls.
import { StreakTracker } from './streak-tracker.js';
// Sprint 12 Task 2.5 Stage 1.5 §7.3 — pre-cell health check. Probes every
// upstream route the run depends on before spending instance-budget on them.
import { preCellHealthCheck } from './health-check.js';
// Sprint 12 Task 2.5 Stage 1.5 §7.4 — single-runner PID+heartbeat lock.
// Encodes `concurrent_runners: FORBIDDEN` at the process level. Uses a
// single global sentinel under benchmarks/results/ so two harness processes
// can never both run even against different datasets/models (they still
// share the LiteLLM proxy + provider bridges — the v2 saturation root cause).
import { acquireRunnerLock, type LockHandle } from './runner-lock.js';
// ── Arg parsing ────────────────────────────────────────────────────────────
interface ParsedArgs {
cell?: string;
allCells: boolean;
control?: string;
dataset: string;
limit: number;
model: string;
seed: number;
budget: number;
output?: string;
dryRun?: boolean;
sampleLock?: string;
/** When set, each cell/control call also triggers a judge call after
* the model answer returns. Value is the single-judge model id
* (e.g. `claude-sonnet-4-6`). Default: undefined → judging skipped. */
judge?: string;
/** Comma-separated list of judge model ids for ensemble mode. First
* id is the tie-breaker (Sonnet by taxonomy §6 convention). Takes
* precedence over `--judge` when both are set. */
judgeEnsemble?: string[];
// ── Sprint 12 Task 1 Blocker #3 — pre-registration flags ─────────────────
/** SHA-256 hex of the bench-spec manifest YAML. When omitted, the
* emitter resolves the manifest path (env → sibling PM-Waggle-OS → throw)
* and computes the hash at run start. When provided, skips the lookup
* entirely — useful for tests and replication runs where the source
* YAML is not available at the runtime cwd. */
manifestHash?: string;
/** Suppresses `bench.preregistration.manifest_hash` pino-style event
* when `false`. Default: true. Tests pass `--no-emit-preregistration-event`
* to keep the log surface quiet. */
emitPreregistrationEvent: boolean;
/** Multi-value cell selection (repeatable `--per-cell raw --per-cell filtered`).
* Overrides `--cell` and `--all-cells` when at least one value is set.
* Also materializes into the `per_cell` field of the pre-registration
* manifest payload regardless of how cells were chosen (falls back to
* derivation from --all-cells or --cell when unset). */
perCell?: string[];
/** Judge tie-break strategy name surfaced into the pre-registration
* payload. Canonical A3 LOCK § B2 values: `quadri-vendor` (fourth
* vendor fires on 1-1-1) and `pm-escalation` (2-2 escalates to PM).
* `majority` is accepted as an explicit-default alias. */
judgeTiebreak?: string;
// ── Sprint 12 Task 2.5 Stage 1 (2026-04-23) — substrate flags ────────────
/** Path to the raw LoCoMo archive (`locomo10.json`). Required when the cell
* roster includes `retrieval` or `agentic`. Relative paths resolve against
* the current working directory. */
locomoRawPath?: string;
/** Retrieval top-K (default 10, GATE-S0 lock). Threaded into CellInput. */
retrievalTopK?: number;
/** Agentic hard turn cap (default 3, GATE-S0 lock). */
agenticMaxTurns?: number;
/** Agentic AbortController timeout in ms (default 180_000). */
agenticTimeoutMs?: number;
}
function parseArgs(argv: string[]): ParsedArgs {
const out: ParsedArgs = {
allCells: false,
dataset: 'synthetic',
limit: 10,
model: 'qwen3.6-35b-a3b',
seed: 42,
budget: Number.POSITIVE_INFINITY,
emitPreregistrationEvent: true,
};
const VALID_TIEBREAK = new Set(['quadri-vendor', 'pm-escalation', 'majority']);
for (let i = 0; i < argv.length; i++) {
const flag = argv[i];
const next = argv[i + 1];
switch (flag) {
case '--cell': out.cell = next; i++; break;
case '--all-cells': out.allCells = true; break;
case '--control': out.control = next; i++; break;
case '--dataset': out.dataset = next; i++; break;
case '--limit': out.limit = Number(next); i++; break;
case '--full': out.limit = Number.POSITIVE_INFINITY; break;
case '--model': out.model = next; i++; break;
case '--seed': out.seed = Number(next); i++; break;
case '--budget': out.budget = Number(next); i++; break;
case '--output': out.output = next; i++; break;
case '--dry-run': out.dryRun = true; break;
case '--live': out.dryRun = false; break;
case '--sample-lock': out.sampleLock = next; i++; break;
case '--judge': out.judge = next; i++; break;
case '--judge-ensemble':
out.judgeEnsemble = (next ?? '').split(',').map(s => s.trim()).filter(Boolean);
i++;
break;
// ── Sprint 12 Task 1 Blocker #3 flags ───────────────────────────
case '--manifest-hash':
if (typeof next !== 'string' || !/^[0-9a-f]{64}$/i.test(next)) {
throw new Error(
`Invalid --manifest-hash value: expected 64-char lowercase SHA-256 hex, got ${next ?? '(missing)'}`,
);
}
out.manifestHash = next.toLowerCase();
i++;
break;
case '--emit-preregistration-event':
out.emitPreregistrationEvent = true;
break;
case '--no-emit-preregistration-event':
out.emitPreregistrationEvent = false;
break;
case '--per-cell':
if (typeof next !== 'string' || next.length === 0) {
throw new Error(`Invalid --per-cell value: expected cell name, got ${next ?? '(missing)'}`);
}
out.perCell = out.perCell ?? [];
out.perCell.push(next);
i++;
break;
case '--judge-tiebreak':
if (typeof next !== 'string' || !VALID_TIEBREAK.has(next)) {
throw new Error(
`Invalid --judge-tiebreak value: expected one of ${[...VALID_TIEBREAK].join(' | ')}, got ${next ?? '(missing)'}`,
);
}
out.judgeTiebreak = next;
i++;
break;
// ── Sprint 12 Task 2.5 Stage 1 substrate flags ───────────────────────
case '--locomo-raw-path':
if (typeof next !== 'string' || next.length === 0) {
throw new Error(`Invalid --locomo-raw-path value: expected path, got ${next ?? '(missing)'}`);
}
out.locomoRawPath = next;
i++;
break;
case '--retrieval-top-k': {
const n = Number(next);
if (!Number.isInteger(n) || n <= 0 || n > 100) {
throw new Error(`Invalid --retrieval-top-k value: expected 1..100 integer, got ${next ?? '(missing)'}`);
}
out.retrievalTopK = n;
i++;
break;
}
case '--agentic-max-turns': {
const n = Number(next);
if (!Number.isInteger(n) || n <= 0 || n > 10) {
throw new Error(`Invalid --agentic-max-turns value: expected 1..10 integer, got ${next ?? '(missing)'}`);
}
out.agenticMaxTurns = n;
i++;
break;
}
case '--agentic-timeout-ms': {
const n = Number(next);
if (!Number.isFinite(n) || n < 1000) {
throw new Error(`Invalid --agentic-timeout-ms value: expected ≥1000 ms, got ${next ?? '(missing)'}`);
}
out.agenticTimeoutMs = n;
i++;
break;
}
case '--help':
case '-h':
printHelp();
process.exit(0);
}
}
return out;
}
function printHelp(): void {
console.log(`Four-cell ablation harness.
Usage:
waggle-bench --cell <raw|filtered|compressed|full-context> --dataset <id> --limit N --model <id>
waggle-bench --all-cells --dataset <id> --limit N --model <id> [--budget USD]
waggle-bench --control verbose-fixed --dataset <id> --limit N --model <id>
Flags:
--cell Single cell name
--all-cells Run all four cells sequentially
--control Diagnostic control (e.g. verbose-fixed)
--dataset locomo | longmemeval | synthetic (default synthetic)
--limit N Instance count cap (default 10)
--full --limit Infinity
--model id Model id from config/models.json (default qwen3.6-35b-a3b)
--seed N PRNG seed (default 42)
--budget USD Hard USD cap (default Infinity)
--output path JSONL output path (default auto)
--dry-run Stub LLM calls (default if LITELLM_URL unset)
--live Force real LLM calls even if LITELLM_URL unset
--sample-lock P Path to a committed sample-lock JSON. Bypasses the dataset
adapter and loads instances directly from the lock. Runtime
asserts category distribution matches 13/13/12/12 for the
Stage 2 preflight gate; fails fast otherwise.
--judge MODEL Enable single-judge mode. After each cell call, invokes
failure-mode-judge with the given model (e.g.
claude-sonnet-4-6) and populates the judge fields on the
JSONL record. Skip the flag to run without judging.
--judge-ensemble M1,M2,...
Enable 3+-judge ensemble mode. First model is the tie-
breaker. Takes precedence over --judge. Real API spend —
keep within the brief's $5 alarm per run.
Pre-registration flags (Sprint 12 Task 1 Blocker #3):
--manifest-hash <sha> Override bench-spec manifest SHA-256 (64-char hex).
When omitted, resolves BENCH_SPEC_MANIFEST_PATH env or
falls back to sibling ../PM-Waggle-OS/decisions/ path.
--emit-preregistration-event
Explicitly enable the bench.preregistration.manifest_hash
event (default: enabled).
--no-emit-preregistration-event
Suppress the event (useful for smoke tests).
--per-cell NAME Multi-value cell selection. Repeat for multiple cells
(--per-cell raw --per-cell filtered). Overrides --cell
and --all-cells when at least one value is supplied.
--judge-tiebreak STRATEGY
Tie-break strategy surfaced into the pre-registration
payload. One of: quadri-vendor | pm-escalation | majority.
Default: quadri-vendor (per A3 LOCK § B2).
Task 2.5 Stage 1 substrate flags (2026-04-23):
--locomo-raw-path P
Path to raw LoCoMo archive (snap-research/locomo10.json).
REQUIRED when --cell is 'retrieval' or 'agentic'. Main()
builds an ephemeral :memory: MindDB + HybridSearch pair,
ingests every turn as one I-frame (frame-per-turn per
GATE-S0 lock), then tears it down at end of run.
--retrieval-top-k N
Top-K turn frames the retrieval cell recalls per question.
Default 10 (GATE-S0 lock). Range 1..100.
--agentic-max-turns N
Hard turn cap for the agentic cell's inner agent-loop.
Default 3 (GATE-S0 lock). Range 1..10.
--agentic-timeout-ms N
AbortController timeout for the agentic cell. Default
180_000 ms. Minimum 1000 ms.
--help, -h This text
`);
}
// ── Config loaders ─────────────────────────────────────────────────────────
function harnessRoot(): string {
// __filename equivalent for ESM. When compiled to dist/runner.js this
// resolves to the dist dir; when run via tsx it resolves to src/runner.ts.
const here = url.fileURLToPath(import.meta.url);
// `harness/dist/runner.js` → harness dir; `harness/src/runner.ts` → harness dir.
return path.resolve(path.dirname(here), '..');
}
function loadModels(): Record<string, ModelSpec> {
const cfg = path.join(harnessRoot(), 'config', 'models.json');
return JSON.parse(fs.readFileSync(cfg, 'utf-8')) as Record<string, ModelSpec>;
}
function loadDatasets(): Record<string, DatasetSpec> {
const cfg = path.join(harnessRoot(), 'config', 'datasets.json');
return JSON.parse(fs.readFileSync(cfg, 'utf-8')) as Record<string, DatasetSpec>;
}
// ── Run driver ─────────────────────────────────────────────────────────────
async function runOne(config: RunConfig): Promise<void> {
const startedAt = new Date().toISOString();
const dataRoot = path.join(harnessRoot(), '..', 'data');
// Sample-lock path, when provided, takes precedence over the dataset adapter
// and enforces the Stage 2 13/13/12/12 distribution inside
// loadPreflightSampleLock. Failure to match → throw before any LLM call,
// which is the Task-1 acceptance requirement.
const all = config.sampleLockPath
? loadPreflightSampleLock(
path.isAbsolute(config.sampleLockPath)
? config.sampleLockPath
: path.resolve(process.cwd(), config.sampleLockPath),
)
: loadDataset(config.dataset, dataRoot);
// Sprint 12 Task 1 Blocker #1: attach dataset_version (SHA-256 of canonical
// archive) to every emitted JSONL record. Sample-lock runs get the
// lock-file hash; regular runs get the dataset archive hash; synthetic
// runs get the static `synthetic-scaffold-v1` string. Computed once per
// runOne call to avoid per-instance disk I/O.
const datasetVersion = config.sampleLockPath
? computeFileHash(
path.isAbsolute(config.sampleLockPath)
? config.sampleLockPath
: path.resolve(process.cwd(), config.sampleLockPath),
)
: getDatasetVersion(config.dataset, dataRoot);
// Sprint 12 Task 1 Blocker #3: emit `bench.preregistration.manifest_hash`
// once per runOne before the first instance iteration. Suppressed when
// the caller sets `emitPreregistrationEvent: false` (test default).
if (config.emitPreregistrationEvent !== false) {
emitPreregistrationManifest(
assemblePreregistrationPayload(config, datasetVersion, all.length),
);
}
// When a sample lock drives the run, honor instance order verbatim — the
// lock file IS the deterministic sample. Cell comparisons require identical
// order across cells, and `sampleInstances` would re-shuffle the lock.
const sampled = config.sampleLockPath
? (Number.isFinite(config.limit) && config.limit < all.length ? all.slice(0, config.limit) : all.slice())
: sampleInstances(all, config.seed, config.limit);
const writer = new JsonlWriter(config.outputPath);
const llm = createLlmClient({
dryRun: config.dryRun,
litellmUrl: config.litellmUrl,
litellmApiKey: config.litellmApiKey,
});
let totalCost = 0;
let budgetStoppedAt: number | null = null;
const latenciesByInstance: number[] = [];
// §7.2 Stage 1.5 streak halt — track consecutive fetch-transport failures.
// Tracker is cell-local: each cell gets a fresh counter so a streak in the
// previous cell doesn't poison the next.
const streakTracker = new StreakTracker();
let streakHaltAt: number | null = null;
let streakHaltSummary: string | null = null;
for (let i = 0; i < sampled.length; i++) {
if (totalCost >= config.budgetUsd) {
budgetStoppedAt = i;
break;
}
const instance = sampled[i];
const turnId = generateTurnId();
// Task 2.5 Stage 1: cells and controls have diverged CellInput/ControlInput
// shapes — cells now accept optional substrate + litellm + per-cell knobs
// that controls don't need. Split the dispatch so each side gets only the
// fields it understands.
let result;
if (config.run.kind === 'cell') {
const cellFn = cells[config.run.name as CellName];
result = await cellFn({
instance,
model: config.model,
llm,
turnId,
substrate: config.substrate,
litellm: config.litellm,
retrievalTopK: config.retrievalTopK,
agenticMaxTurns: config.agenticMaxTurns,
agenticTimeoutMs: config.agenticTimeoutMs,
});
} else {
const controlFn = controls[config.run.name as ControlName];
result = await controlFn({ instance, model: config.model, llm, turnId });
}
latenciesByInstance.push(result.latencyMs);
const accuracy = result.failureMode ? 0 : scoreAccuracy(result.text, instance.expected);
totalCost += result.costUsd;
// Sprint 11 A2: emit structured llm.response event tagged with turnId.
// `reasoningShape` + char count become the canonical observability
// surface; reasoning content itself goes to JSONL only, never to logs,
// per design doc §2.3/§2.4 exclusion rules.
logTurnEvent(turnId, {
stage: 'llm.response',
cell: config.run.name,
model: config.model.id,
textChars: result.text.length,
latencyMs: result.latencyMs,
costUsd: result.costUsd,
failureMode: result.failureMode,
reasoningShape: result.reasoningShape ?? 'none',
reasoningChars: result.reasoningContent?.length ?? 0,
});
// Drift alarm per ratification §Q3: thinking=on was requested but no
// reasoning field present — observable without failing the run.
if (result.reasoningShape === 'unknown') {
logTurnEvent(turnId, {
stage: 'llm.response.reasoning_content_shape_unknown',
cell: config.run.name,
model: config.model.id,
litellmModel: config.model.litellmModel,
});
}
// Sprint 9 Task 2: when a judge is configured, grade the answer
// in-line. The judge runs even when the cell call itself failed
// (failureMode !== null) because the transcript still has value
// for calibration; the aggregator can filter if needed. A judge
// error never aborts the batch — runJudge swallows and annotates.
let judgePayload: import('./judge-runner.js').JudgePayload | null = null;
if (config.judgeConfig && !result.failureMode) {
judgePayload = await runJudge(
{
question: instance.question,
// scoreAccuracy picks the first expected answer; keep parity
// so the judge sees the same ground truth. Remaining entries
// in `expected` are alternate phrasings, not independent
// facts, and are irrelevant to the §4 judge prompt.
groundTruth: instance.expected[0] ?? '',
contextExcerpt: instance.context,
modelAnswer: result.text,
},
config.judgeConfig,
);
}
// p50 / p95 are computed over the running window of observed latencies
// so the JSONL record carries real-time percentiles (not flat per-row).
// Aggregate metrics.ts recomputes globals for the summary.
const record: JsonlRecord = {
turnId,
cell: config.run.name,
instance_id: instance.instance_id,
model: config.model.id,
seed: config.seed,
accuracy,
p50_latency_ms: percentile(latenciesByInstance, 50),
p95_latency_ms: percentile(latenciesByInstance, 95),
usd_per_query: round(result.costUsd, 6),
failure_mode: result.failureMode,
dataset_version: datasetVersion,
...(judgePayload && {
model_answer: judgePayload.model_answer,
judge_verdict: judgePayload.judge_verdict,
judge_failure_mode: judgePayload.judge_failure_mode,
judge_rationale: judgePayload.judge_rationale,
judge_model: judgePayload.judge_model,
judge_timestamp: judgePayload.judge_timestamp,
judge_ensemble: judgePayload.judge_ensemble,
// B2 fold-in observability fields.
...(judgePayload.tie_break_path !== undefined && { tie_break_path: judgePayload.tie_break_path }),
...(judgePayload.tie_break_fourth_vendor !== undefined && { tie_break_fourth_vendor: judgePayload.tie_break_fourth_vendor }),
// Sprint 12 Task 2 §2.1 A3 namespace split (LOCKED 2026-04-23) —
// authoritative A3 LOCK § 6 taxonomy columns. `undefined` a3_failure_code
// (e.g. PM_ESCALATION skipped instance) is spread conditionally so
// the absence of the key is preserved, matching the aggregator's
// "skipped" semantics.
...(judgePayload.a3_failure_code !== undefined && { a3_failure_code: judgePayload.a3_failure_code }),
...(judgePayload.a3_rationale !== undefined && { a3_rationale: judgePayload.a3_rationale }),
}),
// Sprint 11 A2: reasoning_content fields — same-row persistence
// keyed by `turnId` per ratification §Q4. Chars stay separate for
// aggregation even when content is stripped on the read path.
...(result.reasoningContent !== undefined && {
reasoning_content: result.reasoningContent,
reasoning_content_chars: result.reasoningContent.length,
}),
...(result.reasoningShape !== undefined && {
reasoning_shape: result.reasoningShape,
}),
// Sprint 12 Task 1 Blocker #3 / B3 addendum § 4: copy target-model
// pinning surface fields from `config/models.json` onto every row so
// any single JSONL line carries its own audit-replication anchor.
...(config.model.pinning_surface !== undefined && {
model_pinning_surface: config.model.pinning_surface,
model_pinning_carve_out_reason: config.model.pinning_surface_carve_out_reason ?? null,
}),
// B3 addendum § 4 leaves `model_revision_hash` null unless the provider
// exposes one. Future providers (e.g. OpenRouter with revision_id)
// will populate this via a provider-specific hook in Session 3+.
model_revision_hash: null,
};
writer.write(record);
// §7.2 Stage 1.5 streak halt — check AFTER writing the current record so
// the failure-forensic row persists. Break out of the loop; the error is
// thrown AFTER writer.close() below so JSONL + summary files flush
// cleanly. Caller (main) surfaces the error to the operator.
if (streakTracker.record(result.failureMode)) {
streakHaltAt = i + 1;
streakHaltSummary = streakTracker.summary();
break;
}
}
await writer.close();
const finishedAt = new Date().toISOString();
const summary = buildAggregate(config, writer.all(), startedAt, finishedAt, budgetStoppedAt);
const summaryPath = config.outputPath.replace(/\.jsonl$/, '.summary.json');
fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2), 'utf-8');
// Compact stdout summary — machine-parseable prefix makes CI grep easy.
console.log(
`[bench:summary] kind=${summary.run.kind} name=${summary.run.name} ` +
`model=${summary.run.model} seed=${summary.run.seed} ` +
`n=${summary.counts.total} completed=${summary.counts.completed} ` +
`failed=${summary.counts.failed} ` +
`accuracy=${summary.metrics.meanAccuracy} ` +
`p50=${summary.metrics.p50LatencyMs}ms p95=${summary.metrics.p95LatencyMs}ms ` +
`cost=$${summary.metrics.totalUsd} ` +
`jsonl=${config.outputPath}`,
);
// §7.2 Stage 1.5 — throw AFTER writer + summary flushed so partial data
// persists for forensic review (same pattern as budget-stop, except budget
// is a soft stop that doesn't throw). The surfaced error lets main()
// decide whether to continue other cells or bail the whole invocation.
if (streakHaltAt !== null) {
throw new Error(
`[bench:halt] cell '${config.run.name}' aborted at instance ${streakHaltAt}/${sampled.length} ` +
`due to consecutive fetch-transport failures (${streakHaltSummary}). ` +
`Likely upstream saturation — check LiteLLM proxy + provider health before re-running. ` +
`Partial JSONL persisted at ${config.outputPath}.`,
);
}
}
function round(n: number, decimals: number): number {
const f = Math.pow(10, decimals);
return Math.round(n * f) / f;
}
/** SHA-256 hex of a file's bytes. Used for sample-lock version stamping
* when the run is driven by a committed lock file rather than the regular
* dataset archive. */
function computeFileHash(absolutePath: string): string {
const buf = fs.readFileSync(absolutePath);
return crypto.createHash('sha256').update(buf).digest('hex');
}
/**
* Resolve the CLI-requested judge-model roster against `config/models.json`
* so every pre-registration emit carries B3-addendum pinning fields. Keeps
* the registry lookup in main() — runOne only forwards the materialised
* list.
*
* Unknown model ids are passed through with `floating_alias` defaults and
* an explicit carve-out reason that names the ID. This keeps the run going
* while loudly surfacing the gap to downstream audit consumers.
*/
function resolveJudgeModelsForPreregistration(
models: Record<string, ModelSpec>,
args: ParsedArgs,
): JudgeModelManifestEntry[] {
const ids: string[] = [];
if (args.judgeEnsemble && args.judgeEnsemble.length > 0) {
ids.push(...args.judgeEnsemble);
} else if (args.judge) {
ids.push(args.judge);
}
return ids.map((id, idx) => {
const spec = models[id];
if (!spec) {
return {
model_id: id,
provider: 'unknown',
judge_role: (idx === 0 ? 'primary' : idx === 1 ? 'secondary' : 'tertiary'),
pinning_surface: 'floating_alias' as const,
pinning_surface_carve_out_reason: `Model id '${id}' not found in benchmarks/harness/config/models.json — pinning surface cannot be resolved; treating as floating_alias for audit safety`,
};
}
return {
model_id: spec.id,
provider: spec.provider,
judge_role: spec.judge_role ?? (idx === 0 ? 'primary' : idx === 1 ? 'secondary' : 'tertiary'),
pinning_surface: spec.pinning_surface ?? 'floating_alias',
pinning_surface_carve_out_reason: spec.pinning_surface_carve_out_reason ?? null,
};
});
}
/**
* Build the `PreregistrationManifestPayload` from the RunConfig + derived
* per-run context. Split out from `runOne` so tests can exercise the
* assembly without spinning up a full run.
*
* Judge models surfacing: in Sub-deliverable A the list is derived from
* `config.judgeConfig.models` (ensemble) or `config.judgeConfig.model`
* (single-judge), with placeholder pinning fields. Sub-deliverable C
* tightens per-judge `pinning_surface` + `carve_out_reason` by looking
* each model up in `config/models.json`. Until C lands, judges emit as
* `floating_alias` + null carve-out (schema-valid placeholder).
*/
function assemblePreregistrationPayload(
config: RunConfig,
datasetVersion: string,
datasetInstanceCount: number,
): PreregistrationManifestPayload {
const manifestHash = config.manifestHash ?? computeBenchSpecManifestHash();
// `readManifestLockedDate` throws if the YAML can't be located and no
// override is provided. If the caller supplied `manifestHash` explicitly
// (test path), we still try to resolve the YAML for `locked_at`; if
// resolution fails under that branch, fall back to 'unknown' so tests
// that don't carry the YAML keep working.
let manifestLockedAt: string;
try {
manifestLockedAt = readManifestLockedDate();
} catch {
manifestLockedAt = 'unknown';
}
const perCell = config.perCellList ?? [config.run.name];
// Sub-deliverable C: `judgeModelsResolved` is pre-materialised in main()
// from args.judgeEnsemble against config/models.json. When judging is
// disabled the list is empty (schema-valid per brief § 3.1).
const judgeModels: JudgeModelManifestEntry[] = config.judgeModelsResolved ?? [];
return {
manifest_hash: manifestHash,
manifest_path: CANONICAL_MANIFEST_PATH,
manifest_locked_at: manifestLockedAt,
dataset_version: datasetVersion,
dataset_path: config.dataset.dataPath,
dataset_instance_count: datasetInstanceCount,
per_cell: perCell,
judge_tiebreak: config.judgeTiebreak ?? 'quadri-vendor',
judge_models: judgeModels,
emitted_at: new Date().toISOString(),
runner_version: getRunnerVersion(),
runner_invocation: {
argv: sanitizeArgv(process.argv),
cwd: process.cwd(),
},
};
}
// ── Main ───────────────────────────────────────────────────────────────────
function buildRuns(args: ParsedArgs): RunKind[] {
const runs: RunKind[] = [];
// Sprint 12 Task 1 Blocker #3: `--per-cell` (multi-value) overrides the
// single-cell / all-cells dispatch when at least one value is supplied.
// Sprint 12 Task 2.5 Stage 2-Retry §1.5: valid-name list extended to
// include retrieval + agentic (Stage 1) + no-context (Stage 2-Retry).
// `--all-cells` is preserved at the Sprint 9 quartet for backward
// compatibility per PM ratification; `scripts/run-mini-locomo.ts
// --v3-cells` expands to the Stage 2-Retry 5-cell list via --per-cell.
const VALID_CELLS = 'raw | filtered | compressed | full-context | retrieval | agentic | no-context';
if (args.perCell && args.perCell.length > 0) {
for (const name of args.perCell) {
if (!isCellName(name)) {
throw new Error(`Unknown cell: ${name}. Valid: ${VALID_CELLS}`);
}
runs.push({ kind: 'cell', name });
}
return runs;
}
if (args.allCells) {
(['raw', 'filtered', 'compressed', 'full-context'] as CellName[]).forEach(n => {
runs.push({ kind: 'cell', name: n });
});
} else if (args.control) {
if (!isControlName(args.control)) {
throw new Error(`Unknown control: ${args.control}. Valid: verbose-fixed`);
}
runs.push({ kind: 'control', name: args.control });
} else if (args.cell) {
if (!isCellName(args.cell)) {
throw new Error(`Unknown cell: ${args.cell}. Valid: ${VALID_CELLS}`);
}
runs.push({ kind: 'cell', name: args.cell });
} else {
throw new Error('Must specify one of --cell <name>, --all-cells, --per-cell <name>, or --control <name>. Try --help.');
}
return runs;
}
function defaultOutputPath(kind: RunKind, dataset: string): string {
const ts = new Date().toISOString().replace(/[:.]/g, '-');
const resultsDir = path.join(harnessRoot(), '..', 'results');
return path.join(resultsDir, `${kind.name}-${dataset}-${ts}.jsonl`);
}
async function main(): Promise<void> {
const args = parseArgs(process.argv.slice(2));
const models = loadModels();
const datasets = loadDatasets();
const model = models[args.model];
if (!model) {
throw new Error(`Unknown model: ${args.model}. Valid ids: ${Object.keys(models).join(', ')}`);
}
const dataset = datasets[args.dataset];
if (!dataset) {
throw new Error(`Unknown dataset: ${args.dataset}. Valid ids: ${Object.keys(datasets).join(', ')}`);
}
const dryRun = args.dryRun ?? !process.env.LITELLM_URL;
const litellmUrl = process.env.LITELLM_URL ?? 'http://localhost:4000';
const litellmApiKey = process.env.LITELLM_API_KEY ?? 'sk-waggle-dev';
// Build judge config (Sprint 9 Task 2). `--judge-ensemble` wins when
// both flags are present. In dry-run mode judge is force-skipped —
// calling a real LiteLLM judge while the cell path is stubbed would
// produce mixed-signal JSONL that's hard to interpret.
const judgeCosts: JudgeClientCostEntry[] = [];
let judgeConfig: JudgeConfig | undefined;
if (!dryRun) {
if (args.judgeEnsemble && args.judgeEnsemble.length > 0) {
const clients = new Map<string, import('./judge-client.js').LlmClient>();
for (const m of args.judgeEnsemble) {
clients.set(
m,
createJudgeLlmClient({
litellmUrl,
litellmApiKey,
model: m,
onCall: entry => judgeCosts.push(entry),
}),
);
}
// Sprint 11 B2 fold-in (2026-04-22): when the primary ensemble is
// exactly 3 vendors (the Sprint 10 Task 2.2 ratified trio — Opus 4.7
// + GPT-5.4 + Gemini 3.1-Pro), auto-wire xai/grok-4.20 as the
// fourth-vendor tie-breaker per decisions/2026-04-22-tie-break-policy-locked.md.
// Skip wiring when: (a) the ensemble isn't 3-vendor, (b) grok-4.20 is
// already in the primary list (would create dual-role ambiguity).
//
// Sprint 12 Task 2 §2.3 (2026-04-23): split the models.json key (used
// to construct the LiteLLM client with the correct route resolution)
// from the OpenRouter canonical audit slug (used as the
// `tie_break_fourth_vendor` JSONL column + pino `fourth_vendor_slug`).
// Without the split, the JSONL audit log would report the harness-
// local key `'grok-4.20'` and diverge from the §2.2-verified
// OpenRouter slug `x-ai/grok-4.20`.
const TIE_BREAKER_MODEL_KEY = 'grok-4.20'; // models.json key (LiteLLM route)
const TIE_BREAKER_AUDIT_SLUG = 'x-ai/grok-4.20'; // OpenRouter canonical (§2.2 verified 2026-04-23)
const shouldWireTieBreaker =
args.judgeEnsemble.length === 3 && !args.judgeEnsemble.includes(TIE_BREAKER_MODEL_KEY);
let tieBreakerClient: import('./judge-client.js').LlmClient | undefined;
if (shouldWireTieBreaker) {
tieBreakerClient = createJudgeLlmClient({
litellmUrl,
litellmApiKey,
model: TIE_BREAKER_MODEL_KEY,
onCall: entry => judgeCosts.push(entry),
});
}
judgeConfig = {
kind: 'ensemble',
models: args.judgeEnsemble,
clients,
...(shouldWireTieBreaker && tieBreakerClient && {
tieBreakerModel: TIE_BREAKER_AUDIT_SLUG,
tieBreakerClient,
}),
};
} else if (args.judge) {
judgeConfig = {
kind: 'single',
model: args.judge,
client: createJudgeLlmClient({
litellmUrl,
litellmApiKey,
model: args.judge,
onCall: entry => judgeCosts.push(entry),
}),
};
}
}
const runs = buildRuns(args);
// Sprint 12 Task 1 Blocker #3: invocation-level cell list fed into every
// runOne so the pre-registration payload reports the full scope rather
// than a single runOne's slice. Stripping kind='control' entries from the
// list would hide verbose-fixed-only invocations; report them as-is.
const perCellList = runs.map(r => r.name);
// Sub-deliverable C: materialise judge-model roster with B3 addendum
// pinning fields so every pre-registration emit carries audit-anchor
// metadata without repeating the models.json lookup per runOne.
const judgeModelsResolved = resolveJudgeModelsForPreregistration(models, args);
// Sprint 12 Task 2.5 Stage 1: when the cell roster includes retrieval or
// agentic, build an ephemeral substrate and pre-ingest every LoCoMo turn as
// an I-frame (frame-per-turn per GATE-S0 lock). Substrate is shared across
// every cell in this invocation so the corpus is indexed once, not per-cell.
// Lifecycle is owned here; `close()` fires in the `finally` block below.
const substrateNeeded = runs.some(
r => r.kind === 'cell' && (r.name === 'retrieval' || r.name === 'agentic'),
);
let substrate: Substrate | null = null;
let runnerLock: LockHandle | null = null;
try {
// §7.4 Stage 1.5 — acquire the global single-runner lock FIRST. Fail
// fast before any substrate ingest or health-probe cost is spent.
// Sentinel lives under benchmarks/results/ so any two runner invocations
// contend regardless of their --output paths.
const lockSentinel = path.join(harnessRoot(), '..', 'results', '.benchmark-runner');
runnerLock = acquireRunnerLock(lockSentinel);
console.log(`[bench:lock] acquired ${runnerLock.lockPath} (pid=${process.pid})`);
// §7.3 Stage 1.5 — pre-cell health check. Probes /health/liveliness + a
// short ping call per (subject + judge ensemble). Skipped in dry-run
// mode (no upstream to probe). Any 5xx or network error aborts before
// any instance budget is spent.
if (!dryRun) {
const judgePingModels = args.judgeEnsemble
? [...args.judgeEnsemble]
: (args.judge ? [args.judge] : []);
const hc = await preCellHealthCheck({
litellmUrl,
litellmApiKey,
subjectModel: model.litellmModel,
judgeModels: judgePingModels,
});
if (!hc.ok) {
const summary = hc.failures.map(f => `${f.endpoint} → ${f.error}`).join('; ');
throw new Error(
`[bench:health-check] FAILED in ${hc.durationMs}ms against ${litellmUrl}: ${summary}. ` +
`Aborting before any instance budget is spent. Check LiteLLM proxy + provider keys.`,
);
}
console.log(
`[bench:health-check] OK — probed subject + ${judgePingModels.length} judge(s) in ${hc.durationMs}ms`,
);
}
if (substrateNeeded) {
if (dryRun) {
// Retrieval + agentic aren't meaningful in dry-run mode (the agent
// loop would 404 against the stub LLM, and the embedder would still
// need a live Ollama server). Fail loudly rather than ship silently
// broken results.
throw new Error(
'retrieval/agentic cells cannot run in dry-run mode. Remove --dry-run ' +
'or set LITELLM_URL to activate live mode.',
);
}
if (!args.locomoRawPath) {
throw new Error(
'retrieval/agentic cells require --locomo-raw-path <path>. ' +
'Point it at the raw snap-research/locomo10.json archive.',
);
}
const rawPath = path.isAbsolute(args.locomoRawPath)
? args.locomoRawPath
: path.resolve(process.cwd(), args.locomoRawPath);
substrate = createSubstrate(); // default: :memory: + ollama-embedder
const substrateStart = Date.now();
const turns = extractTurnsFromLocomoRaw(rawPath);
const stats = await ingestLoCoMoCorpus(
substrate.db, substrate.search, substrate.frames, substrate.sessions, turns,
);
console.log(
`[bench:substrate] turns_found=${turns.length} frames_created=${stats.count} ` +
`ingest_ms=${stats.ingestMs} index_ms=${stats.indexMs} total_ms=${Date.now() - substrateStart}`,
);
}
for (const run of runs) {
const outputPath = args.output ?? defaultOutputPath(run, args.dataset);
await runOne({
run,
dataset,
model,
limit: args.limit,
seed: args.seed,
budgetUsd: args.budget,
outputPath,
dryRun,
litellmUrl,
litellmApiKey,
sampleLockPath: args.sampleLock,
judgeConfig,
onJudgeCall: entry => judgeCosts.push(entry),
manifestHash: args.manifestHash,
emitPreregistrationEvent: args.emitPreregistrationEvent,
perCellList,
judgeTiebreak: args.judgeTiebreak,
judgeModelsResolved,
// Task 2.5 Stage 1 — substrate deps (undefined for non-substrate cells)
substrate: substrate ?? undefined,
litellm: { url: litellmUrl, apiKey: litellmApiKey },
retrievalTopK: args.retrievalTopK,
agenticMaxTurns: args.agenticMaxTurns,
agenticTimeoutMs: args.agenticTimeoutMs,
});
}
// Surface judge spend separately from cell spend so the brief's "$5
// alarm per run" guardrail applies cleanly to the judge-layer budget.
if (judgeCosts.length > 0) {
const judgeTotalUsd = judgeCosts.reduce((sum, e) => sum + e.usd, 0);
const judgeOk = judgeCosts.filter(e => e.ok).length;
console.log(
`[bench:judge-summary] calls=${judgeCosts.length} ok=${judgeOk} ` +
`failed=${judgeCosts.length - judgeOk} total_usd=$${judgeTotalUsd.toFixed(6)}`,
);
}
} finally {
if (substrate) {
substrate.close();
}
if (runnerLock) {
runnerLock.release();
}
}
}
// Only run main() when invoked directly (not when imported by tests).
const isMain =
typeof process !== 'undefined' &&
Array.isArray(process.argv) &&
process.argv[1] !== undefined &&
url.fileURLToPath(import.meta.url) === path.resolve(process.argv[1]);
if (isMain) {
main().catch(err => {
console.error('[bench:error]', err?.message ?? err);
process.exit(1);
});
}
// Exported for the smoke tests.
export { main, parseArgs, buildRuns, runOne, defaultOutputPath };

View File

@@ -0,0 +1,169 @@
/**
* Sprint 12 Task 1 Blocker #5 — conversation-level cluster bootstrap CI.
*
* Non-parametric 95% CI that respects LoCoMo's hierarchical structure: each
* conversation contributes multiple question instances, so instance-level
* rows are NOT independent. Wilson (which assumes independence) under-
* estimates uncertainty when intra-cluster correlation is high; cluster
* bootstrap resamples WHOLE conversations with replacement and rebuilds
* the correctness distribution from each resample.
*
* A3 LOCK § 2 cluster-bootstrap parameters (LOCKED):
* - iterations = 10 000
* - seed = 42
* - cluster unit = conversation_id
* - resample mode = cluster-level with replacement
* - quantiles = 2.5 / 97.5
*
* Determinism: this module uses a custom Mulberry32 PRNG seeded with
* `input.seed` — no external dep (seedrandom NOT in repo per R4
* verification). Same input + seed produces bit-identical output.
*
* Reference: Efron, B., & Tibshirani, R. J. (1993). "An Introduction to
* the Bootstrap." Chapman & Hall. Cluster-resampling variant: Field &
* Welsh (2007) "Bootstrapping clustered data," JRSS B 69(3).
*/
export interface CorrectnessRow {
conversation_id: string;
/** 1 = correct (passes judge), 0 = incorrect. */
correct: 0 | 1;
}
export interface BootstrapInput {
rows: readonly CorrectnessRow[];
/** Number of bootstrap iterations. A3 LOCK § 2 LOCKED value: 10 000. */
n_bootstrap?: number;
/** PRNG seed for determinism. A3 LOCK § 2 LOCKED value: 42. */
seed?: number;
/** Confidence level — only 0.95 supported in this implementation. */
confidence?: number;
}
export interface BootstrapResult {
point_estimate: number;
ci_lower: number;
ci_upper: number;
n_bootstrap: number;
seed: number;
confidence: number;
/** Number of distinct conversations that contributed rows (cluster count). */
n_clusters: number;
/** Row count (informational — larger than n_clusters when clustering is real). */
n_rows: number;
}
/**
* Mulberry32 PRNG — 32-bit xorshift variant. Uniformly distributed on
* [0, 1) given a 32-bit seed. Same seed ⇒ same sequence across Node
* versions / platforms / architectures. Chosen over xorshift32 (already
* used in datasets.ts) purely because the Fisher-Yates + bootstrap
* idiom in stats literature cites Mulberry32 more often — functionally
* equivalent for our needs.
*
* Reference: https://github.com/bryc/code/blob/master/jshash/PRNGs.md
*/
function mulberry32(seed: number): () => number {
let state = seed >>> 0;
return () => {
state = (state + 0x6D2B79F5) >>> 0;
let t = state;
t = Math.imul(t ^ (t >>> 15), t | 1);
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}
export function computeClusterBootstrapCI(input: BootstrapInput): BootstrapResult {
const {
rows,
n_bootstrap = 10000,
seed = 42,
confidence = 0.95,
} = input;
if (!Array.isArray(rows) || rows.length === 0) {
throw new Error('cluster-bootstrap requires a non-empty rows array');
}
if (!Number.isFinite(n_bootstrap) || n_bootstrap < 1 || !Number.isInteger(n_bootstrap)) {
throw new Error(`cluster-bootstrap requires n_bootstrap ≥ 1 (integer); got ${n_bootstrap}`);
}
if (!Number.isFinite(seed) || !Number.isInteger(seed)) {
throw new Error(`cluster-bootstrap requires an integer seed; got ${seed}`);
}
if (confidence !== 0.95) {
throw new Error(
`cluster-bootstrap only supports confidence=0.95 in this implementation; got ${confidence}`,
);
}
// Group rows by conversation_id. Preserve insertion order so the PRNG
// sees the same cluster sequence on every call with the same input.
const clusterMap = new Map<string, CorrectnessRow[]>();
for (const row of rows) {
if (row.correct !== 0 && row.correct !== 1) {
throw new Error(
`cluster-bootstrap rows must have correct ∈ {0, 1}; got ${row.correct} at conversation ${row.conversation_id}`,
);
}
const bucket = clusterMap.get(row.conversation_id);
if (bucket) bucket.push(row);
else clusterMap.set(row.conversation_id, [row]);
}
const clusters = Array.from(clusterMap.values());
const nClusters = clusters.length;
// Point estimate — mean correctness over the full (un-resampled) input.
let totalCorrect = 0;
for (const r of rows) totalCorrect += r.correct;
const point_estimate = totalCorrect / rows.length;
// Precompute per-cluster (sum, count) so each bootstrap iteration is O(K)
// rather than O(N). Matters at n_bootstrap=10 000 with ~300 clusters.
const clusterSums = new Array<number>(nClusters);
const clusterSizes = new Array<number>(nClusters);
for (let i = 0; i < nClusters; i++) {
let s = 0;
const c = clusters[i];
for (const r of c) s += r.correct;
clusterSums[i] = s;
clusterSizes[i] = c.length;
}
const rand = mulberry32(seed);
const means = new Array<number>(n_bootstrap);
for (let b = 0; b < n_bootstrap; b++) {
let sumCorrect = 0;
let sumSize = 0;
// Draw nClusters clusters with replacement.
for (let k = 0; k < nClusters; k++) {
const pick = Math.floor(rand() * nClusters);
sumCorrect += clusterSums[pick];
sumSize += clusterSizes[pick];
}
means[b] = sumSize === 0 ? 0 : sumCorrect / sumSize;
}
means.sort((a, b) => a - b);
// 2.5th and 97.5th percentiles. Linear-interpolation variant would be
// marginally more accurate but adds implementation surface; standard
// lower-floor / upper-floor indexing is what most published bootstrap
// pipelines use and matches the A3 LOCK § 2 "quantiles: [2.5, 97.5]"
// directive without prescribing interpolation style.
const lowerIdx = Math.floor(0.025 * n_bootstrap);
const upperIdx = Math.floor(0.975 * n_bootstrap);
const ci_lower = Math.max(0, Math.min(1, means[lowerIdx]));
const ci_upper = Math.max(0, Math.min(1, means[upperIdx]));
return {
point_estimate,
ci_lower,
ci_upper,
n_bootstrap,
seed,
confidence,
n_clusters: nClusters,
n_rows: rows.length,
};
}

View File

@@ -0,0 +1,147 @@
/**
* Sprint 12 Task 1 Blocker #5 — Fleiss' κ (1971) implementation.
*
* Measures inter-rater agreement over a fixed number of categorical raters
* assigning items to K mutually exclusive categories. Used in the benchmark
* harness to compute pre-tie-break agreement across the 3-vendor primary
* judge ensemble (Opus 4.7 + GPT-5.4 + Gemini 3.1) per A3 LOCK § 4.
*
* Formula (Fleiss 1971):
* n_ij = number of judges who assigned item i to category j
* p_j = (1 / (N·n)) · Σ_i n_ij // category marginal
* P_i = (1 / (n·(n−1))) · (Σ_j n_ij² − n) // item agreement
* P_bar = (1/N) · Σ_i P_i // mean observed agreement
* P_e = Σ_j p_j² // chance-expected agreement
* κ = (P_bar − P_e) / (1 − P_e)
*
* Reference: Fleiss, J. L. (1971). "Measuring nominal scale agreement
* among many raters." Psychological Bulletin, 76(5), 378-382.
*
* Numerical note: for K=2 this reduces to a well-behaved agreement
* measure. When P_e = 1 (all judges always picked the same category),
* the denominator vanishes and κ is undefined — we return NaN in that
* case and surface the condition via the marginals.
*
* A3 LOCK § 4 HALT threshold: κ < 0.60 mid-run triggers abort. This
* module returns the raw κ; the runner-level HALT check sits alongside.
*/
export interface VoteMatrix {
/**
* Number of judges per item. Fixed across all items (Fleiss requirement).
* For the benchmark harness's 3-primary ensemble, this is 3.
*/
n_judges: number;
/**
* counts[i][k] = number of judges who assigned item i to category k.
* Row invariant: Σ_k counts[i][k] === n_judges for every i.
*/
counts: readonly (readonly number[])[];
/**
* Optional labels for the K categories (e.g. ['correct', 'F1', 'F2', ...,
* 'F_other']). Length must equal the column width of `counts`. Not used
* in the κ math — preserved for report-time display.
*/
categories?: readonly string[];
}
export interface FleissKappaResult {
/** Fleiss' κ ∈ [−1, 1]. NaN when P_e === 1 (uniform judge assignment). */
kappa: number;
/** Item count (rows of `counts`). */
n_items: number;
/** Judge count (fixed, from input). */
n_judges: number;
/** Category count (columns of `counts`). */
n_categories: number;
/** Per-category marginal proportions p_j. Sum across K categories ≈ 1. */
category_marginals: number[];
/** Mean observed agreement over items (P_bar). */
P_bar: number;
/** Chance-expected agreement (P_e = Σ p_j²). */
P_e: number;
}
export function computeFleissKappa(matrix: VoteMatrix): FleissKappaResult {
const { n_judges, counts } = matrix;
if (!Number.isFinite(n_judges) || n_judges < 2 || !Number.isInteger(n_judges)) {
throw new Error(`Fleiss κ requires n_judges ≥ 2 (integer); got ${n_judges}`);
}
if (!Array.isArray(counts) || counts.length === 0) {
throw new Error('Fleiss κ requires a non-empty counts matrix');
}
const N = counts.length;
const K = counts[0].length;
if (K < 2) {
throw new Error(`Fleiss κ requires K ≥ 2 categories; got ${K}`);
}
// Validate rectangular shape + row-sum invariant.
for (let i = 0; i < N; i++) {
if (counts[i].length !== K) {
throw new Error(
`Fleiss κ counts matrix must be rectangular; row ${i} has ${counts[i].length} cols, expected ${K}`,
);
}
let rowSum = 0;
for (let k = 0; k < K; k++) {
const v = counts[i][k];
if (!Number.isFinite(v) || v < 0 || !Number.isInteger(v)) {
throw new Error(
`Fleiss κ counts must be non-negative integers; counts[${i}][${k}] = ${v}`,
);
}
rowSum += v;
}
if (rowSum !== n_judges) {
throw new Error(
`Fleiss κ row sum must equal n_judges; row ${i} sums to ${rowSum}, expected ${n_judges}`,
);
}
}
if (matrix.categories !== undefined && matrix.categories.length !== K) {
throw new Error(
`Fleiss κ categories length (${matrix.categories.length}) must match K=${K}`,
);
}
// Category marginals p_j.
const marginals = new Array<number>(K).fill(0);
for (let k = 0; k < K; k++) {
let total = 0;
for (let i = 0; i < N; i++) total += counts[i][k];
marginals[k] = total / (N * n_judges);
}
// Per-item agreement P_i.
// P_i = (1 / (n·(n−1))) · (Σ_j n_ij² − n)
const denomItem = n_judges * (n_judges - 1);
let P_sum = 0;
for (let i = 0; i < N; i++) {
let sqSum = 0;
for (let k = 0; k < K; k++) {
const v = counts[i][k];
sqSum += v * v;
}
const P_i = (sqSum - n_judges) / denomItem;
P_sum += P_i;
}
const P_bar = P_sum / N;
const P_e = marginals.reduce((acc, p) => acc + p * p, 0);
// κ = (P_bar − P_e) / (1 − P_e). NaN when P_e === 1 (uniform assignment).
const kappa = P_e === 1 ? Number.NaN : (P_bar - P_e) / (1 - P_e);
return {
kappa,
n_items: N,
n_judges,
n_categories: K,
category_marginals: marginals,
P_bar,
P_e,
};
}

View File

@@ -0,0 +1,26 @@
/**
* Sprint 12 Task 1 Blocker #5 — statistics module barrel.
*
* Re-exports the three A3 LOCK § 2 / § 4 numerical surfaces used by the
* aggregate JSON writer and smoke test:
*
* computeFleissKappa — pre-tie-break judge agreement
* computeWilsonCI — frequentist 95% binomial CI (primary)
* computeClusterBootstrapCI — non-parametric 95% CI (secondary,
* conversation-level resampling)
*
* Import surface for consumers in Task 2 (C3 mini execution) and
* downstream Task 4 (H-42 full run):
*
* import { computeFleissKappa, computeWilsonCI, computeClusterBootstrapCI }
* from '../stats/index.js';
*/
export { computeFleissKappa } from './fleiss-kappa.js';
export type { VoteMatrix, FleissKappaResult } from './fleiss-kappa.js';
export { computeWilsonCI, Z_95_TWO_SIDED } from './wilson-ci.js';
export type { WilsonInput, WilsonResult } from './wilson-ci.js';
export { computeClusterBootstrapCI } from './cluster-bootstrap.js';
export type { CorrectnessRow, BootstrapInput, BootstrapResult } from './cluster-bootstrap.js';

View File

@@ -0,0 +1,98 @@
/**
* Sprint 12 Task 1 Blocker #5 — Wilson score interval (95% two-sided).
*
* Frequentist binomial confidence interval on a proportion. Primary CI per
* A3 LOCK § 2 for binary correctness rates; tighter than Wald at the
* boundaries (p̂ near 0 or 1) and does not require normal approximation.
*
* Formula (Wilson 1927):
* p̂ = successes / trials
* z = 1.959964 (two-sided 95%)
* denom = 1 + z²/n
* center = (p̂ + z²/(2n)) / denom
* half = z · √(p̂(1−p̂)/n + z²/(4n²)) / denom
* CI = [center − half, center + half]
*
* Reference: Wilson, E. B. (1927). "Probable inference, the law of
* succession, and statistical inference." JASA, 22(158), 209-212.
*
* A3 LOCK § 2 STRONG-PUBLISHABLE gate: Wilson lower bound ≥ 91.6% when
* computed over the full H-42a 4620-eval run. This module returns the
* raw CI; tier classification sits alongside in the aggregate writer.
*/
export interface WilsonInput {
/** Number of successes (correct verdicts). Must be integer in [0, trials]. */
successes: number;
/** Total trials. Must be positive integer. */
trials: number;
/**
* Confidence level. Only 0.95 is implemented (z=1.959964 hardcoded).
* Defaults to 0.95. Non-0.95 values throw until someone extends the
* z-lookup table — deliberate conservatism to prevent silent
* miscalibration in a launch-gating metric.
*/
confidence?: number;
}
export interface WilsonResult {
/** Point estimate p̂ = successes / trials. */
point_estimate: number;
/** Lower bound of the 95% CI, clamped to [0, 1]. */
ci_lower: number;
/** Upper bound of the 95% CI, clamped to [0, 1]. */
ci_upper: number;
/** (ci_upper − ci_lower) / 2 — symmetric half-width. */
half_width: number;
/** Confidence level echo-back (always 0.95 in this implementation). */
confidence: number;
}
/** z-score for the two-sided 95% Wilson interval. Matches standard
* tabular value: Φ⁻¹(0.975) ≈ 1.959964. */
export const Z_95_TWO_SIDED = 1.959964;
export function computeWilsonCI(input: WilsonInput): WilsonResult {
const { successes, trials, confidence = 0.95 } = input;
if (!Number.isFinite(trials) || trials <= 0 || !Number.isInteger(trials)) {
throw new Error(`Wilson CI requires trials ≥ 1 (integer); got ${trials}`);
}
if (
!Number.isFinite(successes) ||
successes < 0 ||
successes > trials ||
!Number.isInteger(successes)
) {
throw new Error(
`Wilson CI requires successes ∈ [0, trials]; got ${successes} (trials=${trials})`,
);
}
if (confidence !== 0.95) {
throw new Error(
`Wilson CI only supports confidence=0.95 in this implementation; got ${confidence}`,
);
}
const n = trials;
const p_hat = successes / n;
const z = Z_95_TWO_SIDED;
const z2 = z * z;
const denom = 1 + z2 / n;
const center = (p_hat + z2 / (2 * n)) / denom;
const halfInner = p_hat * (1 - p_hat) / n + z2 / (4 * n * n);
const half = (z * Math.sqrt(halfInner)) / denom;
// Clamp to [0, 1] defensively — Wilson is well-behaved at boundaries
// but floating-point can produce −1e-17 etc. at p̂=0 / p̂=1 edges.
const ci_lower = Math.max(0, center - half);
const ci_upper = Math.min(1, center + half);
return {
point_estimate: p_hat,
ci_lower,
ci_upper,
half_width: (ci_upper - ci_lower) / 2,
confidence,
};
}

View File

@@ -0,0 +1,90 @@
/**
* Sprint 12 Task 2.5 Stage 1.5 §7.2 — consecutive fetch-transport failure halt.
*
* The v2 full-context cell exhibited a cascade of fetch_error_TypeError
* after concurrent runner processes saturated the OpenRouter bridge (see
* sessions/2026-04-23-task25-s0-v2-fullcontext-forensic.md §4). Once
* saturation kicked in, every subsequent instance failed identically.
* Without a structural halt, the runner burned its full 100-instance budget
* on dead calls before terminating.
*
* StreakTracker counts CONSECUTIVE `fetch_error_*` results. On the Nth
* consecutive failure (default N=5), `record()` returns `true` and the
* caller should throw a hard abort. Success resets the counter. Non-fetch
* failures (timeout, http_5xx, other classes) reset the counter — those
* aren't bridge-saturation symptoms.
*
* The `recent[]` rolling window (size 10) is kept for observability only —
* it's surfaced in the halt error message so operators can see the
* immediate-history context.
*/
const DEFAULT_THRESHOLD = 5;
const DEFAULT_WINDOW_SIZE = 10;
/** Pattern match for bridge/network transport failures. Excludes `timeout`
* (per-call AbortError) and `http_5xx` (server-side error) — only counts
* errors from the client-side fetch() throwing before/during I/O. */
export function isFetchTransportFailure(failureMode: string | null | undefined): boolean {
return failureMode !== null && failureMode !== undefined && /^fetch_error_/.test(failureMode);
}
export interface StreakTrackerOptions {
/** Consecutive-failure count that triggers halt. Default 5. */
threshold?: number;
/** Rolling-window size kept for observability in the halt error message.
* Default 10. Does NOT affect the halt decision — the counter resets on
* any non-fetch result regardless of window state. */
windowSize?: number;
}
export class StreakTracker {
private consecutive = 0;
private readonly recent: boolean[] = [];
private readonly threshold: number;
private readonly windowSize: number;
constructor(options: StreakTrackerOptions = {}) {
this.threshold = Math.max(1, options.threshold ?? DEFAULT_THRESHOLD);
this.windowSize = Math.max(1, options.windowSize ?? DEFAULT_WINDOW_SIZE);
}
/**
* Record one evaluation outcome. Returns `true` when the consecutive
* fetch-transport failure count has reached the halt threshold. Caller
* should throw a clear abort error on `true` and stop the cell loop.
*
* Idempotent — calling `record(null)` multiple times resets the counter
* to zero each time; subsequent non-null fetch_error_* calls count up
* from zero again.
*/
record(failureMode: string | null | undefined): boolean {
const transportFailure = isFetchTransportFailure(failureMode);
this.consecutive = transportFailure ? this.consecutive + 1 : 0;
this.recent.push(transportFailure);
if (this.recent.length > this.windowSize) this.recent.shift();
return this.consecutive >= this.threshold;
}
/** Current consecutive-failure count. */
getConsecutiveFailures(): number {
return this.consecutive;
}
/** Snapshot of the last `windowSize` outcomes (true=transport-failure). */
getRecentWindow(): readonly boolean[] {
return [...this.recent];
}
/** Human-readable summary for halt error messages / logs. */
summary(): string {
const windowStr = this.recent.map(b => (b ? 'X' : '.')).join('');
return `consecutive=${this.consecutive} window_last${this.windowSize}=[${windowStr}] threshold=${this.threshold}`;
}
/** Reset all state. For reuse across cells if desired. */
reset(): void {
this.consecutive = 0;
this.recent.length = 0;
}
}

View File

@@ -0,0 +1,79 @@
/**
* Task 2.5 Stage 1 — ephemeral substrate factory.
*
* Composes the three @waggle/core primitives the retrieval + agentic cells
* need into a single lifecycle object:
*
* MindDB(dbPath) — SQLite + sqlite-vec handle. Default `:memory:`.
* FrameStore(db) — I/P/B-frame CRUD + FTS5 auto-index.
* HybridSearch(db, embedder) — RRF-fused FTS5 + vec0 search.
*
* `close()` releases the DB handle; for `:memory:` paths this also frees the
* FTS5 + vec0 indices that live inside the same process-local handle.
*
* Embedder defaults to `createOllamaEmbedder()` (baseUrl http://localhost:11434,
* model `nomic-embed-text`, 1024 dims — matches VEC_TABLE_SQL). Callers can
* override to inject a deterministic fake (unit tests) or swap to
* LiteLLM / API embedders later without touching substrate internals.
*
* No network I/O at construction time if the caller supplies an embedder.
* When the default ollama-embedder is used, HybridSearch will only hit the
* Ollama server during `search()` / `indexFramesBatch()` calls — construction
* itself stays cheap.
*/
import {
FrameStore,
HybridSearch,
MindDB,
SessionStore,
createOllamaEmbedder,
type Embedder,
} from '@waggle/core';
export interface SubstrateOptions {
/** SQLite path. Defaults to `:memory:` for benchmark-ephemeral use. */
dbPath?: string;
/** Pre-built embedder. When omitted, a fresh ollama-embedder is created. */
embedder?: Embedder;
}
export interface Substrate {
db: MindDB;
frames: FrameStore;
/** Sessions keyed by gop_id — required by the `memory_frames.gop_id ->
* sessions.gop_id` FK constraint. Callers that insert frames directly
* must first `substrate.sessions.ensure(gopId, ...)` for each new gop. */
sessions: SessionStore;
search: HybridSearch;
embedder: Embedder;
/** Release the DB handle. Safe to call multiple times. */
close(): void;
}
/**
* Build a fresh substrate. Caller owns lifecycle — MUST call `close()` when
* done (typically in a `try/finally`).
*/
export function createSubstrate(opts: SubstrateOptions = {}): Substrate {
const dbPath = opts.dbPath ?? ':memory:';
const embedder = opts.embedder ?? createOllamaEmbedder();
const db = new MindDB(dbPath);
const frames = new FrameStore(db);
const sessions = new SessionStore(db);
const search = new HybridSearch(db, embedder);
let closed = false;
return {
db,
frames,
sessions,
search,
embedder,
close(): void {
if (closed) return;
closed = true;
db.close();
},
};
}

View File

@@ -0,0 +1,507 @@
/**
* Four-cell ablation harness — shared types.
*
* The four cells isolate causal contributions to end-to-end quality:
* Cell 1 — raw: LLM only, stateless per turn.
* Cell 2 — filtered: LLM + memory retrieval, no prompt evolution.
* Cell 3 — compressed: LLM + prompt evolution, no memory retrieval.
* Cell 4 — full-context: LLM + memory + prompt evolution.
*
* Sprint 12 Task 1 Blocker #2 (2026-04-22) renamed cell keys from the
* Sprint 10 technical labels to the A3 LOCK publication-ready labels.
* Architecture unchanged — only key strings. Migration script
* (benchmarks/scripts/migrate-cell-names.ts) rewrites any legacy JSONL
* artefacts with the old keys.
*
* Same dataset, same seed, same model across all four. Difference at report
* time between a baseline cell and a treatment cell isolates the causal
* contribution of the ablated component.
*
* Controls (not cells) sit alongside for sanity checking:
* verbose-fixed: fixed-prompt-with-verbose-instructions. Day 1 check that
* the harness is not broken (should underperform raw).
*
* Out-of-scope (Week 2+): naive-RAG, oracle-memory ceiling, Llama-3.1-8B,
* Opus 4.6, Gemma 2 9B probe, full τ-bench + LongMemEval loaders.
*/
// ── Sprint 12 Task 2 / §2.1 A3 namespace split (LOCKED 2026-04-23) ──────
// A3 LOCK § 6 failure taxonomy + its aggregate distribution are surfaced
// here as type-only re-exports so any downstream consumer (harness reader,
// exit-ping generator, external audit tooling) gets a single import
// entry point for the canonical A3 taxonomy column types.
export type { FailureCode } from './failure-taxonomy/codes.js';
export type { FailureDistribution } from './failure-taxonomy/aggregate.js';
import type { FailureCode } from './failure-taxonomy/codes.js';
import type { FailureDistribution } from './failure-taxonomy/aggregate.js';
// Sprint 12 Task 2.5 Stage 1 (2026-04-23): Substrate is imported type-only so
// types.ts has zero runtime dependency on substrate.ts / @waggle/core. The
// field threads through RunConfig → runOne → CellInput for the retrieval +
// agentic cells.
import type { Substrate } from './substrate.js';
/**
* Cell names. Sprint 9 / Sprint 12 Task 1 Blocker #2 shipped the first four
* (`raw`, `filtered`, `compressed`, `full-context`). Sprint 12 Task 2.5
* Stage 1 (2026-04-23) added `retrieval` and `agentic` — backed by real
* `@waggle/core::HybridSearch` and `@waggle/agent::agent-loop` respectively.
*
* Sprint 12 Task 2.5 Stage 2-Retry (2026-04-24) added `no-context` — a
* true zero-memory baseline (question-only prompt, no `instance.context`,
* no retrieval). The Stage 2 N=20 FAIL exit revealed that Sprint 9 `raw`
* is NOT a zero-context baseline on LoCoMo (its prompt embeds the
* oracle-selected `instance.context`); `no-context` is the honest
* comparator for the retrieval memory-lift criterion.
*
* The v3 PM-facing vocabulary for Stage 2-Retry is
* `[no-context, oracle-context, full-context, retrieval, agentic]`; mapping
* to harness internal ids via `scripts/run-mini-locomo.ts::V3_TO_V1_CELLS`:
* no-context → no-context (NEW; true zero-memory baseline)
* oracle-context → raw (alias — oracle-fed diagnostic; harness `raw` kept for back-compat)
* full-context → full-context (unchanged)
* retrieval → retrieval (now conv-scope top-K=20)
* agentic → agentic (now conv-scope + softened SYSTEM_AGENTIC + fallback)
*/
export type CellName =
| 'raw'
| 'filtered'
| 'compressed'
| 'full-context'
| 'retrieval'
| 'agentic'
| 'no-context';
export type ControlName = 'verbose-fixed';
export type RunKind = { kind: 'cell'; name: CellName } | { kind: 'control'; name: ControlName };
export interface DatasetInstance {
/** Stable id used in the JSONL record for cross-cell joining. */
instance_id: string;
/** Question or task statement shown to the model. */
question: string;
/** Dataset-supplied context that raw / compressed cells receive verbatim
* and memory cells may ignore in favor of their retrieval layer. */
context: string;
/** Canonical reference answer(s) for automated scoring. */
expected: string[];
/** Sprint 12 Task 2.5 Stage 2-Retry (2026-04-24): identifier of the
* conversation this QA pair was authored within. For LoCoMo this is the
* `conversation_id` / `sample_id` carried by the canonical archive (e.g.
* `conv-26`). Retrieval + agentic cells use this as a `gopId` filter so
* HybridSearch scopes to the instance's conversation only, matching the
* QA-pair locality LoCoMo was authored for. Undefined for synthetic runs
* and pre-Stage-2-Retry JSONL artefacts. */
conversation_id?: string;
}
export interface DatasetSpec {
id: 'locomo' | 'longmemeval' | 'beam-128k' | 'beam-1m' | 'synthetic';
displayName: string;
/** Where the loader looks for the data. Relative to `benchmarks/data/`. */
dataPath: string;
/** `synthetic` dataset has instances hard-coded in the harness so scaffold
* smoke tests don't need external downloads. */
source: 'synthetic' | 'external';
}
/**
* Pinning surface enum per B3 LOCK addendum § 4. Tags every model entry
* (targets + judges) with the audit guarantee it offers:
*
* anthropic_immutable — Anthropic-direct routes, dated snapshot pinned
* upstream. H-AUDIT-2 spot-check re-runs will hit
* the exact same model bytes.
* floating_alias — provider rotates underlying model silently;
* replication tolerates semantic equivalence only,
* not byte-level match. Requires non-null
* pinning_surface_carve_out_reason.
* revision_hash_pinned — provider exposes a revision hash we capture into
* the JSONL row; replication binds to that hash.
*/
export type PinningSurface = 'anthropic_immutable' | 'floating_alias' | 'revision_hash_pinned';
/**
* Judge ensemble role classification.
*
* Sprint 12 Task 1 judge-role remap (2026-04-22, brief
* `briefs/2026-04-22-cc-sprint-12-task1-judge-role-remap.md`):
* B2 LOCK § 1 treats Opus 4.7 + GPT-5.4 + Gemini 3.1 as a 3-vendor
* primary ensemble (all `primary`); Grok 4.20 is the tie-break `reserve`.
* `secondary` / `tertiary` are retained in the enum for backward
* compatibility — future models may populate them, but no current entry
* in models.json uses them after this remap.
*/
export type JudgeRole = 'primary' | 'secondary' | 'tertiary' | 'reserve';
export type ModelProvider =
| 'alibaba'
| 'anthropic'
| 'ollama'
| 'litellm-proxy'
| 'local'
| 'openai_via_openrouter'
| 'google_via_openrouter'
| 'xai_via_openrouter';
export interface ModelSpec {
id: string;
displayName: string;
provider: ModelProvider;
/** Route string the LiteLLM proxy recognizes. */
litellmModel: string;
/** USD per 1M input tokens. */
pricePerMillionInput: number;
/** USD per 1M output tokens. */
pricePerMillionOutput: number;
/** Context window in tokens (for truncation decisions). */
contextWindow: number;
// ── Sprint 12 Task 1 Blocker #4 / B3 addendum § 4 fields ─────────────────
/** Pinning surface classification per B3 LOCK addendum § 4. Absence is
* tolerated for pre-Sprint-12 entries but Blocker #4 seeds every entry
* explicitly so the models-config test covers the registry. */
pinning_surface?: PinningSurface;
/** Human-readable rationale for floating_alias / revision_hash_pinned
* entries. MUST be null for anthropic_immutable. MUST be non-null for
* the other two surfaces (enforced in models-config.test.ts). */
pinning_surface_carve_out_reason?: string | null;
/** Role classification for judge-ensemble entries. Targets (systems under
* test) leave this undefined. Primary judges are the 3-vendor ensemble
* (Sprint 10 Task 2.2 ratified trio); secondary / tertiary are reserve /
* tie-break roles per A3 LOCK § 4. */
judge_role?: JudgeRole;
/**
* Sprint 11 Task B1 (2026-04-22): Stage 2 LOCKED config (thinking=on,
* max_tokens=64000) per decisions/2026-04-22-stage-2-primary-config-locked.md.
*
* When present, C2/C3 harness runs apply these overrides to every LLM call
* explicitly (not inherited from request defaults). Absent = harness uses
* the legacy max_tokens=600 default and no reasoning flag.
*/
stage2Config?: {
/** Request `reasoning: { enabled: true }` (OpenRouter unified shape). */
thinking: boolean;
/** Override `max_tokens` in the request body (Stage 2 LOCK: 64000). */
maxTokens: number;
/**
* Response-side parser hint. OpenRouter unified returns
* `message.reasoning`; DashScope native returns `message.reasoning_content`.
* Parser accepts either shape regardless; this is an annotation for
* routing expectations.
*/
reasoningShape: 'openrouter-unified' | 'dashscope-native';
};
}
export interface RunConfig {
run: RunKind;
dataset: DatasetSpec;
model: ModelSpec;
/** Limit the number of instances to run. `Infinity` = full dataset. */
limit: number;
/** Reproducibility seed. Same seed → same instance order + same prompts. */
seed: number;
/** Hard USD cap — run stops when cumulative cost exceeds. `Infinity` disables. */
budgetUsd: number;
/** Absolute path to the JSONL output file. */
outputPath: string;
/** When true, the LLM client returns a deterministic stub response instead
* of calling LiteLLM. Smoke tests + offline scaffolding. */
dryRun: boolean;
/** LiteLLM proxy URL. Only used when dryRun is false. */
litellmUrl: string;
/** LiteLLM bearer key. Only used when dryRun is false. */
litellmApiKey: string;
/** Optional path to a committed sample-lock JSON. When set, bypasses the
* dataset adapter and loads instances directly from the lock file. Runtime
* asserts the distribution required for the Stage 2 preflight gate. */
sampleLockPath?: string;
/** Sprint 9 Task 2. When set, runner invokes `failure-mode-judge` after
* each cell call with this config. Single-judge or ensemble depending
* on the shape; undefined means judging is disabled. The runner keeps
* its own budget ledger for judge spend separate from cell spend so
* the brief's "$5 alarm per run" can be respected independently. */
judgeConfig?: import('./judge-runner.js').JudgeConfig;
/** Optional sink for per-call judge cost entries. Typically the caller
* collects into an array to summarise in the run output. */
onJudgeCall?: (entry: import('./judge-client.js').JudgeClientCostEntry) => void;
// ── Sprint 12 Task 1 Blocker #3 — pre-registration inputs ────────────────
/** Pre-computed SHA-256 of the bench-spec manifest YAML. When undefined,
* the runner resolves the YAML path (env → sibling PM-Waggle-OS → throw)
* and computes the hash at run start. */
manifestHash?: string;
/** Suppress the `bench.preregistration.manifest_hash` pino-style event
* when `false`. Defaults to emitting (true). Tests pass `false` to keep
* log noise down. */
emitPreregistrationEvent?: boolean;
/** Invocation-level cell list — the full scope of cells this benchmark
* invocation spans. Each `runOne` call receives the same list so its
* emitted pre-registration event reports the full scope (not the
* single-cell subset it runs). Derived from CLI `--per-cell` or
* `--all-cells` / `--cell` expansion. */
perCellList?: string[];
/** Judge tie-break strategy surfaced into the pre-registration payload. */
judgeTiebreak?: string;
/**
* Resolved judge model roster with per-model pinning fields, already
* looked up against `config/models.json`. Main() materializes this list
* once from CLI `--judge-ensemble` args so runOne doesn't re-read the
* model registry on every cell iteration. Undefined when judging is
* disabled — pre-registration payload emits an empty array in that case.
*/
judgeModelsResolved?: import('./preregistration.js').JudgeModelManifestEntry[];
// ── Sprint 12 Task 2.5 Stage 1 (2026-04-23) — substrate deps for retrieval + agentic cells ─
/**
* Ephemeral MindDB + HybridSearch pair built by main() via
* `createSubstrate({embedder})` and pre-populated via
* `ingestLoCoMoCorpus(...)` BEFORE any cell fires. Required only when the
* run roster includes `retrieval` or `agentic`; the other four cells ignore
* it. Lifecycle is owned by main() — `close()` is called in its `finally`.
*/
substrate?: Substrate;
/**
* LiteLLM URL + API key surfaced to CellInput so the `agentic` cell's
* inner `runAgentLoop` can talk to the proxy directly (agent-loop does
* not route through the cell's `LlmClient`). Other cells ignore.
*/
litellm?: { url: string; apiKey: string };
/** Retrieval cell top-K override. Default 10 (GATE-S0 lock). */
retrievalTopK?: number;
/** Agentic cell turn cap override. Default 3 (GATE-S0 lock). */
agenticMaxTurns?: number;
/** Agentic cell AbortController timeout override. Default 180_000 ms. */
agenticTimeoutMs?: number;
}
/** Judge failure-mode taxonomy codes. Must match the enum the judge module
* returns — see `packages/server/src/benchmarks/judge/failure-mode-judge.ts`.
*
* Sprint 9 5-value space. Retained as a legacy read-only surface per the
* 2026-04-23 `decisions/2026-04-23-jsonl-record-taxonomy-split-locked.md`
* namespace-split LOCK — C2 stage 1 and pre-A3 JSONL archives continue to
* parse against this shape. A3 LOCK § 6 callers use `FailureCode` (8-value)
* from `./failure-taxonomy/codes.js` via the `a3_failure_code` / `a3_rationale`
* columns on `JsonlRecord` below. */
export type FailureMode = 'F1' | 'F2' | 'F3' | 'F4' | 'F5';
/** Single-judge verdict payload embedded in a JsonlRecord. Shape matches
* taxonomy spec §9 (binary `verdict` + separate `failure_mode` slot) —
* not the brief Task-1 6-value combined enum. The binary shape is what
* the failure-mode-judge module already returns, so keeping them aligned
* avoids a lossy conversion at the wiring step. If downstream wants the
* 6-value form ("correct" / "F1_abstain" / … / "F5_offtopic"), the
* aggregator (Task 3) computes it from `judge_verdict` + `failure_mode`
* at report time. */
export type JudgeVerdict = 'correct' | 'incorrect';
export interface JudgeEnsembleEntry {
model: string;
verdict: JudgeVerdict;
failure_mode: FailureMode | null;
rationale?: string;
/** Per-judge wall-clock latency in ms. Aggregated into Task 3's cost
* summary "median ms per judge call". */
latency_ms?: number;
}
/** One entry in the per-instance JSONL output. Shape intentionally flat so
* downstream analysis (pandas, jq, DuckDB) stays trivial.
*
* Judge fields are all **optional** so pre-judge JSONL files (any output
* produced before Sprint 9 Task 2 lands) still parse. Consumers treat
* `judge_verdict === undefined` as "not judged yet". */
export interface JsonlRecord {
turnId: string;
cell: CellName | ControlName;
instance_id: string;
model: string;
seed: number;
accuracy: number;
p50_latency_ms: number;
p95_latency_ms: number;
usd_per_query: number;
/** Existing since Sprint 7 — retained as-is. Populated from the cell's
* own LLM call failure-mode signal (transport errors, quota, etc.),
* NOT from the judge. For the judge's taxonomy code, use the separate
* `judge_failure_mode` field below. */
failure_mode: string | null;
// ── Judge extension (taxonomy §9 / Sprint 9 Task 1) ──────────────
/** Verbatim answer the model under test produced for this instance.
* Stored so the judge + aggregator + any later re-judge pass can
* operate on the same string without re-running the model. */
model_answer?: string;
/** Binary judge verdict from `judgeAnswer` / `judgeEnsemble` majority.
* "correct" implies `judge_failure_mode === null`; "incorrect"
* implies it's one of F1..F5. This invariant is enforced by the
* judge module's Zod schema and must hold here too. */
judge_verdict?: JudgeVerdict;
/** Failure-mode taxonomy code when `judge_verdict === "incorrect"`.
* `null` when verdict is "correct" or when judging is skipped. */
judge_failure_mode?: FailureMode | null;
/** One-sentence rationale string the judge returned alongside the
* verdict. Not parsed — preserved for human spot-check + downstream
* disagreement analysis in Task 3. */
judge_rationale?: string;
/** Model id used for the judge call (e.g. `claude-sonnet-4-6`). For
* ensemble mode this is `ensemble_majority`; per-judge ids live in
* `judge_ensemble[]`. */
judge_model?: string;
/** ISO-8601 timestamp of the judge call. Traceability for EU-AI-Act
* Art. 14 audit surface. */
judge_timestamp?: string;
/** 0.0–1.0 confidence the judge expressed in its verdict. Optional at
* the schema level because not every judge prompt variant includes
* it; the current §4 prompt does not elicit a confidence number so
* this field is reserved for future calibration work. */
judge_confidence?: number;
/** When the run used `judgeEnsemble`, the per-judge verdicts with
* their own rationales + failure_modes. `judge_verdict` above still
* holds the majority. A single-judge run leaves this `undefined`. */
judge_ensemble?: JudgeEnsembleEntry[];
// ── H-AUDIT-1 reasoning_content extension (Sprint 11 Task A2, 2026-04-22) ──
/**
* Captured chain-of-thought when thinking=on. Per design doc §2.2 +
* ratification §Q4: persisted in the SAME JSONL row under the same
* `turnId`, so reconstruction from a single turnId yields the full
* turn graph including reasoning. Populated by the runner from
* `LlmCallResult.reasoningContent`.
*
* HARD EXCLUSION rules (design doc §2.4): NEVER passed to judges, NEVER
* written to frames/memory/KG/UI payloads, NEVER exported in summary
* briefs. Visibility is JSONL-read-only; use `readJsonl(path, {
* includeReasoning: false })` to prune on the consumer side.
*/
reasoning_content?: string;
/**
* Character count of `reasoning_content`. The canonical observability
* field for aggregation — `metrics.ts` computes sum / p50 / p95 here,
* NOT on the content itself. Ratification §Q4 affirms this as
* non-redundant (separate aggregation surface from the content storage).
*/
reasoning_content_chars?: number;
/**
* Which response-shape yielded the reasoning_content. Ratification §Q3
* parser precedence: `message.reasoning_content` (DashScope native),
* `message.reasoning` (OpenRouter unified), `body.reasoning_content`
* (legacy fallback), or `unknown` when thinking=on was requested but no
* field was present. `undefined` when thinking was off.
*/
reasoning_shape?: 'message.reasoning_content' | 'message.reasoning' | 'body.reasoning_content' | 'unknown';
// ── B2 fold-in (Sprint 11 Task B2, 2026-04-22) ────────────────────────────
/**
* Path the tie-break resolver took. `undefined` on single-judge runs and
* on 3-primary ensembles that reached majority without escalation.
* `'quadri-vendor'` when 1-1-1 was escalated to `xai/grok-4.20` and
* resolved. `'pm-escalation'` when even the fourth vote produced 1-1-1-1
* — accompanied by `judge_error: 'PM_ESCALATION'` so the aggregator
* treats the row as a skipped judge instance.
*/
tie_break_path?: 'none' | 'majority' | 'quadri-vendor' | 'pm-escalation';
/** Fourth-vendor slug when tie_break_path ∈ {quadri-vendor, pm-escalation}. */
tie_break_fourth_vendor?: string;
// ── Sprint 12 Task 1 Blocker #1 — dataset version (2026-04-22) ────────────
/**
* SHA-256 of the canonical dataset archive (or the static
* `synthetic-scaffold-v1` string for synthetic runs). Populated by the
* runner from `getDatasetVersion(dataset, dataRoot)` and attached per-row
* so any downstream replication check can resolve the exact input set
* from a single JSONL line.
*/
dataset_version?: string;
// ── Sprint 12 Task 1 Blocker #3 / B3 addendum § 4 piggy-back ─────────────
/**
* Pinning surface classification for the target model that produced this
* row. Populated by the runner from `config.model.pinning_surface`. A row
* without this field belongs to a pre-Sprint-12 artefact (backward-compat
* tolerated via optional).
*/
model_pinning_surface?: PinningSurface;
/**
* Non-null rationale when `model_pinning_surface === 'floating_alias'` or
* `revision_hash_pinned`. Null for `anthropic_immutable`. The exact text
* is copied verbatim from `config/models.json` so a grep across JSONL
* artefacts recovers the full carve-out set per B3 addendum § 5.
*/
model_pinning_carve_out_reason?: string | null;
/**
* Provider-exposed revision hash (e.g., OpenRouter `revision_id`, vLLM
* `model_hash`) when available. Null when the provider does not surface
* one — floating-alias runs set this to null. Future hook: Session 3+
* wires provider-specific extraction where applicable.
*/
model_revision_hash?: string | null;
// ── Sprint 12 Task 2 §2.1 A3 namespace split (LOCKED 2026-04-23) ─────────
/**
* A3 LOCK § 6 failure taxonomy code (8-value: null | F1..F6 | F_other)
* per `decisions/2026-04-23-jsonl-record-taxonomy-split-locked.md`.
*
* On A3 pipeline writes this column is populated for every judged row
* (default `null` for correct verdicts). Pre-Sprint-12 JSONL artefacts
* (C2 stage 1, B1/B2/B3 smoke) do NOT carry this field — consumers that
* handle both arcs treat `undefined` as "pre-A3 row, use Sprint 9
* `judge_failure_mode` instead".
*
* Authoritative column for all A3 exit criteria (Task 2 brief §6
* criterion +12) and downstream grep (`jq '.a3_failure_code'`). Sprint 9
* `judge_failure_mode` above is retained as a read-only legacy surface.
*/
a3_failure_code?: FailureCode;
/**
* A3 rationale. MUST be a non-empty ≥10-token string when
* `a3_failure_code === 'F_other'` (enforced by
* `failure-taxonomy/validator.ts`). `null` for correct verdicts and
* F1..F6 codes where rationale is optional.
*/
a3_rationale?: string | null;
}
/** Summary shape emitted at the end of a run — written alongside the JSONL. */
export interface AggregateSummary {
run: {
kind: 'cell' | 'control';
name: string;
dataset: string;
model: string;
seed: number;
startedAt: string;
finishedAt: string;
};
counts: {
total: number;
completed: number;
failed: number;
budgetStoppedAt: number | null;
};
metrics: {
meanAccuracy: number;
p50LatencyMs: number;
p95LatencyMs: number;
totalUsd: number;
meanUsdPerQuery: number;
};
failureModes: Record<string, number>;
/**
* Sprint 11 Task A2 (2026-04-22): reasoning_content aggregates when at
* least one record in the run carried a populated `reasoning_content`.
* Always character counts only — the content itself lives in JSONL only
* per design doc §2.4 exclusion rule.
* `undefined` when no records in the run had reasoning (thinking=off
* runs), so consumers can easily distinguish "no data" from "zero chars".
*/
reasoningContent?: {
count: number; // records with non-empty reasoning_content
sumChars: number;
p50Chars: number;
p95Chars: number;
shapeDistribution: Record<string, number>; // e.g. { 'message.reasoning': 200, 'unknown': 2 }
};
/**
* Sprint 12 Task 2 §2.1 (LOCKED 2026-04-23): A3 LOCK § 6 failure
* distribution computed from the `a3_failure_code` + `a3_rationale`
* columns of every judged row. `undefined` when the run contained zero
* A3-namespace rows (pre-Sprint-12 run or judge disabled).
*
* Authoritative source for Task 2 brief §6 exit criterion +12 grep
* (`jq '.a3_failure_code' *.jsonl | sort | uniq -c` ↔ `counts`) and for
* A3 LOCK § 6 F_other ≥10% review-flag tripwire.
*/
failure_distribution?: FailureDistribution;
}