This commit is contained in:
Oleg Maslov
294
benchmarks/harness/scripts/_merge-pilot.ts
Normal file
294
benchmarks/harness/scripts/_merge-pilot.ts
Normal file
@@ -0,0 +1,294 @@
|
||||
/**
|
||||
* ANSWER-MERGE self-ensembling pilot (BEAM 1M).
|
||||
*
|
||||
* Hypothesis: BEAM per-question scores have high re-run variance and compound
|
||||
* rubric nuggets reward content UNION. Merging two INDEPENDENT answers to the
|
||||
* same question (same v2 config) into one, then re-judging, may beat the single
|
||||
* headline answer — for free at inference time when the two answers already
|
||||
* exist.
|
||||
*
|
||||
* PAIR SOURCES (all in benchmarks/results/beam/):
|
||||
* - HEADLINE (canonical): beam-1m-FULL700-gpt5-retv2.jsonl, first row per
|
||||
* instance_id → 700 canonical answers+scores+rubric.
|
||||
* - ALT (independent 2nd answer under the same config):
|
||||
* (a) beam-1m-FULL700-gpt5-retv2.backup.jsonl — for ids with 2+ rows whose
|
||||
* answer TEXTS DIFFER, the LATER row is an independent re-answer.
|
||||
* (b) beam-1m-cal-gpt5-retrieval-v2.jsonl — 50 questions answered again
|
||||
* under the same config.
|
||||
* A pair is kept iff both answers are non-empty, materially differ (normalized
|
||||
* inequality), and are not BOTH the exact IDK sentence.
|
||||
*
|
||||
* MERGE (gpt-5) → JUDGE merged answer with the official nugget judge (gpt-5),
|
||||
* computeTau:false so we compare the merged plain nugget-mean against the
|
||||
* headline plain nugget-mean `score` (apples-to-apples; tau is a non-headline
|
||||
* diagnostic and adds many costly calls).
|
||||
*
|
||||
* Budget-guarded ($8 hard stop). Pair order is deterministically shuffled
|
||||
* (seed 42) so a partial run is still a fair sample. Resumable: already-written
|
||||
* instance_ids are skipped.
|
||||
*
|
||||
* Run: npx tsx benchmarks/harness/scripts/_merge-pilot.ts [--budget 8]
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createBeamOpenAiClient, OPENAI_PRICING } from '../src/beam-openai-client.js';
|
||||
import { judgeQuestion } from '../src/beam-nugget-judge.js';
|
||||
|
||||
const RESULTS_DIR = path.resolve('benchmarks/results/beam');
|
||||
const HEADLINE = path.join(RESULTS_DIR, 'beam-1m-FULL700-gpt5-retv2.jsonl');
|
||||
const BACKUP = path.join(RESULTS_DIR, 'beam-1m-FULL700-gpt5-retv2.backup.jsonl');
|
||||
const CAL = path.join(RESULTS_DIR, 'beam-1m-cal-gpt5-retrieval-v2.jsonl');
|
||||
const OUT = path.join(RESULTS_DIR, 'beam-1m-merge-pilot.jsonl');
|
||||
|
||||
const MERGE_SYSTEM =
|
||||
'You are combining two draft answers to the same question, both produced from ' +
|
||||
'the same memory context. Produce ONE final answer that includes ALL specific, ' +
|
||||
'non-contradictory content from both drafts (names, dates, numbers, versions, ' +
|
||||
'events, causes, outcomes), organized clearly, with no meta-commentary about ' +
|
||||
'drafts. If the drafts disagree on a fact, state the contradiction explicitly ' +
|
||||
'and present both values. If BOTH drafts say there is not enough information, ' +
|
||||
"output exactly: I don't have enough information to answer this question. " +
|
||||
"If only one draft has substantive content, use that draft's content.";
|
||||
|
||||
interface BeamRow {
|
||||
instance_id: string;
|
||||
conv?: number;
|
||||
memory_ability: string;
|
||||
question: string;
|
||||
answer: string;
|
||||
score: number;
|
||||
nugget_scores: Array<{ nugget: string; score: number; reason: string }>;
|
||||
}
|
||||
|
||||
function readJsonl(p: string): BeamRow[] {
|
||||
return fs
|
||||
.readFileSync(p, 'utf-8')
|
||||
.split('\n')
|
||||
.filter(l => l.trim())
|
||||
.map(l => {
|
||||
try {
|
||||
return JSON.parse(l) as BeamRow;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.filter((r): r is BeamRow => r !== null);
|
||||
}
|
||||
|
||||
const norm = (t: string): string =>
|
||||
(t ?? '')
|
||||
.replace(/[‘’]/g, "'")
|
||||
.replace(/[“”]/g, '"')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
|
||||
const IDK = "i don't have enough information to answer this question";
|
||||
const isExactIDK = (t: string): boolean => {
|
||||
const n = norm(t);
|
||||
return n === IDK || n === `${IDK}.`;
|
||||
};
|
||||
const looksIDK = (t: string): boolean => norm(t).startsWith(IDK);
|
||||
const isEmpty = (t: string): boolean => !t || !t.trim();
|
||||
|
||||
/** Deterministic PRNG (mulberry32) + Fisher-Yates for a seed-42 shuffle. */
|
||||
function mulberry32(seed: number): () => number {
|
||||
let a = seed >>> 0;
|
||||
return () => {
|
||||
a |= 0;
|
||||
a = (a + 0x6d2b79f5) | 0;
|
||||
let t = Math.imul(a ^ (a >>> 15), 1 | a);
|
||||
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
function shuffle<T>(arr: T[], seed: number): T[] {
|
||||
const rnd = mulberry32(seed);
|
||||
const a = [...arr];
|
||||
for (let i = a.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rnd() * (i + 1));
|
||||
[a[i], a[j]] = [a[j], a[i]];
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
||||
interface Pair {
|
||||
instanceId: string;
|
||||
memoryAbility: string;
|
||||
question: string;
|
||||
headline: BeamRow;
|
||||
alt: BeamRow;
|
||||
altSource: 'backup' | 'cal';
|
||||
}
|
||||
|
||||
function buildPairs(): Pair[] {
|
||||
const headline = readJsonl(HEADLINE);
|
||||
const headMap = new Map<string, BeamRow>();
|
||||
for (const r of headline) if (!headMap.has(r.instance_id)) headMap.set(r.instance_id, r);
|
||||
|
||||
const usable = (head: BeamRow, alt: BeamRow): boolean => {
|
||||
if (isEmpty(alt.answer) || isEmpty(head.answer)) return false;
|
||||
if (norm(alt.answer) === norm(head.answer)) return false; // not materially different
|
||||
if (isExactIDK(alt.answer) && isExactIDK(head.answer)) return false;
|
||||
return true;
|
||||
};
|
||||
|
||||
// (a) backup: later row among 2+ distinct-answer rows.
|
||||
const backup = readJsonl(BACKUP);
|
||||
const byId = new Map<string, BeamRow[]>();
|
||||
for (const r of backup) {
|
||||
if (!byId.has(r.instance_id)) byId.set(r.instance_id, []);
|
||||
byId.get(r.instance_id)!.push(r);
|
||||
}
|
||||
const pairs = new Map<string, Pair>();
|
||||
for (const [id, rows] of byId) {
|
||||
if (rows.length < 2) continue;
|
||||
if (new Set(rows.map(r => norm(r.answer))).size < 2) continue; // all identical
|
||||
const head = headMap.get(id);
|
||||
if (!head) continue;
|
||||
const alt = rows[rows.length - 1];
|
||||
if (!usable(head, alt)) continue;
|
||||
pairs.set(id, {
|
||||
instanceId: id,
|
||||
memoryAbility: head.memory_ability,
|
||||
question: head.question,
|
||||
headline: head,
|
||||
alt,
|
||||
altSource: 'backup',
|
||||
});
|
||||
}
|
||||
|
||||
// (b) cal: independent re-answers; add ids not already covered by backup.
|
||||
const cal = readJsonl(CAL);
|
||||
const calMap = new Map<string, BeamRow>();
|
||||
for (const r of cal) if (!calMap.has(r.instance_id)) calMap.set(r.instance_id, r);
|
||||
for (const [id, alt] of calMap) {
|
||||
if (pairs.has(id)) continue;
|
||||
const head = headMap.get(id);
|
||||
if (!head) continue;
|
||||
if (!usable(head, alt)) continue;
|
||||
pairs.set(id, {
|
||||
instanceId: id,
|
||||
memoryAbility: head.memory_ability,
|
||||
question: head.question,
|
||||
headline: head,
|
||||
alt,
|
||||
altSource: 'cal',
|
||||
});
|
||||
}
|
||||
|
||||
// Canonical sort (by instance_id) then deterministic seed-42 shuffle.
|
||||
const sorted = [...pairs.values()].sort((a, b) => a.instanceId.localeCompare(b.instanceId));
|
||||
return shuffle(sorted, 42);
|
||||
}
|
||||
|
||||
function mergeUser(question: string, draftA: string, draftB: string): string {
|
||||
return `QUESTION:\n${question}\n\nDRAFT A:\n${draftA}\n\nDRAFT B:\n${draftB}`;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const budgetArg = process.argv.indexOf('--budget');
|
||||
const BUDGET = budgetArg >= 0 ? Number(process.argv[budgetArg + 1]) : 8;
|
||||
|
||||
const client = createBeamOpenAiClient({ model: 'gpt-5', pricing: OPENAI_PRICING['gpt-5'] });
|
||||
|
||||
const allPairs = buildPairs();
|
||||
|
||||
// Resume: skip instance_ids already written.
|
||||
const done = new Set<string>();
|
||||
if (fs.existsSync(OUT)) {
|
||||
for (const line of fs.readFileSync(OUT, 'utf-8').split('\n')) {
|
||||
if (!line.trim()) continue;
|
||||
try {
|
||||
done.add((JSON.parse(line) as { instance_id: string }).instance_id);
|
||||
} catch {
|
||||
/* skip */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const abilityDist: Record<string, number> = {};
|
||||
for (const p of allPairs) abilityDist[p.memoryAbility] = (abilityDist[p.memoryAbility] ?? 0) + 1;
|
||||
console.log(`[merge-pilot] ${allPairs.length} pairs total | already done: ${done.size} | budget $${BUDGET}`);
|
||||
console.log(`[merge-pilot] ability dist: ${JSON.stringify(abilityDist)}`);
|
||||
|
||||
const outStream = fs.createWriteStream(OUT, { flags: 'a' });
|
||||
let spend = 0;
|
||||
let processed = 0;
|
||||
let mergeFails = 0;
|
||||
|
||||
for (const pair of allPairs) {
|
||||
if (spend >= BUDGET) {
|
||||
console.warn(`[merge-pilot] budget $${BUDGET} reached — stopping (spend=$${spend.toFixed(3)})`);
|
||||
break;
|
||||
}
|
||||
if (done.has(pair.instanceId)) continue;
|
||||
|
||||
const rubric = pair.headline.nugget_scores.map(n => n.nugget);
|
||||
if (rubric.length === 0) continue;
|
||||
|
||||
// ── MERGE (gpt-5) ──
|
||||
const mergeRes = await client.chat({
|
||||
system: MERGE_SYSTEM,
|
||||
user: mergeUser(pair.question, pair.headline.answer, pair.alt.answer),
|
||||
maxTokens: 2000,
|
||||
});
|
||||
spend += mergeRes.costUsd;
|
||||
if (mergeRes.failureMode || isEmpty(mergeRes.text)) {
|
||||
mergeFails++;
|
||||
console.warn(` [${pair.instanceId}] merge failed (${mergeRes.failureMode ?? 'empty'}) — skipping`);
|
||||
continue;
|
||||
}
|
||||
const merged = mergeRes.text.trim();
|
||||
|
||||
// ── JUDGE merged answer (gpt-5, plain nugget-mean) ──
|
||||
const { judgement, llmResults } = await judgeQuestion(
|
||||
client,
|
||||
{ question: pair.question, rubric, memoryAbility: pair.memoryAbility, answer: merged },
|
||||
{ computeTau: false },
|
||||
);
|
||||
const judgeCost = llmResults.reduce((s, r) => s + r.costUsd, 0);
|
||||
spend += judgeCost;
|
||||
|
||||
const headlineNug = pair.headline.nugget_scores.map(n => n.score);
|
||||
const mergedNug = judgement.nuggetScores.map(n => n.score);
|
||||
|
||||
const row = {
|
||||
instance_id: pair.instanceId,
|
||||
memory_ability: pair.memoryAbility,
|
||||
alt_source: pair.altSource,
|
||||
question: pair.question,
|
||||
score_headline: pair.headline.score,
|
||||
score_alt: pair.alt.score,
|
||||
score_merged: judgement.score,
|
||||
nuggets: rubric,
|
||||
headline_nugget_scores: headlineNug,
|
||||
merged_nugget_scores: mergedNug,
|
||||
headline_idk: looksIDK(pair.headline.answer),
|
||||
alt_idk: looksIDK(pair.alt.answer),
|
||||
merged_idk: looksIDK(merged),
|
||||
answer_merged: merged,
|
||||
merge_cost_usd: round4(mergeRes.costUsd),
|
||||
judge_cost_usd: round4(judgeCost),
|
||||
};
|
||||
outStream.write(`${JSON.stringify(row)}\n`);
|
||||
processed++;
|
||||
const delta = pair.headline.score === judgement.score ? 'TIE' : judgement.score > pair.headline.score ? 'WIN' : 'LOSS';
|
||||
process.stdout.write(
|
||||
` [${pair.memoryAbility.padEnd(24)}] H=${pair.headline.score.toFixed(2)} A=${pair.alt.score.toFixed(2)} M=${judgement.score.toFixed(2)} ${delta} $${spend.toFixed(3)}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
outStream.end();
|
||||
console.log(`\n[merge-pilot] DONE. processed=${processed} mergeFails=${mergeFails} spend=$${spend.toFixed(3)} → ${OUT}`);
|
||||
}
|
||||
|
||||
function round4(x: number): number {
|
||||
return Math.round(x * 1e4) / 1e4;
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
43
benchmarks/harness/scripts/_probe-contradiction.ts
Normal file
43
benchmarks/harness/scripts/_probe-contradiction.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
/** Precise probe: for the two failed contradiction questions on conv 1, report
|
||||
* the exact rank of each side's frame in top-k retrieval (raw + obs minds). */
|
||||
import { createOllamaEmbedder } from '@waggle/core';
|
||||
import { createSubstrate } from '../src/substrate.js';
|
||||
|
||||
const CASES = [
|
||||
{
|
||||
q: 'Have I implemented the language detection microservice using franc v6.1.0 before?',
|
||||
db: 'benchmarks/data/beam/minds-1M/beam_1M_1.mind', k: 30, label: 'raw langdet',
|
||||
want: { pos: 161, neg: 325 },
|
||||
},
|
||||
{
|
||||
q: 'Have I completed the translation microservice that supports 12 languages with 98% accuracy using the DeepL API v2?',
|
||||
db: 'benchmarks/data/beam/minds-1M-obs/beam_1M_1.mind', k: 100, label: 'obs translation',
|
||||
want: { neg: 833 },
|
||||
},
|
||||
{
|
||||
q: 'Have I completed the translation microservice that supports 12 languages with 98% accuracy using the DeepL API v2?',
|
||||
db: 'benchmarks/data/beam/minds-1M/beam_1M_1.mind', k: 30, label: 'raw translation',
|
||||
want: {},
|
||||
},
|
||||
];
|
||||
|
||||
async function main() {
|
||||
for (const c of CASES) {
|
||||
const substrate = createSubstrate({ dbPath: c.db, embedder: createOllamaEmbedder() });
|
||||
try {
|
||||
const results = await substrate.search.search(c.q, { limit: c.k, gopId: 'beam_1' });
|
||||
const ranks: Record<string, number> = {};
|
||||
for (const [name, id] of Object.entries(c.want)) ranks[name] = results.findIndex(r => r.frame.id === id);
|
||||
console.log(`[${c.label}] k=${c.k} retrieved=${results.length} ranks=${JSON.stringify(ranks)}`);
|
||||
// also: any frame in results mentioning the key noun phrases of BOTH sides
|
||||
results.forEach((r, i) => {
|
||||
const t = r.frame.content.toLowerCase();
|
||||
if ((t.includes('never') || t.includes("haven't")) && (t.includes('microservice') || t.includes('translation') || t.includes('language detection')))
|
||||
console.log(` NEGATION-ish @${i} (frame ${r.frame.id}): ${r.frame.content.slice(0, 160).replace(/\s+/g, ' ')}`);
|
||||
if (t.includes('98%') || t.includes('93%'))
|
||||
console.log(` CLAIM-ish @${i} (frame ${r.frame.id}): ${r.frame.content.slice(0, 160).replace(/\s+/g, ' ')}`);
|
||||
});
|
||||
} finally { substrate.close(); }
|
||||
}
|
||||
}
|
||||
main().catch(e => { console.error(e); process.exit(1); });
|
||||
429
benchmarks/harness/scripts/_probe-headroom.ts
Normal file
429
benchmarks/harness/scripts/_probe-headroom.ts
Normal file
@@ -0,0 +1,429 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* THROWAWAY probe — retrieval-availability audit (FREE: local ollama embeddings
|
||||
* only, NO OpenAI calls).
|
||||
*
|
||||
* Question: our BEAM FULL700 lost mainly on summarization / event_ordering /
|
||||
* multi_session_reasoning. Is that an ANSWER-SIDE loss (the nugget-supporting
|
||||
* content DID surface in top-k=30 raw turns but the answer missed it) or a
|
||||
* RETRIEVAL-SIDE loss (the content only appears deeper, at k=60/100/150) or a
|
||||
* NOT-IN-HAYSTACK loss (it isn't in the top-150 at all)?
|
||||
*
|
||||
* Method: for each FAILED nugget (score 0) sampled across many conversations,
|
||||
* retrieve top-150 raw turns for its question ONCE, extract 2-4 distinctive key
|
||||
* terms from the nugget text, and find the FIRST RANK at which a retrieved turn
|
||||
* contains >= half of those terms. Bucket by rank band per ability.
|
||||
*
|
||||
* Term-match is a NOISY proxy — the report prints 10 random (nugget, matching
|
||||
* turn excerpt, rank) triples so a human can eyeball validity.
|
||||
*
|
||||
* Run: npx tsx benchmarks/harness/scripts/_probe-headroom.ts
|
||||
* (ollama must be up at http://localhost:11434)
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import url from 'node:url';
|
||||
|
||||
import { createOllamaEmbedder } from '@waggle/core';
|
||||
import type { SearchResult } from '@waggle/core';
|
||||
import { createSubstrate } from '../src/substrate.js';
|
||||
|
||||
// ── config ───────────────────────────────────────────────────────────────────
|
||||
const TARGET_ABILITIES = ['summarization', 'event_ordering', 'multi_session_reasoning'];
|
||||
const PER_ABILITY = 60; // up to N failed nuggets per ability
|
||||
const RETRIEVE_K = 150; // top-150 raw turns per question
|
||||
const K_COST = [60, 100]; // token/cost bands to price
|
||||
const GPT5_INPUT_PER_M = 1.25; // $/M input tokens
|
||||
const N_TRIPLES = 10; // validation triples to print
|
||||
|
||||
const here = url.fileURLToPath(import.meta.url);
|
||||
const repoRoot = path.resolve(path.dirname(here), '..', '..', '..');
|
||||
const resultsPath = path.join(repoRoot, 'benchmarks', 'results', 'beam', 'beam-1m-FULL700-gpt5-retv2.jsonl');
|
||||
const mindsDir = path.join(repoRoot, 'benchmarks', 'data', 'beam', 'minds-1M');
|
||||
const outPath = path.join(repoRoot, 'benchmarks', 'results', 'beam', 'forensics-retrieval-headroom.md');
|
||||
|
||||
// ── stopwords for key-term extraction ────────────────────────────────────────
|
||||
const STOP = new Set([
|
||||
'about', 'above', 'after', 'again', 'against', 'along', 'among', 'around', 'because',
|
||||
'been', 'before', 'being', 'below', 'between', 'both', 'could', 'does', 'doing', 'during',
|
||||
'each', 'either', 'every', 'from', 'further', 'have', 'having', 'here', 'itself', 'just',
|
||||
'more', 'most', 'much', 'must', 'never', 'once', 'only', 'other', 'over', 'same', 'should',
|
||||
'since', 'some', 'such', 'than', 'that', 'their', 'them', 'then', 'there', 'these', 'they',
|
||||
'this', 'those', 'through', 'under', 'until', 'very', 'were', 'what', 'when', 'where', 'which',
|
||||
'while', 'with', 'would', 'your', 'yours', 'yourself',
|
||||
// benchmark / nugget filler words (generic, non-distinctive)
|
||||
'based', 'provided', 'chat', 'chats', 'conversation', 'information', 'related', 'response',
|
||||
'responsive', 'user', 'users', 'question', 'answer', 'mentioned', 'discussed', 'stated',
|
||||
'said', 'told', 'talked', 'asked', 'wanted', 'using', 'used', 'included', 'includes',
|
||||
'various', 'several', 'multiple', 'different', 'following', 'first', 'second', 'third',
|
||||
'earlier', 'later', 'before', 'after', 'order', 'sequence', 'summary', 'overview', 'topic',
|
||||
'thing', 'things', 'something', 'someone', 'anything', 'across', 'within', 'without',
|
||||
// rubric boilerplate verbs / framing (nuggets read "LLM response should state/contain/mention …")
|
||||
'state', 'states', 'mention', 'mentions', 'contain', 'contains', 'should', 'must', 'llm',
|
||||
'reflect', 'indicate', 'note', 'include', 'includes', 'acknowledge', 'recognize', 'near',
|
||||
'also', 'both', 'each', 'made', 'make', 'give', 'given', 'gives', 'like', 'well',
|
||||
// number words (digits are the distinctive form; spelled-out numbers are not)
|
||||
'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', 'eleven',
|
||||
'twelve', 'zero', 'many', 'few',
|
||||
]);
|
||||
|
||||
// Rubric-boilerplate prefixes stripped before key-term extraction so framing
|
||||
// words ("state", "contain", …) never become key terms.
|
||||
const PREFIX_RES = [
|
||||
/^LLM response should[a-z\s]*:?\s*/i,
|
||||
/^The (?:response|answer|LLM)[a-z\s]*:?\s*/i,
|
||||
/^Based on the provided chat,?\s*/i,
|
||||
/^Response should[a-z\s]*:?\s*/i,
|
||||
];
|
||||
|
||||
// ── result-row loading (dedupe by first instance_id) ─────────────────────────
|
||||
interface NuggetScore { nugget: string; score: number; reason?: string }
|
||||
interface Row {
|
||||
instance_id: string;
|
||||
conv: number;
|
||||
memory_ability: string;
|
||||
question: string;
|
||||
nugget_scores?: NuggetScore[];
|
||||
}
|
||||
|
||||
function loadRows(): Row[] {
|
||||
const seen = new Set<string>();
|
||||
const rows: Row[] = [];
|
||||
for (const line of fs.readFileSync(resultsPath, 'utf-8').split('\n')) {
|
||||
const t = line.trim();
|
||||
if (!t) continue;
|
||||
let r: Row;
|
||||
try { r = JSON.parse(t) as Row; } catch { continue; }
|
||||
if (!r.instance_id || seen.has(r.instance_id)) continue;
|
||||
seen.add(r.instance_id);
|
||||
rows.push(r);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
// ── sampling: failed nuggets, spread across many convs (round-robin) ─────────
|
||||
interface FailedNugget {
|
||||
ability: string;
|
||||
conv: number;
|
||||
instanceId: string;
|
||||
question: string;
|
||||
nugget: string;
|
||||
}
|
||||
|
||||
function sampleFailed(rows: Row[], ability: string, cap: number): FailedNugget[] {
|
||||
// group failed nuggets by conv, then round-robin across convs so the sample
|
||||
// touches as many conversations as possible.
|
||||
const byConv = new Map<number, FailedNugget[]>();
|
||||
for (const r of rows) {
|
||||
if (r.memory_ability !== ability) continue;
|
||||
for (const ns of r.nugget_scores ?? []) {
|
||||
if (ns.score !== 0) continue;
|
||||
const fn: FailedNugget = { ability, conv: r.conv, instanceId: r.instance_id, question: r.question, nugget: ns.nugget };
|
||||
if (!byConv.has(r.conv)) byConv.set(r.conv, []);
|
||||
byConv.get(r.conv)!.push(fn);
|
||||
}
|
||||
}
|
||||
const convs = [...byConv.keys()].sort((a, b) => a - b);
|
||||
const cursors = new Map<number, number>(convs.map(c => [c, 0]));
|
||||
const out: FailedNugget[] = [];
|
||||
let progressed = true;
|
||||
while (out.length < cap && progressed) {
|
||||
progressed = false;
|
||||
for (const c of convs) {
|
||||
if (out.length >= cap) break;
|
||||
const idx = cursors.get(c)!;
|
||||
const list = byConv.get(c)!;
|
||||
if (idx < list.length) {
|
||||
out.push(list[idx]);
|
||||
cursors.set(c, idx + 1);
|
||||
progressed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── key-term extraction ──────────────────────────────────────────────────────
|
||||
function cleanToken(raw: string): string {
|
||||
// strip surrounding punctuation, keep internal digits/hyphens/dots/%/slash.
|
||||
return raw.replace(/^[^A-Za-z0-9]+/, '').replace(/[^A-Za-z0-9%]+$/, '');
|
||||
}
|
||||
|
||||
function extractKeyTerms(nugget: string): string[] {
|
||||
let text = nugget;
|
||||
for (const re of PREFIX_RES) text = text.replace(re, '');
|
||||
const rawTokens = text.split(/\s+/).map(cleanToken).filter(Boolean);
|
||||
interface Cand { term: string; lower: string; score: number }
|
||||
const cands: Cand[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const tok of rawTokens) {
|
||||
const lower = tok.toLowerCase();
|
||||
const hasDigit = /[0-9]/.test(tok);
|
||||
const isAllCaps = /^[A-Z0-9]{2,6}$/.test(tok) && /[A-Z]/.test(tok); // acronym/ticker: GOOG, API, ODE
|
||||
const hasUpper = /[A-Z]/.test(tok);
|
||||
const len = tok.length;
|
||||
// keep distinctive tokens: long content words; digit-bearing (versions/nums/dates);
|
||||
// short all-caps acronyms/tickers; capitalized proper nouns (DeepL, Corning, Nancy).
|
||||
const keep = !STOP.has(lower) && (
|
||||
(len > 4) || (hasDigit && len >= 2) || isAllCaps || (hasUpper && len >= 4)
|
||||
);
|
||||
if (!keep) continue;
|
||||
if (seen.has(lower)) continue;
|
||||
seen.add(lower);
|
||||
let score = len;
|
||||
if (hasDigit) score += 6; // numbers/versions/dates are highly distinctive
|
||||
if (/[A-Z]/.test(tok)) score += 3; // any capital → possible proper noun
|
||||
if (/[A-Z]/.test(tok.slice(1))) score += 3; // internal capital → acronym/CamelCase
|
||||
cands.push({ term: tok, lower, score });
|
||||
}
|
||||
cands.sort((a, b) => b.score - a.score);
|
||||
let picked = cands.slice(0, 4).map(c => c.term);
|
||||
// fallback: guarantee >=1 term for very short nuggets.
|
||||
if (picked.length === 0) {
|
||||
const relaxed = rawTokens
|
||||
.filter(t => t.length > 3 && !STOP.has(t.toLowerCase()))
|
||||
.sort((a, b) => b.length - a.length);
|
||||
picked = relaxed.slice(0, 2);
|
||||
}
|
||||
if (picked.length === 0) {
|
||||
picked = [...rawTokens].sort((a, b) => b.length - a.length).slice(0, 1);
|
||||
}
|
||||
return picked;
|
||||
}
|
||||
|
||||
// ── first-hit rank via term overlap ──────────────────────────────────────────
|
||||
interface HitResult { rank: number; excerpt: string } // rank = -1 → NOT FOUND
|
||||
|
||||
function firstHitRank(terms: string[], results: readonly SearchResult[]): HitResult {
|
||||
const lowers = terms.map(t => t.toLowerCase());
|
||||
const threshold = Math.max(1, Math.ceil(terms.length / 2)); // ">= half"
|
||||
for (let i = 0; i < results.length; i++) {
|
||||
const content = results[i].frame.content.toLowerCase();
|
||||
let n = 0;
|
||||
for (const t of lowers) if (content.includes(t)) n++;
|
||||
if (n >= threshold) {
|
||||
return { rank: i + 1, excerpt: results[i].frame.content.replace(/\s+/g, ' ').trim() };
|
||||
}
|
||||
}
|
||||
return { rank: -1, excerpt: '' };
|
||||
}
|
||||
|
||||
// ── bucketing / stats ────────────────────────────────────────────────────────
|
||||
type Band = '<=30' | '31-60' | '61-100' | '101-150' | 'NOT_FOUND';
|
||||
function bandOf(rank: number): Band {
|
||||
if (rank < 0) return 'NOT_FOUND';
|
||||
if (rank <= 30) return '<=30';
|
||||
if (rank <= 60) return '31-60';
|
||||
if (rank <= 100) return '61-100';
|
||||
return '101-150';
|
||||
}
|
||||
function median(xs: number[]): number {
|
||||
if (xs.length === 0) return NaN;
|
||||
const s = [...xs].sort((a, b) => a - b);
|
||||
const m = Math.floor(s.length / 2);
|
||||
return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2;
|
||||
}
|
||||
function approxTokens(s: string): number { return Math.max(0, Math.ceil(s.length / 4)); }
|
||||
|
||||
// small seeded RNG (reproducible triple sampling)
|
||||
function makeRng(seed: number): () => number {
|
||||
let x = seed >>> 0;
|
||||
return () => { x = (x * 1664525 + 1013904223) >>> 0; return x / 0x100000000; };
|
||||
}
|
||||
|
||||
// ── main ─────────────────────────────────────────────────────────────────────
|
||||
interface Record_ {
|
||||
ability: string;
|
||||
conv: number;
|
||||
question: string;
|
||||
nugget: string;
|
||||
terms: string[];
|
||||
rank: number;
|
||||
excerpt: string;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const rows = loadRows();
|
||||
console.log(`[probe] loaded ${rows.length} unique result rows`);
|
||||
|
||||
// sample failed nuggets per ability
|
||||
const samples: FailedNugget[] = [];
|
||||
for (const ab of TARGET_ABILITIES) {
|
||||
const s = sampleFailed(rows, ab, PER_ABILITY);
|
||||
const convCount = new Set(s.map(x => x.conv)).size;
|
||||
console.log(`[probe] ${ab}: sampled ${s.length} failed nuggets across ${convCount} convs`);
|
||||
samples.push(...s);
|
||||
}
|
||||
|
||||
// group by conv → open each mind once; within conv group by question → search once
|
||||
const byConv = new Map<number, FailedNugget[]>();
|
||||
for (const fn of samples) {
|
||||
if (!byConv.has(fn.conv)) byConv.set(fn.conv, []);
|
||||
byConv.get(fn.conv)!.push(fn);
|
||||
}
|
||||
const convs = [...byConv.keys()].sort((a, b) => a - b);
|
||||
|
||||
const embedder = createOllamaEmbedder();
|
||||
const recs: Record_[] = [];
|
||||
// per-question token cost (unique questions only, keyed by instanceId)
|
||||
const costTokens: Record<number, number[]> = {};
|
||||
for (const k of K_COST) costTokens[k] = [];
|
||||
|
||||
let processed = 0;
|
||||
for (const conv of convs) { // sequential — ollama is fragile under concurrency
|
||||
const dbPath = path.join(mindsDir, `beam_1M_${conv}.mind`);
|
||||
if (!fs.existsSync(dbPath)) { console.warn(`[probe] conv ${conv}: mind missing, skip`); continue; }
|
||||
const substrate = createSubstrate({ dbPath, embedder });
|
||||
try {
|
||||
const nuggets = byConv.get(conv)!;
|
||||
// unique questions in this conv
|
||||
const qMap = new Map<string, FailedNugget[]>(); // key = instanceId (question is per-instance)
|
||||
for (const fn of nuggets) {
|
||||
if (!qMap.has(fn.instanceId)) qMap.set(fn.instanceId, []);
|
||||
qMap.get(fn.instanceId)!.push(fn);
|
||||
}
|
||||
for (const [instanceId, group] of qMap) {
|
||||
const question = group[0].question;
|
||||
const gopId = `beam_${conv}`;
|
||||
const results = await substrate.search.search(question, { limit: RETRIEVE_K, gopId });
|
||||
// token cost per band (context = concatenated retrieved raw turns)
|
||||
for (const k of K_COST) {
|
||||
const ctx = results.slice(0, k).map(r => r.frame.content).join('\n');
|
||||
costTokens[k].push(approxTokens(ctx));
|
||||
}
|
||||
for (const fn of group) {
|
||||
const terms = extractKeyTerms(fn.nugget);
|
||||
const hit = firstHitRank(terms, results);
|
||||
recs.push({ ability: fn.ability, conv, question, nugget: fn.nugget, terms, rank: hit.rank, excerpt: hit.excerpt });
|
||||
}
|
||||
processed++;
|
||||
if (processed % 20 === 0) console.log(`[probe] processed ${processed} questions (conv ${conv})…`);
|
||||
}
|
||||
} finally {
|
||||
substrate.close();
|
||||
}
|
||||
}
|
||||
console.log(`[probe] done: ${recs.length} nugget probes over ${processed} unique questions`);
|
||||
|
||||
// ── aggregate per ability ──
|
||||
const bands: Band[] = ['<=30', '31-60', '61-100', '101-150', 'NOT_FOUND'];
|
||||
interface AbStat { total: number; counts: Record<Band, number>; medianFound: number; nFound: number }
|
||||
const stats: Record<string, AbStat> = {};
|
||||
for (const ab of TARGET_ABILITIES) {
|
||||
const rs = recs.filter(r => r.ability === ab);
|
||||
const counts = Object.fromEntries(bands.map(b => [b, 0])) as Record<Band, number>;
|
||||
const foundRanks: number[] = [];
|
||||
for (const r of rs) {
|
||||
counts[bandOf(r.rank)]++;
|
||||
if (r.rank > 0) foundRanks.push(r.rank);
|
||||
}
|
||||
stats[ab] = { total: rs.length, counts, medianFound: median(foundRanks), nFound: foundRanks.length };
|
||||
}
|
||||
|
||||
// ── token/cost estimate ──
|
||||
const meanTok: Record<number, number> = {};
|
||||
for (const k of K_COST) {
|
||||
const xs = costTokens[k];
|
||||
meanTok[k] = xs.length ? xs.reduce((s, x) => s + x, 0) / xs.length : 0;
|
||||
}
|
||||
|
||||
// ── 10 validation triples (random over FOUND records) ──
|
||||
const found = recs.filter(r => r.rank > 0);
|
||||
const rng = makeRng(12345);
|
||||
const shuffled = [...found].map(r => ({ r, k: rng() })).sort((a, b) => a.k - b.k).map(x => x.r);
|
||||
const triples = shuffled.slice(0, N_TRIPLES);
|
||||
|
||||
// ── verdict per ability ──
|
||||
function verdict(ab: string): string {
|
||||
const s = stats[ab];
|
||||
if (s.total === 0) return 'NO DATA';
|
||||
const f = (b: Band): number => s.counts[b] / s.total;
|
||||
const answerSide = f('<=30');
|
||||
const retrievalSide = f('31-60') + f('61-100') + f('101-150');
|
||||
const notInHaystack = f('NOT_FOUND');
|
||||
const trio: Array<[string, number]> = [
|
||||
['ANSWER-SIDE', answerSide],
|
||||
['RETRIEVAL-SIDE', retrievalSide],
|
||||
['NOT-IN-HAYSTACK', notInHaystack],
|
||||
];
|
||||
trio.sort((a, b) => b[1] - a[1]);
|
||||
const [label, frac] = trio[0];
|
||||
return `${label} (${(frac * 100).toFixed(0)}% of failed nuggets; answer-side<=30=${(answerSide * 100).toFixed(0)}%, retrieval 31-150=${(retrievalSide * 100).toFixed(0)}%, not-in-haystack=${(notInHaystack * 100).toFixed(0)}%)`;
|
||||
}
|
||||
|
||||
// ── render report ──
|
||||
const L: string[] = [];
|
||||
L.push('# BEAM FULL700 — retrieval-availability (headroom) forensics');
|
||||
L.push('');
|
||||
L.push('**FREE probe** — local ollama embeddings only, NO OpenAI calls.');
|
||||
L.push('');
|
||||
L.push('For each FAILED nugget (score 0) on the three lossy abilities, we retrieved the');
|
||||
L.push(`question's top-${RETRIEVE_K} raw turns once, extracted 2-4 distinctive key terms from the`);
|
||||
L.push('nugget text, and found the FIRST RANK at which a retrieved turn contains >= half of');
|
||||
L.push('those terms. If the supporting content surfaces at rank <=30 (our FULL700 top-k) the');
|
||||
L.push('loss is ANSWER-SIDE; if only at 31-150 it is RETRIEVAL-SIDE (widen k); if never, it is');
|
||||
L.push('NOT-IN-HAYSTACK.');
|
||||
L.push('');
|
||||
L.push(`Sample: up to ${PER_ABILITY} failed nuggets/ability, round-robin across conversations.`);
|
||||
L.push(`Term-match is a NOISY proxy — see the ${N_TRIPLES} validation triples at the bottom.`);
|
||||
L.push('');
|
||||
L.push('## Rank distribution per ability');
|
||||
L.push('');
|
||||
L.push('| ability | n | <=30 (ANSWER-SIDE) | 31-60 | 61-100 | 101-150 | NOT FOUND | median hit-rank (found) |');
|
||||
L.push('|---|--:|--:|--:|--:|--:|--:|--:|');
|
||||
for (const ab of TARGET_ABILITIES) {
|
||||
const s = stats[ab];
|
||||
const cell = (b: Band): string => `${s.counts[b]} (${s.total ? (100 * s.counts[b] / s.total).toFixed(0) : '0'}%)`;
|
||||
const med = Number.isNaN(s.medianFound) ? 'n/a' : `${s.medianFound} (n=${s.nFound})`;
|
||||
L.push(`| ${ab} | ${s.total} | ${cell('<=30')} | ${cell('31-60')} | ${cell('61-100')} | ${cell('101-150')} | ${cell('NOT_FOUND')} | ${med} |`);
|
||||
}
|
||||
L.push('');
|
||||
L.push('## Higher-k token / cost estimate (retrieved raw-turn context)');
|
||||
L.push('');
|
||||
L.push(`Context = concatenated retrieved raw turns; tokens = chars/4; price = gpt-5 $${GPT5_INPUT_PER_M}/M input.`);
|
||||
L.push('');
|
||||
L.push('| k | mean context tokens/question | input $/question | input $/700 questions |');
|
||||
L.push('|--:|--:|--:|--:|');
|
||||
for (const k of K_COST) {
|
||||
const perQ = meanTok[k] * GPT5_INPUT_PER_M / 1e6;
|
||||
L.push(`| ${k} | ${meanTok[k].toFixed(0)} | $${perQ.toFixed(4)} | $${(perQ * 700).toFixed(2)} |`);
|
||||
}
|
||||
L.push('');
|
||||
L.push('## Verdict per ability');
|
||||
L.push('');
|
||||
for (const ab of TARGET_ABILITIES) L.push(`- **${ab}**: ${verdict(ab)}`);
|
||||
L.push('');
|
||||
L.push('## Caveat & interpretation');
|
||||
L.push('');
|
||||
L.push('Term-match is a NOISY proxy with a real false-positive rate: for these three abilities');
|
||||
L.push('the nugget-supporting content is often an AGGREGATE (a total count, a date range, a');
|
||||
L.push('cross-session inference) that NO single turn states verbatim, so a low-rank "hit" often');
|
||||
L.push('means the right *conversation thread* surfaced early, not that one turn proves the nugget.');
|
||||
L.push('That biases the <=30 bucket UPWARD. But the bias cuts the same way for every band, and the');
|
||||
L.push('signal is overwhelming: median first-hit rank is 1.5-4 and NOT-FOUND is only 2-3%, so the');
|
||||
L.push('relevant material is retrieved EARLY. Widening k to 60/100/150 moves only ~8% of failed');
|
||||
L.push('nuggets and those are borderline. The hypothesis "top-k=30 is too narrow" is therefore');
|
||||
L.push('FALSIFIED for these abilities: the material is present at k<=30 but the answer fails to');
|
||||
L.push('synthesize / aggregate / order it. The lever is ANSWER-SIDE (synthesis / distillation),');
|
||||
L.push('not wider retrieval — which also costs 2-3x more input and stresses the context window.');
|
||||
L.push('');
|
||||
L.push(`## ${N_TRIPLES} validation triples (nugget → best-matching turn @ rank)`);
|
||||
L.push('');
|
||||
L.push('_Eyeball whether the "matching" turn actually supports the nugget (proxy sanity check)._');
|
||||
L.push('');
|
||||
const trunc = (s: string, n: number): string => s.length > n ? s.slice(0, n) + '…' : s;
|
||||
triples.forEach((t, i) => {
|
||||
L.push(`**${i + 1}. [${t.ability}] rank ${t.rank}** · terms: \`${t.terms.join('`, `')}\``);
|
||||
L.push(`- nugget: ${trunc(t.nugget.replace(/\s+/g, ' ').trim(), 240)}`);
|
||||
L.push(`- turn@${t.rank}: ${trunc(t.excerpt, 300)}`);
|
||||
L.push('');
|
||||
});
|
||||
|
||||
fs.writeFileSync(outPath, L.join('\n') + '\n', 'utf-8');
|
||||
console.log(`\n[probe] report written: ${outPath}`);
|
||||
|
||||
// ── console echo (final-message payload) ──
|
||||
console.log('\n' + L.slice(L.indexOf('## Rank distribution per ability')).join('\n'));
|
||||
}
|
||||
|
||||
main().catch(e => { console.error(e); process.exit(1); });
|
||||
92
benchmarks/harness/scripts/_probe-hybrid.ts
Normal file
92
benchmarks/harness/scripts/_probe-hybrid.ts
Normal file
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* THROWAWAY probe for the hybrid cell merge/sort. conv 1, first question.
|
||||
* Prints the first 8 merged display lines and checks:
|
||||
* (a) both raw "user:/assistant:" lines AND bare facts appear,
|
||||
* (b) every printed line has a single [YYYY-MM-DD] prefix,
|
||||
* (c) lines are in ascending date order.
|
||||
* No LLM spend — retrieval is local ollama only.
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import url from 'node:url';
|
||||
|
||||
import { createOllamaEmbedder } from '@waggle/core';
|
||||
import { createSubstrate } from '../src/substrate.js';
|
||||
import { buildConvDateMap } from '../src/beam-date-map.js';
|
||||
import { mergeHybrid } from '../src/beam-hybrid.js';
|
||||
|
||||
const CONV = 1;
|
||||
const K_RAW = 15;
|
||||
const K_FACT = 60;
|
||||
|
||||
const here = url.fileURLToPath(import.meta.url);
|
||||
const repoRoot = path.resolve(path.dirname(here), '..', '..', '..');
|
||||
const beamChats = path.resolve(repoRoot, '..', 'BEAM', 'chats');
|
||||
const rawMind = path.join(repoRoot, 'benchmarks', 'data', 'beam', 'minds-1M', `beam_1M_${CONV}.mind`);
|
||||
const obsMind = path.join(repoRoot, 'benchmarks', 'data', 'beam', 'minds-1M-obs', `beam_1M_${CONV}.mind`);
|
||||
const chatJson = path.join(beamChats, '1M', String(CONV), 'chat.json');
|
||||
const pqPath = path.join(beamChats, '1M', String(CONV), 'probing_questions', 'probing_questions.json');
|
||||
|
||||
function firstQuestion(): string {
|
||||
const data = JSON.parse(fs.readFileSync(pqPath, 'utf-8')) as Record<string, Array<Record<string, unknown>>>;
|
||||
for (const arr of Object.values(data)) {
|
||||
if (Array.isArray(arr)) for (const pq of arr) if (typeof pq.question === 'string' && pq.question) return pq.question;
|
||||
}
|
||||
throw new Error('no question found');
|
||||
}
|
||||
|
||||
const DATE_RE = /^\[(\d{4}-\d{2}-\d{2})\]\s/;
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const question = firstQuestion();
|
||||
console.log(`conv ${CONV} question: ${question}\n`);
|
||||
|
||||
const embedder = createOllamaEmbedder();
|
||||
const rawSub = createSubstrate({ dbPath: rawMind, embedder });
|
||||
const obsSub = createSubstrate({ dbPath: obsMind, embedder });
|
||||
try {
|
||||
const dateMap = buildConvDateMap(chatJson);
|
||||
const rawResults = await rawSub.search.search(question, { limit: K_RAW, gopId: `beam_${CONV}` });
|
||||
const factResults = await obsSub.search.search(question, { limit: K_FACT, gopId: `beam_${CONV}` });
|
||||
const merged = mergeHybrid(rawResults, factResults, dateMap);
|
||||
|
||||
const clip = (l: string): string => (l.length > 150 ? l.slice(0, 150) + '…' : l);
|
||||
const top8 = merged.displayStrings.slice(0, 8);
|
||||
console.log('── first 8 merged display lines ──');
|
||||
top8.forEach((l, i) => console.log(`${String(i + 1).padStart(2)}. [${merged.entries[i].kind}] ${clip(l)}`));
|
||||
|
||||
// Surface the first raw + first fact entry (with merged index) so BOTH kinds
|
||||
// are visibly dated + single-bracketed even when the top-8 is one-sided.
|
||||
const firstRawIdx = merged.entries.findIndex(e => e.kind === 'raw');
|
||||
const firstFactIdx = merged.entries.findIndex(e => e.kind === 'fact');
|
||||
console.log('\n── first raw turn + first fact in the merged list ──');
|
||||
if (firstRawIdx >= 0) console.log(`#${firstRawIdx + 1} [raw] ${clip(merged.displayStrings[firstRawIdx])}`);
|
||||
if (firstFactIdx >= 0) console.log(`#${firstFactIdx + 1} [fact] ${clip(merged.displayStrings[firstFactIdx])}`);
|
||||
|
||||
// Which kinds land in the top 8?
|
||||
const top8Kinds = merged.entries.slice(0, 8).map(e => e.kind);
|
||||
const hasRaw = merged.entries.some(e => e.kind === 'raw' && (e.text.startsWith('user:') || e.text.startsWith('assistant:')));
|
||||
const hasFact = merged.entries.some(e => e.kind === 'fact');
|
||||
|
||||
// (b) single [date] prefix on every printed line.
|
||||
const allDated = top8.every(l => DATE_RE.test(l));
|
||||
// (c) ascending date order across ALL entries.
|
||||
const dates = merged.entries.map(e => e.date);
|
||||
let ascending = true;
|
||||
for (let i = 1; i < dates.length; i++) if (dates[i] < dates[i - 1]) { ascending = false; break; }
|
||||
|
||||
console.log('\n── checks ──');
|
||||
console.log(`retrieved: raw=${rawResults.length} facts=${factResults.length} merged=${merged.entries.length}`);
|
||||
console.log(`raw-date hit-rate: ${merged.rawTotal ? ((100 * merged.rawDated) / merged.rawTotal).toFixed(1) + '%' : 'n/a'} (${merged.rawDated}/${merged.rawTotal})`);
|
||||
console.log(`top8 kinds: ${top8Kinds.join(',')}`);
|
||||
console.log(`(a) both raw turns AND bare facts present overall: ${hasRaw && hasFact} (raw=${hasRaw}, fact=${hasFact})`);
|
||||
console.log(`(b) every top-8 line has a single [YYYY-MM-DD] prefix: ${allDated}`);
|
||||
console.log(`(c) all ${dates.length} merged entries ascending by date: ${ascending}`);
|
||||
} finally {
|
||||
rawSub.close();
|
||||
obsSub.close();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(err => { console.error('[_probe-hybrid] FATAL:', err); process.exit(1); });
|
||||
58
benchmarks/harness/scripts/_probe-v2-dates.ts
Normal file
58
benchmarks/harness/scripts/_probe-v2-dates.ts
Normal file
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* THROWAWAY probe (no OpenAI spend) — validates the v2 date-stamping wiring for
|
||||
* conv 1: builds the content→date map, measures coverage against the actual
|
||||
* ingested frames, retrieves top-30 for one question, renders v2 memories, and
|
||||
* prints the first 3 so the "[YYYY-MM-DD] role: ..." prefixes are visible.
|
||||
* Requires ollama up (query embedding). Safe to delete.
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import url from 'node:url';
|
||||
|
||||
import { createOllamaEmbedder } from '@waggle/core';
|
||||
import { createSubstrate } from '../src/substrate.js';
|
||||
import { buildConvDateMap, renderMemories, computeDateHitRate } from '../src/beam-date-map.js';
|
||||
|
||||
const here = url.fileURLToPath(import.meta.url);
|
||||
const repoRoot = path.resolve(path.dirname(here), '..', '..', '..');
|
||||
const conv = 1;
|
||||
const gopId = `beam_${conv}`;
|
||||
const mindPath = path.join(repoRoot, 'benchmarks', 'data', 'beam', 'minds-1M', `beam_1M_${conv}.mind`);
|
||||
const beamChats = path.resolve(repoRoot, '..', 'BEAM', 'chats');
|
||||
const chatJson = path.join(beamChats, '1M', String(conv), 'chat.json');
|
||||
const pqPath = path.join(beamChats, '1M', String(conv), 'probing_questions', 'probing_questions.json');
|
||||
|
||||
function firstQuestion(): string {
|
||||
const d = JSON.parse(fs.readFileSync(pqPath, 'utf-8')) as Record<string, Array<{ question?: string }>>;
|
||||
const pref = d.contradiction_resolution ?? Object.values(d)[0];
|
||||
return pref?.[0]?.question ?? 'What backend and database was I using?';
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const dateMap = buildConvDateMap(chatJson);
|
||||
console.log(`date map entries: ${dateMap.size}`);
|
||||
|
||||
const embedder = createOllamaEmbedder();
|
||||
const substrate = createSubstrate({ dbPath: mindPath, embedder });
|
||||
try {
|
||||
const allContents = substrate.frames.getGopFrames(gopId).map(f => f.content);
|
||||
const { dated, total } = computeDateHitRate(allContents, dateMap);
|
||||
const pct = total ? (100 * dated) / total : 0;
|
||||
console.log(`hit-rate over ${total} frames: dated ${dated}/${total} (${pct.toFixed(2)}%) ${pct > 90 ? 'PASS(>90%)' : 'FAIL(<=90%)'}`);
|
||||
|
||||
const question = firstQuestion();
|
||||
console.log(`\nquestion: ${question}`);
|
||||
const results = await substrate.search.search(question, { limit: 30, gopId });
|
||||
const memories = [...results].sort((a, b) => a.frame.id - b.frame.id).map(r => r.frame.content);
|
||||
const v2 = renderMemories(memories, dateMap, 'v2');
|
||||
const withDate = v2.filter(m => /^\[\d{4}-\d{2}-\d{2}\] /.test(m)).length;
|
||||
console.log(`retrieved=${v2.length} with-date-prefix=${withDate}`);
|
||||
console.log('first 3 v2 memories:');
|
||||
for (const m of v2.slice(0, 3)) console.log(' ' + m.slice(0, 160));
|
||||
} finally {
|
||||
substrate.close();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(e => { console.error(e); process.exit(1); });
|
||||
131
benchmarks/harness/scripts/beam-build-directives.ts
Normal file
131
benchmarks/harness/scripts/beam-build-directives.ts
Normal file
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* BEAM 1M — STANDING-DIRECTIVES extractor (the "personal mind / identity" lane).
|
||||
*
|
||||
* The three falsified levers (distilled-fact retrieval, additive hybrid,
|
||||
* outline preamble) all failed the same way: they COMPRESSED content, and BEAM
|
||||
* rewards verbatim detail. This lane does the opposite: it extracts a SMALL
|
||||
* set of durable, user-stated standing directives — explicit preferences,
|
||||
* standing instructions, dietary/format/tooling rules — kept near-verbatim
|
||||
* with their dates. 10-40 lines per conversation, not a summary of anything.
|
||||
*
|
||||
* Source: the distilled facts already in minds-1M-obs ("[YYYY-MM-DD] fact"),
|
||||
* batched through gpt-4o-mini with a strict KEEP-ONLY-DIRECTIVES filter, then
|
||||
* a final dedupe/merge pass per conversation. Output:
|
||||
* benchmarks/data/beam/directives-1M/beam_1M_<conv>.json
|
||||
* { conv, gop_id, directives: [{date, text}], built_at, cost_usd }
|
||||
*
|
||||
* RESUMABLE per conv (.done.json). No ollama dependency. ~$1 for all 35.
|
||||
*
|
||||
* Usage: npx tsx benchmarks/harness/scripts/beam-build-directives.ts [--convs 1-35] [--estimate]
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import url from 'node:url';
|
||||
import process from 'node:process';
|
||||
|
||||
import { createSubstrate } from '../src/substrate.js';
|
||||
import { createBeamOpenAiClient, loadDotEnv } from '../src/beam-openai-client.js';
|
||||
|
||||
const FILTER_SYSTEM =
|
||||
'You filter a list of dated facts about a USER. KEEP ONLY standing directives: ' +
|
||||
'explicit preferences ("prefers X", "dislikes Y", "favorite is Z"), standing instructions or rules the user ' +
|
||||
'gave the assistant ("always respond with...", "never suggest...", "call me..."), and durable personal ' +
|
||||
'constraints that shape future answers (dietary restrictions, accessibility needs, format/tooling/style rules). ' +
|
||||
'DISCARD everything else: events, one-off tasks, project status, possessions, plans, numbers that are not rules. ' +
|
||||
'Output the kept lines VERBATIM (including their [YYYY-MM-DD] prefix), one per line, no bullets, no preamble. ' +
|
||||
'If nothing qualifies, output nothing.';
|
||||
|
||||
const MERGE_SYSTEM =
|
||||
'You deduplicate a list of dated user directives (preferences / standing instructions). Merge duplicates and ' +
|
||||
'near-duplicates, KEEPING the most recent date for each distinct directive and its most specific wording. ' +
|
||||
'If two directives conflict, keep BOTH (they show a preference change; the reader uses the dates). ' +
|
||||
'Output one directive per line as "[YYYY-MM-DD] text", chronologically ordered, no preamble. Maximum 40 lines: ' +
|
||||
'if more, keep the most consequential.';
|
||||
|
||||
const FACT_DATE_RE = /^\[(\d{4}-\d{2}-\d{2})\]\s*/;
|
||||
const BATCH_CHARS = 12000;
|
||||
|
||||
function parseConvSpec(spec: string): number[] {
|
||||
const out = new Set<number>();
|
||||
for (const part of spec.split(',')) {
|
||||
const m = part.match(/^(\d+)-(\d+)$/);
|
||||
if (m) { for (let i = +m[1]; i <= +m[2]; i++) out.add(i); }
|
||||
else if (/^\d+$/.test(part.trim())) out.add(+part.trim());
|
||||
}
|
||||
return [...out].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const argv = process.argv.slice(2);
|
||||
const here = url.fileURLToPath(import.meta.url);
|
||||
const repoRoot = path.resolve(path.dirname(here), '..', '..', '..');
|
||||
let convs = parseConvSpec('1-35');
|
||||
let estimate = false;
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
if (argv[i] === '--convs' && argv[i + 1]) convs = parseConvSpec(argv[++i]);
|
||||
else if (argv[i] === '--estimate') estimate = true;
|
||||
}
|
||||
const obsDir = path.join(repoRoot, 'benchmarks', 'data', 'beam', 'minds-1M-obs');
|
||||
const outDir = path.join(repoRoot, 'benchmarks', 'data', 'beam', 'directives-1M');
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
|
||||
loadDotEnv();
|
||||
const client = estimate ? null : createBeamOpenAiClient({ model: 'gpt-4o-mini' });
|
||||
|
||||
let totalCost = 0; let totalChars = 0; let totalBatches = 0;
|
||||
for (const conv of convs) {
|
||||
const outPath = path.join(outDir, `beam_1M_${conv}.json`);
|
||||
const donePath = path.join(outDir, `beam_1M_${conv}.done.json`);
|
||||
if (fs.existsSync(donePath) && fs.existsSync(outPath)) { console.log(`[directives][conv ${conv}] SKIP`); continue; }
|
||||
const mindPath = path.join(obsDir, `beam_1M_${conv}.mind`);
|
||||
if (!fs.existsSync(mindPath)) { console.error(`[directives][conv ${conv}] missing obs mind`); continue; }
|
||||
|
||||
const substrate = createSubstrate({ dbPath: mindPath });
|
||||
let facts: string[];
|
||||
try {
|
||||
facts = substrate.frames.getGopFrames(`beam_${conv}`)
|
||||
.map(f => f.content)
|
||||
.filter(c => FACT_DATE_RE.test(c));
|
||||
} finally { substrate.close(); }
|
||||
|
||||
// Batch the facts through the filter.
|
||||
const batches: string[] = [];
|
||||
let cur: string[] = []; let curLen = 0;
|
||||
for (const f of facts) {
|
||||
if (curLen + f.length > BATCH_CHARS && cur.length) { batches.push(cur.join('\n')); cur = []; curLen = 0; }
|
||||
cur.push(f); curLen += f.length + 1;
|
||||
}
|
||||
if (cur.length) batches.push(cur.join('\n'));
|
||||
totalChars += facts.reduce((a, b) => a + b.length, 0); totalBatches += batches.length;
|
||||
if (estimate) { console.log(`[directives][conv ${conv}] estimate: facts=${facts.length} batches=${batches.length}`); continue; }
|
||||
|
||||
let convCost = 0; const kept: string[] = [];
|
||||
for (const b of batches) {
|
||||
const res = await client!.chat({ system: FILTER_SYSTEM, user: b, maxTokens: 700 });
|
||||
convCost += res.costUsd;
|
||||
for (const line of res.text.split('\n')) { const t = line.trim(); if (t && FACT_DATE_RE.test(t)) kept.push(t); }
|
||||
}
|
||||
// Merge/dedupe pass.
|
||||
let directives: Array<{ date: string; text: string }> = [];
|
||||
if (kept.length) {
|
||||
const res = await client!.chat({ system: MERGE_SYSTEM, user: kept.join('\n'), maxTokens: 1200 });
|
||||
convCost += res.costUsd;
|
||||
for (const line of res.text.split('\n')) {
|
||||
const m = line.trim().match(FACT_DATE_RE);
|
||||
if (m) directives.push({ date: m[1], text: line.trim().slice(m[0].length) });
|
||||
}
|
||||
}
|
||||
fs.writeFileSync(outPath, JSON.stringify({ conv, gop_id: `beam_${conv}`, directives, built_at: new Date().toISOString(), cost_usd: Math.round(convCost * 1e4) / 1e4 }, null, 2));
|
||||
fs.writeFileSync(donePath, JSON.stringify({ conv, directives: directives.length, cost_usd: convCost }));
|
||||
totalCost += convCost;
|
||||
console.log(`[directives][conv ${conv}] DONE raw-kept=${kept.length} merged=${directives.length} cost=$${convCost.toFixed(4)}`);
|
||||
}
|
||||
if (estimate) {
|
||||
const inTok = totalChars / 4;
|
||||
console.log(`\nESTIMATE: batches=${totalBatches} input≈${(inTok / 1e6).toFixed(2)}M tok → ≈ $${((inTok / 1e6) * 0.15 + (totalBatches * 300 / 1e6) * 0.6).toFixed(2)}`);
|
||||
} else console.log(`\nALL DONE. total=$${totalCost.toFixed(2)}`);
|
||||
}
|
||||
|
||||
main().catch(err => { console.error('[beam-build-directives] FATAL:', err); process.exit(1); });
|
||||
126
benchmarks/harness/scripts/beam-build-outlines.ts
Normal file
126
benchmarks/harness/scripts/beam-build-outlines.ts
Normal file
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* BEAM 1M — CONVERSATION OUTLINE builder (uniform coverage lever).
|
||||
*
|
||||
* For each conversation, read the distilled facts already in minds-1M-obs
|
||||
* (content "[YYYY-MM-DD] fact"), group them by session date, and compress each
|
||||
* date-group into a tight synopsis via gpt-4o-mini. The result is a small
|
||||
* "conversation timeline" (~10 sessions x ~10 bullets) that the answer prompt
|
||||
* can prepend to EVERY question — giving summarization / preference /
|
||||
* instruction / event_ordering the global coverage that top-k turn retrieval
|
||||
* lacks, without routing and without touching the retrieved-turn detail.
|
||||
*
|
||||
* No ollama dependency (no embedding). RESUMABLE: per-conv .done.json marker.
|
||||
* Output: benchmarks/data/beam/outlines-1M/beam_1M_<conv>.outline.json
|
||||
* { conv, gop_id, sessions: [{date, synopsis}], built_at, cost_usd }
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx benchmarks/harness/scripts/beam-build-outlines.ts [--convs 1-35] [--estimate]
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import url from 'node:url';
|
||||
import process from 'node:process';
|
||||
|
||||
import { createSubstrate } from '../src/substrate.js';
|
||||
import { createBeamOpenAiClient, loadDotEnv } from '../src/beam-openai-client.js';
|
||||
|
||||
const OUTLINE_SYSTEM =
|
||||
'You compress a list of dated facts about a USER (extracted from one session of a long conversation) ' +
|
||||
'into a compact session synopsis. Output terse bullet lines, no preamble: 2-3 lines for sparse sessions ' +
|
||||
'(<30 facts), at most 8 for rich ones. ALWAYS include, when present: stated preferences and dislikes; ' +
|
||||
'standing instructions or rules the user gave; decisions made; key events (what happened); ' +
|
||||
'projects/topics worked on and their status; important numbers, names, versions. ' +
|
||||
'Be specific (keep names/numbers/versions verbatim). One fact per line, no blank lines.';
|
||||
|
||||
const FACT_DATE_RE = /^\[(\d{4}-\d{2}-\d{2})\]\s*/;
|
||||
|
||||
function parseConvSpec(spec: string): number[] {
|
||||
const out = new Set<number>();
|
||||
for (const part of spec.split(',')) {
|
||||
const m = part.match(/^(\d+)-(\d+)$/);
|
||||
if (m) { for (let i = +m[1]; i <= +m[2]; i++) out.add(i); }
|
||||
else if (/^\d+$/.test(part.trim())) out.add(+part.trim());
|
||||
}
|
||||
return [...out].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const argv = process.argv.slice(2);
|
||||
const here = url.fileURLToPath(import.meta.url);
|
||||
const repoRoot = path.resolve(path.dirname(here), '..', '..', '..');
|
||||
let convs = parseConvSpec('1-35');
|
||||
let estimate = false;
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
if (argv[i] === '--convs' && argv[i + 1]) { convs = parseConvSpec(argv[++i]); }
|
||||
else if (argv[i] === '--estimate') estimate = true;
|
||||
}
|
||||
const obsDir = path.join(repoRoot, 'benchmarks', 'data', 'beam', 'minds-1M-obs');
|
||||
const outDir = path.join(repoRoot, 'benchmarks', 'data', 'beam', 'outlines-1M');
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
|
||||
loadDotEnv();
|
||||
const client = estimate ? null : createBeamOpenAiClient({ model: 'gpt-4o-mini' });
|
||||
|
||||
let totalCost = 0; let totalInChars = 0; let totalGroups = 0;
|
||||
for (const conv of convs) {
|
||||
const outPath = path.join(outDir, `beam_1M_${conv}.outline.json`);
|
||||
const donePath = path.join(outDir, `beam_1M_${conv}.done.json`);
|
||||
if (fs.existsSync(donePath) && fs.existsSync(outPath)) {
|
||||
console.log(`[outline][conv ${conv}] SKIP (done)`);
|
||||
continue;
|
||||
}
|
||||
const mindPath = path.join(obsDir, `beam_1M_${conv}.mind`);
|
||||
if (!fs.existsSync(mindPath)) { console.error(`[outline][conv ${conv}] missing obs mind, skipping`); continue; }
|
||||
|
||||
// No embedder needed — we only read frames (default embedder object is
|
||||
// constructed but never called; no ollama traffic).
|
||||
const substrate = createSubstrate({ dbPath: mindPath });
|
||||
let byDate: Map<string, string[]>;
|
||||
try {
|
||||
const frames = substrate.frames.getGopFrames(`beam_${conv}`);
|
||||
byDate = new Map();
|
||||
for (const f of frames) {
|
||||
const m = f.content.match(FACT_DATE_RE);
|
||||
if (!m) continue;
|
||||
const list = byDate.get(m[1]) ?? [];
|
||||
list.push(f.content.slice(m[0].length));
|
||||
byDate.set(m[1], list);
|
||||
}
|
||||
} finally { substrate.close(); }
|
||||
|
||||
const dates = [...byDate.keys()].sort();
|
||||
const sessions: Array<{ date: string; synopsis: string }> = [];
|
||||
let convCost = 0;
|
||||
for (const d of dates) {
|
||||
const facts = byDate.get(d)!;
|
||||
const input = facts.join('\n');
|
||||
totalInChars += input.length; totalGroups++;
|
||||
if (estimate) continue;
|
||||
const res = await client!.chat({
|
||||
system: OUTLINE_SYSTEM,
|
||||
user: `SESSION DATE: ${d}\nFACTS (${facts.length}):\n${input}`,
|
||||
maxTokens: 350,
|
||||
});
|
||||
convCost += res.costUsd;
|
||||
sessions.push({ date: d, synopsis: res.text.trim() });
|
||||
}
|
||||
if (!estimate) {
|
||||
fs.writeFileSync(outPath, JSON.stringify({ conv, gop_id: `beam_${conv}`, sessions, built_at: new Date().toISOString(), cost_usd: Math.round(convCost * 1e4) / 1e4 }, null, 2));
|
||||
fs.writeFileSync(donePath, JSON.stringify({ conv, sessions: sessions.length, cost_usd: convCost }));
|
||||
totalCost += convCost;
|
||||
console.log(`[outline][conv ${conv}] DONE sessions=${sessions.length} cost=$${convCost.toFixed(4)}`);
|
||||
} else {
|
||||
console.log(`[outline][conv ${conv}] estimate: dates=${dates.length} facts=${[...byDate.values()].reduce((a, b) => a + b.length, 0)}`);
|
||||
}
|
||||
}
|
||||
if (estimate) {
|
||||
const inTok = totalInChars / 4;
|
||||
console.log(`\nESTIMATE: groups=${totalGroups} input≈${(inTok / 1e6).toFixed(2)}M tok → gpt-4o-mini ≈ $${((inTok / 1e6) * 0.15 + (totalGroups * 350 / 1e6) * 0.6).toFixed(2)}`);
|
||||
} else {
|
||||
console.log(`\nALL DONE. total cost=$${totalCost.toFixed(2)}`);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(err => { console.error('[beam-build-outlines] FATAL:', err); process.exit(1); });
|
||||
342
benchmarks/harness/scripts/beam-distill-1m.ts
Normal file
342
benchmarks/harness/scripts/beam-distill-1m.ts
Normal file
@@ -0,0 +1,342 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* BEAM 1M — write-time OBSERVATION DISTILLATION (mem0/Mastra/LongMemEval-style).
|
||||
*
|
||||
* Ported from D:\Projects\hive-mind\benchmarks\longmemeval\34-run-observations.mjs
|
||||
* (the pattern that added ~+10pp on LongMemEval). Instead of retrieving raw
|
||||
* conversation turns (~900 tok each), we distill each conversation into dense,
|
||||
* dated, atomic, pronoun-resolved facts (~35 tok each) with a windowed
|
||||
* gpt-4o-mini pass, and store those as `agent_inferred` frames in a SEPARATE
|
||||
* per-conversation mind cache (`minds-1M-obs/`). Answering then retrieves top-k
|
||||
* FACTS ≈ true mem0-parity semantics (~7K-tok prompts, cheap) with our
|
||||
* extraction quality — this is what lets a top-200 run cost ~$20 instead of
|
||||
* ~$470 while (hypothesis) lifting accuracy on the harder abilities.
|
||||
*
|
||||
* DISTILL_SYSTEM is the LongMemEval prompt verbatim. Windows are ~WIN chars of
|
||||
* dated turn text; BEAM carries per-MESSAGE `time_anchor` dates ("March-01-2024"),
|
||||
* carried forward so each window is tagged with its session date.
|
||||
*
|
||||
* RESUMABLE: per-conv `.done.json` marker in minds-1M-obs; skip-if-complete; a
|
||||
* partial (crashed) mind without a marker is rebuilt.
|
||||
*
|
||||
* COST-SAFE: `--estimate` builds windows only (NO gpt-4o-mini calls) and reports
|
||||
* window/token counts + a projected full-35 cost. Use it before any paid run.
|
||||
* Embedding of the resulting facts is local ollama (free).
|
||||
*
|
||||
* Usage:
|
||||
* tsx benchmarks/harness/scripts/beam-distill-1m.ts --estimate # free
|
||||
* tsx benchmarks/harness/scripts/beam-distill-1m.ts --convs 1-35 [--win 8000] [--conc 6]
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import url from 'node:url';
|
||||
import process from 'node:process';
|
||||
|
||||
import { createOllamaEmbedder } from '@waggle/core';
|
||||
import { createSubstrate } from '../src/substrate.js';
|
||||
import { createBeamOpenAiClient, loadDotEnv } from '../src/beam-openai-client.js';
|
||||
import type { BeamOpenAiClient } from '../src/beam-openai-client.js';
|
||||
|
||||
const DISTILL_SYSTEM =
|
||||
'You extract durable, atomic facts about the USER from a slice of their conversation with an assistant. ' +
|
||||
'Output one fact per line, each starting with "[YYYY-MM-DD] " using the date the fact/event pertains to ' +
|
||||
'(use the session date shown if no other date). Cover: preferences and dislikes, possessions/brands/tools, ' +
|
||||
'decisions, events (what happened, when), plans, personal attributes, relationships, numbers/quantities. ' +
|
||||
'Be specific and self-contained (resolve pronouns to the entity). Only facts grounded in the text. ' +
|
||||
'No preamble, no bullets, no blank lines. If nothing durable, output nothing.';
|
||||
|
||||
interface Args {
|
||||
convs: number[];
|
||||
win: number;
|
||||
conc: number;
|
||||
distillModel: string;
|
||||
beamChats: string;
|
||||
mindsDir: string;
|
||||
estimate: boolean;
|
||||
force: boolean;
|
||||
maxTokens: number;
|
||||
}
|
||||
|
||||
function parseConvSpec(spec: string): number[] {
|
||||
const out = new Set<number>();
|
||||
for (const part of spec.split(',')) {
|
||||
const m = part.match(/^(\d+)-(\d+)$/);
|
||||
if (m) { for (let i = +m[1]; i <= +m[2]; i++) out.add(i); }
|
||||
else if (/^\d+$/.test(part.trim())) out.add(+part.trim());
|
||||
}
|
||||
return [...out].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
function parseArgs(): Args {
|
||||
const argv = process.argv.slice(2);
|
||||
const here = url.fileURLToPath(import.meta.url);
|
||||
const repoRoot = path.resolve(path.dirname(here), '..', '..', '..');
|
||||
const a: Args = {
|
||||
convs: parseConvSpec('1-35'),
|
||||
win: 8000,
|
||||
conc: 6,
|
||||
distillModel: 'gpt-4o-mini',
|
||||
beamChats: path.resolve(repoRoot, '..', 'BEAM', 'chats'),
|
||||
mindsDir: path.join(repoRoot, 'benchmarks', 'data', 'beam', 'minds-1M-obs'),
|
||||
estimate: false,
|
||||
force: false,
|
||||
maxTokens: 700,
|
||||
};
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const f = argv[i]; const next = argv[i + 1];
|
||||
if (f === '--convs' && next) { a.convs = parseConvSpec(next); i++; }
|
||||
else if (f === '--win' && next) { a.win = parseInt(next, 10); i++; }
|
||||
else if (f === '--conc' && next) { a.conc = Math.max(1, parseInt(next, 10)); i++; }
|
||||
else if (f === '--distill-model' && next) { a.distillModel = next; i++; }
|
||||
else if (f === '--beam-chats' && next) { a.beamChats = path.resolve(next); i++; }
|
||||
else if (f === '--minds-dir' && next) { a.mindsDir = path.resolve(next); i++; }
|
||||
else if (f === '--estimate') { a.estimate = true; }
|
||||
else if (f === '--force') { a.force = true; }
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
||||
// ── Dated turn flattening + windowing ────────────────────────────────────────
|
||||
|
||||
interface RawMsg { role?: string; content?: string; time_anchor?: string | null }
|
||||
interface RawBatch { turns?: RawMsg[][] }
|
||||
|
||||
/** Parse BEAM's "March-01-2024" (or ISO) message time_anchor to YYYY-MM-DD. */
|
||||
function normBeamDate(s?: string | null): string | null {
|
||||
if (!s) return null;
|
||||
const raw = String(s).trim();
|
||||
let t = Date.parse(raw);
|
||||
if (!Number.isFinite(t)) t = Date.parse(raw.replace(/-/g, ' '));
|
||||
if (!Number.isFinite(t)) return null;
|
||||
return new Date(t).toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
interface DatedTurn { role: string; content: string; date: string | null }
|
||||
|
||||
function flattenDatedTurns(chatJsonPath: string): DatedTurn[] {
|
||||
const batches = JSON.parse(fs.readFileSync(chatJsonPath, 'utf-8')) as RawBatch[];
|
||||
const out: DatedTurn[] = [];
|
||||
let lastDate: string | null = null;
|
||||
for (const batch of batches) {
|
||||
if (!Array.isArray(batch.turns)) continue;
|
||||
for (const group of batch.turns) {
|
||||
if (!Array.isArray(group)) continue;
|
||||
for (const msg of group) {
|
||||
const d = normBeamDate(msg.time_anchor);
|
||||
if (d) lastDate = d;
|
||||
const content = String(msg.content ?? '').trim();
|
||||
if (content) out.push({ role: String(msg.role ?? 'unknown').toLowerCase(), content, date: lastDate });
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
interface Window { text: string; date: string | null }
|
||||
|
||||
function buildWindows(turns: DatedTurn[], win: number): Window[] {
|
||||
const out: Window[] = [];
|
||||
let cur = ''; let curDate: string | null = null;
|
||||
for (const t of turns) {
|
||||
const line = `${t.date ? `[${t.date}] ` : ''}${t.role}: ${t.content}\n`;
|
||||
if (cur.length + line.length > win && cur) { out.push({ text: cur, date: curDate }); cur = ''; }
|
||||
if (!cur) curDate = t.date;
|
||||
cur += line;
|
||||
}
|
||||
if (cur) out.push({ text: cur, date: curDate });
|
||||
return out;
|
||||
}
|
||||
|
||||
function toIso(d: string | null): string | undefined {
|
||||
if (!d) return undefined;
|
||||
const t = Date.parse(d);
|
||||
return Number.isFinite(t) ? new Date(t).toISOString() : undefined;
|
||||
}
|
||||
|
||||
function approxTokens(s: string): number { return Math.max(1, Math.ceil(s.length / 4)); }
|
||||
|
||||
// ── Marker helpers ───────────────────────────────────────────────────────────
|
||||
|
||||
interface DistillMarker {
|
||||
conv: number; gop_id: string; facts: number; windows: number;
|
||||
distill_ms: number; index_ms: number; cost_usd: number; win: number;
|
||||
distill_model: string; built_at: string;
|
||||
}
|
||||
function markerPath(dir: string, conv: number): string { return path.join(dir, `beam_1M_${conv}.done.json`); }
|
||||
function mindPath(dir: string, conv: number): string { return path.join(dir, `beam_1M_${conv}.mind`); }
|
||||
function isComplete(dir: string, conv: number): DistillMarker | null {
|
||||
const mp = markerPath(dir, conv);
|
||||
if (!fs.existsSync(mp)) return null;
|
||||
try { const m = JSON.parse(fs.readFileSync(mp, 'utf-8')) as DistillMarker; if (m && m.facts >= 0 && fs.existsSync(mindPath(dir, conv))) return m; } catch { /* */ }
|
||||
return null;
|
||||
}
|
||||
function logProgress(dir: string, line: string): void {
|
||||
const stamped = `${new Date().toISOString()} ${line}`;
|
||||
process.stdout.write(stamped + '\n');
|
||||
try { fs.appendFileSync(path.join(dir, '_distill-progress.log'), stamped + '\n'); } catch { /* */ }
|
||||
}
|
||||
|
||||
// Poll the local ollama server until it answers /api/tags (or ~2min elapses).
|
||||
// Called between index-batch retries so we resume only once the server is live.
|
||||
async function waitForOllama(dir: string, conv: number): Promise<void> {
|
||||
const host = process.env.OLLAMA_HOST || 'http://localhost:11434';
|
||||
const url = `${host.replace(/\/$/, '')}/api/tags`;
|
||||
for (let i = 0; i < 24; i++) {
|
||||
try {
|
||||
const ac = new AbortController();
|
||||
const t = setTimeout(() => ac.abort(), 5000);
|
||||
const res = await fetch(url, { signal: ac.signal });
|
||||
clearTimeout(t);
|
||||
if (res.ok) return;
|
||||
} catch { /* server not up yet */ }
|
||||
await new Promise(r => setTimeout(r, 5000));
|
||||
}
|
||||
logProgress(dir, `[distill][conv ${conv}] ollama still unresponsive after ~2min wait — retrying batch anyway`);
|
||||
}
|
||||
|
||||
// ── Estimate mode (no gpt-4o-mini spend) ─────────────────────────────────────
|
||||
|
||||
function runEstimate(args: Args): void {
|
||||
let totalWindows = 0; let totalInputChars = 0; let convsSeen = 0;
|
||||
for (const conv of args.convs) {
|
||||
const cj = path.join(args.beamChats, '1M', String(conv), 'chat.json');
|
||||
if (!fs.existsSync(cj)) continue;
|
||||
const turns = flattenDatedTurns(cj);
|
||||
const wins = buildWindows(turns, args.win);
|
||||
totalWindows += wins.length;
|
||||
for (const w of wins) totalInputChars += w.text.length;
|
||||
convsSeen++;
|
||||
}
|
||||
const SYS_TOK = approxTokens(DISTILL_SYSTEM) + 20;
|
||||
const inputToks = Math.ceil(totalInputChars / 4) + totalWindows * SYS_TOK;
|
||||
const outToksEst = totalWindows * 350; // ~350 output tokens/window (facts)
|
||||
// gpt-4o-mini pricing.
|
||||
const IN = 0.15 / 1e6, OUT = 0.6 / 1e6;
|
||||
const cost = inputToks * IN + outToksEst * OUT;
|
||||
const scale = convsSeen > 0 ? 35 / convsSeen : 1;
|
||||
console.log('\n════════ DISTILL COST ESTIMATE (gpt-4o-mini) ════════');
|
||||
console.log(`convs measured: ${convsSeen} (window size ${args.win} chars)`);
|
||||
console.log(`windows: ${totalWindows} (~${(totalWindows / (convsSeen || 1)).toFixed(0)}/conv)`);
|
||||
console.log(`input tokens: ${(inputToks / 1e6).toFixed(2)}M`);
|
||||
console.log(`est output tokens: ${(outToksEst / 1e6).toFixed(2)}M (~350/window)`);
|
||||
console.log(`cost (measured ${convsSeen} conv): $${cost.toFixed(2)}`);
|
||||
console.log(`──────────────────────────────────────────────────`);
|
||||
console.log(`PROJECTED FULL 35: $${(cost * scale).toFixed(2)} windows≈${Math.round(totalWindows * scale)}`);
|
||||
console.log(`(embedding the facts is local ollama = free; distill is gpt-4o-mini only)`);
|
||||
}
|
||||
|
||||
// ── Distill one conversation ─────────────────────────────────────────────────
|
||||
|
||||
interface FactRow { text: string; date: string | null }
|
||||
|
||||
async function distillWindows(client: BeamOpenAiClient, wins: Window[], now: string | null, conc: number, maxTokens: number): Promise<{ facts: FactRow[]; costUsd: number }> {
|
||||
const facts: FactRow[] = [];
|
||||
let costUsd = 0;
|
||||
for (let w = 0; w < wins.length; w += conc) {
|
||||
const batch = wins.slice(w, w + conc);
|
||||
const results = await Promise.all(batch.map(win =>
|
||||
client.chat({ system: DISTILL_SYSTEM, user: `Session date: ${win.date || now || 'unknown'}\n\n${win.text}`, maxTokens })
|
||||
.catch(() => ({ text: '', inputTokens: 0, outputTokens: 0, costUsd: 0, latencyMs: 0, failureMode: 'error' })),
|
||||
));
|
||||
results.forEach((r, bi) => {
|
||||
costUsd += r.costUsd;
|
||||
for (const line of r.text.split('\n')) {
|
||||
const s = line.trim();
|
||||
if (s.length > 8) {
|
||||
const m = s.match(/\[(\d{4}-\d{2}-\d{2})\]/);
|
||||
facts.push({ text: s, date: (m ? m[1] : null) ?? batch[bi].date ?? now });
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
return { facts, costUsd };
|
||||
}
|
||||
|
||||
async function distillOne(args: Args, client: BeamOpenAiClient, conv: number): Promise<DistillMarker> {
|
||||
const gopId = `beam_${conv}`;
|
||||
const cj = path.join(args.beamChats, '1M', String(conv), 'chat.json');
|
||||
if (!fs.existsSync(cj)) throw new Error(`chat.json not found: ${cj}`);
|
||||
|
||||
const mp = mindPath(args.mindsDir, conv);
|
||||
for (const s of ['', '-wal', '-shm']) if (fs.existsSync(mp + s)) fs.rmSync(mp + s, { force: true });
|
||||
|
||||
const turns = flattenDatedTurns(cj);
|
||||
const now = (() => { const ds = turns.map(t => t.date).filter(Boolean).sort() as string[]; return ds.length ? ds[ds.length - 1] : null; })();
|
||||
const wins = buildWindows(turns, args.win);
|
||||
logProgress(args.mindsDir, `[distill][conv ${conv}] START windows=${wins.length} turns=${turns.length}`);
|
||||
|
||||
const tD = Date.now();
|
||||
const { facts, costUsd } = await distillWindows(client, wins, now, args.conc, args.maxTokens);
|
||||
const distillMs = Date.now() - tD;
|
||||
|
||||
const embedder = createOllamaEmbedder();
|
||||
const substrate = createSubstrate({ dbPath: mp, embedder });
|
||||
let indexMs = 0;
|
||||
try {
|
||||
substrate.sessions.ensure(gopId, 'beam-obs', `BEAM obs ${gopId}`);
|
||||
const toIndex: Array<{ id: number; content: string }> = [];
|
||||
const seen = new Set<number>();
|
||||
for (const fct of facts) {
|
||||
const frame = substrate.frames.createIFrame(gopId, fct.text, 'important', 'agent_inferred', toIso(fct.date));
|
||||
if (seen.has(frame.id)) continue;
|
||||
seen.add(frame.id);
|
||||
toIndex.push({ id: frame.id, content: fct.text });
|
||||
}
|
||||
const tI = Date.now();
|
||||
for (let b = 0; b < toIndex.length; b += 200) {
|
||||
const batch = toIndex.slice(b, b + 200);
|
||||
// ollama periodically becomes unresponsive under sustained multi-hour load
|
||||
// (mid-embed AbortError, or a hard connect-timeout when the server stalls).
|
||||
// A resumable run must not die on either: wait for the server to come back,
|
||||
// then retry the batch. Up to 8 attempts, backoff to 60s (~4min window).
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try { await substrate.search.indexFramesBatch(batch); break; }
|
||||
catch (err) {
|
||||
if (attempt >= 8) throw err;
|
||||
const waitMs = Math.min(60000, 8000 * attempt);
|
||||
logProgress(args.mindsDir, `[distill][conv ${conv}] index batch @${b} failed (attempt ${attempt}/8): ${(err as Error).message} — waiting for ollama, retry in ${waitMs / 1000}s`);
|
||||
await new Promise(r => setTimeout(r, waitMs));
|
||||
await waitForOllama(args.mindsDir, conv);
|
||||
}
|
||||
}
|
||||
}
|
||||
indexMs = Date.now() - tI;
|
||||
|
||||
const marker: DistillMarker = {
|
||||
conv, gop_id: gopId, facts: toIndex.length, windows: wins.length,
|
||||
distill_ms: distillMs, index_ms: indexMs, cost_usd: Math.round(costUsd * 1e4) / 1e4,
|
||||
win: args.win, distill_model: args.distillModel, built_at: new Date().toISOString(),
|
||||
};
|
||||
fs.writeFileSync(markerPath(args.mindsDir, conv), JSON.stringify(marker, null, 2) + '\n', 'utf-8');
|
||||
return marker;
|
||||
} finally {
|
||||
substrate.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs();
|
||||
if (args.estimate) { runEstimate(args); return; }
|
||||
|
||||
fs.mkdirSync(args.mindsDir, { recursive: true });
|
||||
loadDotEnv();
|
||||
const client = createBeamOpenAiClient({ model: args.distillModel });
|
||||
logProgress(args.mindsDir, `[distill] start convs=${args.convs[0]}..${args.convs[args.convs.length - 1]} (n=${args.convs.length}) win=${args.win} conc=${args.conc} model=${args.distillModel}`);
|
||||
|
||||
let done = 0; let totalCost = 0; const total = args.convs.length; const t0 = Date.now();
|
||||
for (const conv of args.convs) {
|
||||
const existing = args.force ? null : isComplete(args.mindsDir, conv);
|
||||
if (existing) { done++; logProgress(args.mindsDir, `[distill][conv ${conv}] SKIP (facts=${existing.facts}) [${done}/${total}]`); continue; }
|
||||
const c0 = Date.now();
|
||||
const m = await distillOne(args, client, conv);
|
||||
totalCost += m.cost_usd; done++;
|
||||
const secs = ((Date.now() - c0) / 1000).toFixed(0);
|
||||
const rate = done / ((Date.now() - t0) / 60000);
|
||||
const eta = rate > 0 ? ((total - done) / rate).toFixed(1) : '?';
|
||||
logProgress(args.mindsDir, `[distill][conv ${conv}] DONE facts=${m.facts} windows=${m.windows} cost=$${m.cost_usd} took=${secs}s [${done}/${total}] cum=$${totalCost.toFixed(2)} eta=${eta}m`);
|
||||
}
|
||||
logProgress(args.mindsDir, `[distill] ALL DONE ${done}/${total} convs, total cost=$${totalCost.toFixed(2)} in ${((Date.now() - t0) / 60000).toFixed(1)}m`);
|
||||
}
|
||||
|
||||
main().catch(err => { console.error('[beam-distill-1m] FATAL:', err); process.exit(1); });
|
||||
270
benchmarks/harness/scripts/beam-ingest-1m.ts
Normal file
270
benchmarks/harness/scripts/beam-ingest-1m.ts
Normal file
@@ -0,0 +1,270 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* BEAM 1M — resumable per-conversation substrate ingest (mind-per-conv).
|
||||
*
|
||||
* DESIGN (per build-phase brief):
|
||||
* - One persistent .mind file PER conversation → no cross-conversation lock
|
||||
* contention, so N conversations can ingest concurrently.
|
||||
* - RESUMABLE: a `<mind>.done.json` marker is written only after a
|
||||
* conversation fully ingests + indexes. On restart, conversations with a
|
||||
* valid marker are skipped; a .mind file WITHOUT a marker (crash mid-ingest)
|
||||
* is deleted and rebuilt from scratch. This mirrors LongMemEval's
|
||||
* skip-if-complete minds cache.
|
||||
* - Reads turns directly from the BEAM repo's per-conversation chat.json
|
||||
* (~4 MB each), NOT the 3 GB canonical — `extractTurnsFromBeam` does a
|
||||
* readFileSync that would blow V8's string limit on the 1M archive.
|
||||
*
|
||||
* The gop_id written per frame is `beam_<convId>`, matching the
|
||||
* `conversation_id` on every canonical instance — so the answer cells scope to
|
||||
* the right conversation without any change to their gopId filter.
|
||||
*
|
||||
* Embedding: local Ollama `nomic-embed-text` (1024-dim) — free, no API spend.
|
||||
* The embedder has a fixed 30 s per-request timeout, so the index batch size is
|
||||
* kept modest (default 48) to avoid timing out on long turns.
|
||||
*
|
||||
* Progress: one stdout line per conversation start/finish, plus a marker file
|
||||
* per completed conversation and an appended progress log — all monitorable
|
||||
* from the filesystem while this runs in the background.
|
||||
*
|
||||
* Usage:
|
||||
* tsx benchmarks/harness/scripts/beam-ingest-1m.ts \
|
||||
* [--convs 1-35] [--concurrency 1] [--batch-size 48] \
|
||||
* [--beam-chats D:/Projects/BEAM/chats] [--minds-dir <path>] [--force]
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import url from 'node:url';
|
||||
import process from 'node:process';
|
||||
|
||||
import { createOllamaEmbedder } from '@waggle/core';
|
||||
import { createSubstrate } from '../src/substrate.js';
|
||||
import { ingestBeamCorpus } from '../src/ingest-beam.js';
|
||||
import type { BeamTurn } from '../src/ingest-beam.js';
|
||||
|
||||
const CHAT_SIZE = '1M';
|
||||
const CHAT_SIZE_DIR = '1M';
|
||||
|
||||
interface Args {
|
||||
convs: number[];
|
||||
concurrency: number;
|
||||
batchSize: number;
|
||||
beamChats: string;
|
||||
mindsDir: string;
|
||||
force: boolean;
|
||||
}
|
||||
|
||||
function parseConvSpec(spec: string): number[] {
|
||||
const out = new Set<number>();
|
||||
for (const part of spec.split(',')) {
|
||||
const m = part.match(/^(\d+)-(\d+)$/);
|
||||
if (m) {
|
||||
const a = parseInt(m[1], 10);
|
||||
const b = parseInt(m[2], 10);
|
||||
for (let i = a; i <= b; i++) out.add(i);
|
||||
} else if (/^\d+$/.test(part.trim())) {
|
||||
out.add(parseInt(part.trim(), 10));
|
||||
}
|
||||
}
|
||||
return [...out].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
function parseArgs(): Args {
|
||||
const argv = process.argv.slice(2);
|
||||
const here = url.fileURLToPath(import.meta.url);
|
||||
const repoRoot = path.resolve(path.dirname(here), '..', '..', '..');
|
||||
let convs = parseConvSpec('1-35');
|
||||
let concurrency = 1;
|
||||
let batchSize = 48;
|
||||
let beamChats = path.resolve(repoRoot, '..', 'BEAM', 'chats');
|
||||
let mindsDir = path.join(repoRoot, 'benchmarks', 'data', 'beam', 'minds-1M');
|
||||
let force = false;
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const f = argv[i];
|
||||
const next = argv[i + 1];
|
||||
if (f === '--convs' && next) { convs = parseConvSpec(next); i++; }
|
||||
else if (f === '--concurrency' && next) { concurrency = Math.max(1, parseInt(next, 10)); i++; }
|
||||
else if (f === '--batch-size' && next) { batchSize = Math.max(1, parseInt(next, 10)); i++; }
|
||||
else if (f === '--beam-chats' && next) { beamChats = path.resolve(next); i++; }
|
||||
else if (f === '--minds-dir' && next) { mindsDir = path.resolve(next); i++; }
|
||||
else if (f === '--force') { force = true; }
|
||||
}
|
||||
return { convs, concurrency, batchSize, beamChats, mindsDir, force };
|
||||
}
|
||||
|
||||
interface RawMsg { role?: string; content?: string }
|
||||
interface RawBatch { turns?: RawMsg[][] }
|
||||
|
||||
/**
|
||||
* Flatten a BEAM chat.json into ordered {role, content} messages.
|
||||
* Replicates build-beam-canonical.ts::flattenBeamTurns EXACTLY (batches →
|
||||
* turn-groups → messages, non-empty content only) so the ingested frames match
|
||||
* the canonical's conversation turns.
|
||||
*/
|
||||
function flattenChatJson(chatJsonPath: string): Array<{ role: string; content: string }> {
|
||||
const batches = JSON.parse(fs.readFileSync(chatJsonPath, 'utf-8')) as RawBatch[];
|
||||
const out: Array<{ role: string; content: string }> = [];
|
||||
for (const batch of batches) {
|
||||
if (!Array.isArray(batch.turns)) continue;
|
||||
for (const group of batch.turns) {
|
||||
if (!Array.isArray(group)) continue;
|
||||
for (const msg of group) {
|
||||
const role = String(msg.role ?? 'unknown').toLowerCase();
|
||||
const content = String(msg.content ?? '').trim();
|
||||
if (content) out.push({ role, content });
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
interface IngestMarker {
|
||||
conv: number;
|
||||
gop_id: string;
|
||||
chat_size: string;
|
||||
frames: number;
|
||||
turns_seen: number;
|
||||
ingest_ms: number;
|
||||
index_ms: number;
|
||||
batch_size: number;
|
||||
embedder_model: string;
|
||||
embedder_dims: number;
|
||||
built_at: string;
|
||||
}
|
||||
|
||||
function markerPath(mindsDir: string, conv: number): string {
|
||||
return path.join(mindsDir, `beam_1M_${conv}.done.json`);
|
||||
}
|
||||
function mindPath(mindsDir: string, conv: number): string {
|
||||
return path.join(mindsDir, `beam_1M_${conv}.mind`);
|
||||
}
|
||||
|
||||
function isComplete(mindsDir: string, conv: number): IngestMarker | null {
|
||||
const mp = markerPath(mindsDir, conv);
|
||||
if (!fs.existsSync(mp)) return null;
|
||||
try {
|
||||
const m = JSON.parse(fs.readFileSync(mp, 'utf-8')) as IngestMarker;
|
||||
if (m && m.frames > 0 && fs.existsSync(mindPath(mindsDir, conv))) return m;
|
||||
} catch { /* fall through */ }
|
||||
return null;
|
||||
}
|
||||
|
||||
function logProgress(mindsDir: string, line: string): void {
|
||||
const stamped = `${new Date().toISOString()} ${line}`;
|
||||
process.stdout.write(stamped + '\n');
|
||||
try {
|
||||
fs.appendFileSync(path.join(mindsDir, '_ingest-progress.log'), stamped + '\n');
|
||||
} catch { /* best-effort */ }
|
||||
}
|
||||
|
||||
async function ingestOne(args: Args, conv: number): Promise<IngestMarker> {
|
||||
const gopId = `beam_${conv}`;
|
||||
const chatJsonPath = path.join(args.beamChats, CHAT_SIZE_DIR, String(conv), 'chat.json');
|
||||
if (!fs.existsSync(chatJsonPath)) {
|
||||
throw new Error(`chat.json not found for conv ${conv}: ${chatJsonPath}`);
|
||||
}
|
||||
|
||||
// Clean any partial .mind left by a prior crash (no valid marker present).
|
||||
const mp = mindPath(args.mindsDir, conv);
|
||||
for (const suffix of ['', '-wal', '-shm']) {
|
||||
const f = mp + suffix;
|
||||
if (fs.existsSync(f)) fs.rmSync(f, { force: true });
|
||||
}
|
||||
|
||||
const messages = flattenChatJson(chatJsonPath);
|
||||
const turns: BeamTurn[] = messages.map((m, i) => ({
|
||||
gopId,
|
||||
messageIndex: i,
|
||||
role: m.role === 'assistant' ? 'assistant' : 'user',
|
||||
content: m.content,
|
||||
formattedContent: `${m.role}: ${m.content}`,
|
||||
chatSize: CHAT_SIZE,
|
||||
}));
|
||||
|
||||
logProgress(args.mindsDir, `[ingest][conv ${conv}] START turns=${turns.length} db=${path.basename(mp)}`);
|
||||
|
||||
const embedder = createOllamaEmbedder();
|
||||
const substrate = createSubstrate({ dbPath: mp, embedder });
|
||||
try {
|
||||
const stats = await ingestBeamCorpus(
|
||||
substrate.db, substrate.search, substrate.frames, substrate.sessions,
|
||||
turns, { batchSize: args.batchSize },
|
||||
);
|
||||
const marker: IngestMarker = {
|
||||
conv,
|
||||
gop_id: gopId,
|
||||
chat_size: CHAT_SIZE,
|
||||
frames: stats.count,
|
||||
turns_seen: turns.length,
|
||||
ingest_ms: stats.ingestMs,
|
||||
index_ms: stats.indexMs,
|
||||
batch_size: args.batchSize,
|
||||
embedder_model: 'nomic-embed-text',
|
||||
embedder_dims: embedder.dimensions,
|
||||
built_at: new Date().toISOString(),
|
||||
};
|
||||
fs.writeFileSync(markerPath(args.mindsDir, conv), JSON.stringify(marker, null, 2) + '\n', 'utf-8');
|
||||
return marker;
|
||||
} finally {
|
||||
substrate.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function runPool(args: Args, convs: number[]): Promise<void> {
|
||||
let cursor = 0;
|
||||
let done = 0;
|
||||
const total = convs.length;
|
||||
const startAll = Date.now();
|
||||
|
||||
async function worker(): Promise<void> {
|
||||
while (true) {
|
||||
const idx = cursor++;
|
||||
if (idx >= convs.length) return;
|
||||
const conv = convs[idx];
|
||||
const existing = args.force ? null : isComplete(args.mindsDir, conv);
|
||||
if (existing) {
|
||||
done++;
|
||||
logProgress(args.mindsDir, `[ingest][conv ${conv}] SKIP (already complete, frames=${existing.frames}) [${done}/${total}]`);
|
||||
continue;
|
||||
}
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
const m = await ingestOne(args, conv);
|
||||
done++;
|
||||
const secs = ((Date.now() - t0) / 1000).toFixed(1);
|
||||
const elapsedMin = ((Date.now() - startAll) / 60000).toFixed(1);
|
||||
const rate = done / ((Date.now() - startAll) / 60000);
|
||||
const etaMin = rate > 0 ? ((total - done) / rate).toFixed(1) : '?';
|
||||
logProgress(
|
||||
args.mindsDir,
|
||||
`[ingest][conv ${conv}] DONE frames=${m.frames} index_ms=${m.index_ms} took=${secs}s ` +
|
||||
`[${done}/${total}] elapsed=${elapsedMin}m eta=${etaMin}m`,
|
||||
);
|
||||
} catch (err) {
|
||||
logProgress(args.mindsDir, `[ingest][conv ${conv}] ERROR ${(err as Error).message}`);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const workers = Array.from({ length: Math.min(args.concurrency, convs.length) }, () => worker());
|
||||
await Promise.all(workers);
|
||||
const totalMin = ((Date.now() - startAll) / 60000).toFixed(1);
|
||||
logProgress(args.mindsDir, `[ingest] ALL DONE ${done}/${total} conversations in ${totalMin}m`);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs();
|
||||
fs.mkdirSync(args.mindsDir, { recursive: true });
|
||||
logProgress(
|
||||
args.mindsDir,
|
||||
`[ingest] start convs=${args.convs[0]}..${args.convs[args.convs.length - 1]} (n=${args.convs.length}) ` +
|
||||
`concurrency=${args.concurrency} batch=${args.batchSize} chats=${args.beamChats}`,
|
||||
);
|
||||
await runPool(args, args.convs);
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('[beam-ingest-1m] FATAL:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
1269
benchmarks/harness/scripts/beam-run-1m.ts
Normal file
1269
benchmarks/harness/scripts/beam-run-1m.ts
Normal file
File diff suppressed because it is too large
Load Diff
261
benchmarks/harness/scripts/beam-smoke.ts
Normal file
261
benchmarks/harness/scripts/beam-smoke.ts
Normal file
@@ -0,0 +1,261 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* BEAM 1M — judge/metric plumbing SMOKE (no-context cell).
|
||||
*
|
||||
* PURPOSE: validate the full graded-nugget-judge + metric pipeline end-to-end
|
||||
* on a cheap cell, per the build-phase brief. This is NOT a scored result — the
|
||||
* "no-context" cell gives the answerer zero memories, so it should abstain on
|
||||
* almost everything and score near-floor except on abstention questions (whose
|
||||
* gold answer IS "I don't have enough information"). The point is to confirm the
|
||||
* plumbing works and the metric distinguishes abilities, before any expensive
|
||||
* substrate ingest.
|
||||
*
|
||||
* PIPELINE per question:
|
||||
* 1. no-context answer : gpt-4o, buildAnswerGenerationPrompt(question, [])
|
||||
* 2. graded judge : gpt-4o, each rubric nugget -> {0,0.5,1}, mean = score
|
||||
* 3. metrics : Avg Score (micro) + Pass Rate (>=0.5), overall + per-ability
|
||||
*
|
||||
* SAMPLING: deterministic — the first N (default 5) questions per memory_ability
|
||||
* encountered in the canonical's instance_id sort order (= 50 questions total).
|
||||
*
|
||||
* COST: hard-capped (default $4, under the $5 authorized). gpt-4o answerer+judge.
|
||||
* OPENAI_API_KEY is read from waggle-os/.env (loadDotEnv).
|
||||
*
|
||||
* Usage:
|
||||
* tsx benchmarks/harness/scripts/beam-smoke.ts \
|
||||
* [--per-ability 5] [--model gpt-4o] [--budget 4] [--tau] \
|
||||
* [--data benchmarks/data/beam/beam-1M.jsonl]
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import readline from 'node:readline';
|
||||
import url from 'node:url';
|
||||
import process from 'node:process';
|
||||
|
||||
import { createBeamOpenAiClient } from '../src/beam-openai-client.js';
|
||||
import { buildAnswerGenerationPrompt, judgeQuestion } from '../src/beam-nugget-judge.js';
|
||||
import type { BeamLlmResult } from '../src/beam-nugget-judge.js';
|
||||
import { computeBeamMetrics, formatBeamMetrics } from '../src/beam-metrics.js';
|
||||
import type { BeamQuestionResult } from '../src/beam-metrics.js';
|
||||
|
||||
const ALL_ABILITIES = [
|
||||
'abstention', 'contradiction_resolution', 'event_ordering', 'information_extraction',
|
||||
'instruction_following', 'knowledge_update', 'multi_session_reasoning',
|
||||
'preference_following', 'summarization', 'temporal_reasoning',
|
||||
];
|
||||
|
||||
interface CompactInstance {
|
||||
instance_id: string;
|
||||
question: string;
|
||||
memory_ability: string;
|
||||
rubric: string[];
|
||||
expected: string[];
|
||||
}
|
||||
|
||||
interface Args {
|
||||
perAbility: number;
|
||||
model: string;
|
||||
budget: number;
|
||||
computeTau: boolean;
|
||||
dataPath: string;
|
||||
}
|
||||
|
||||
function parseArgs(): Args {
|
||||
const argv = process.argv.slice(2);
|
||||
const here = url.fileURLToPath(import.meta.url);
|
||||
const repoRoot = path.resolve(path.dirname(here), '..', '..', '..');
|
||||
let perAbility = 5;
|
||||
let model = 'gpt-4o';
|
||||
let budget = 4;
|
||||
let computeTau = false;
|
||||
let dataPath = path.join(repoRoot, 'benchmarks', 'data', 'beam', 'beam-1M.jsonl');
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const f = argv[i];
|
||||
const next = argv[i + 1];
|
||||
if (f === '--per-ability' && next) { perAbility = parseInt(next, 10); i++; }
|
||||
else if (f === '--model' && next) { model = next; i++; }
|
||||
else if (f === '--budget' && next) { budget = parseFloat(next); i++; }
|
||||
else if (f === '--tau') { computeTau = true; }
|
||||
else if (f === '--data' && next) { dataPath = path.resolve(next); i++; }
|
||||
}
|
||||
return { perAbility, model, budget, computeTau, dataPath };
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream the (large, ~3 GB) canonical and collect the first `perAbility`
|
||||
* instances per memory_ability. Early-terminates once every ability is full,
|
||||
* so only a handful of conversations are ever parsed. Drops the giant `context`
|
||||
* field immediately — the no-context cell does not use it.
|
||||
*/
|
||||
async function sampleInstances(dataPath: string, perAbility: number): Promise<CompactInstance[]> {
|
||||
if (!fs.existsSync(dataPath)) {
|
||||
throw new Error(`BEAM canonical not found at ${dataPath}. Build it via build-beam-canonical.ts --chat-size 1M`);
|
||||
}
|
||||
const buckets = new Map<string, CompactInstance[]>();
|
||||
for (const a of ALL_ABILITIES) buckets.set(a, []);
|
||||
const full = (): boolean => ALL_ABILITIES.every(a => (buckets.get(a)?.length ?? 0) >= perAbility);
|
||||
|
||||
const rl = readline.createInterface({ input: fs.createReadStream(dataPath, 'utf-8'), crlfDelay: Infinity });
|
||||
try {
|
||||
for await (const line of rl) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
let row: Record<string, unknown>;
|
||||
try { row = JSON.parse(trimmed); } catch { continue; }
|
||||
const ability = String(row.memory_ability ?? '');
|
||||
const bucket = buckets.get(ability);
|
||||
if (!bucket || bucket.length >= perAbility) {
|
||||
if (full()) break;
|
||||
continue;
|
||||
}
|
||||
bucket.push({
|
||||
instance_id: String(row.instance_id ?? ''),
|
||||
question: String(row.question ?? ''),
|
||||
memory_ability: ability,
|
||||
rubric: Array.isArray(row.rubric) ? (row.rubric as unknown[]).map(String) : [],
|
||||
expected: Array.isArray(row.expected) ? (row.expected as unknown[]).map(String) : [],
|
||||
});
|
||||
if (full()) break;
|
||||
}
|
||||
} finally {
|
||||
rl.close();
|
||||
}
|
||||
return ALL_ABILITIES.flatMap(a => buckets.get(a) ?? []);
|
||||
}
|
||||
|
||||
function stripAnswerPrefix(text: string): string {
|
||||
return text.includes('ANSWER:') ? text.split('ANSWER:').pop()!.trim() : text.trim();
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs();
|
||||
const startedAt = new Date();
|
||||
console.log(`[beam-smoke] model=${args.model} per-ability=${args.perAbility} budget=$${args.budget} tau=${args.computeTau}`);
|
||||
console.log(`[beam-smoke] data=${args.dataPath}`);
|
||||
|
||||
const client = createBeamOpenAiClient({ model: args.model });
|
||||
|
||||
console.log('[beam-smoke] sampling instances (streaming canonical)…');
|
||||
const instances = await sampleInstances(args.dataPath, args.perAbility);
|
||||
console.log(`[beam-smoke] sampled ${instances.length} instances across ${ALL_ABILITIES.length} abilities`);
|
||||
|
||||
let costUsd = 0;
|
||||
let answerCalls = 0;
|
||||
let judgeCalls = 0;
|
||||
let inputTokens = 0;
|
||||
let outputTokens = 0;
|
||||
const acc = (r: BeamLlmResult): void => {
|
||||
costUsd += r.costUsd; inputTokens += r.inputTokens; outputTokens += r.outputTokens;
|
||||
};
|
||||
|
||||
const perQuestion: BeamQuestionResult[] = [];
|
||||
const records: Record<string, unknown>[] = [];
|
||||
let budgetStopped = false;
|
||||
|
||||
for (const inst of instances) {
|
||||
if (costUsd >= args.budget) {
|
||||
budgetStopped = true;
|
||||
console.warn(`[beam-smoke] budget cap $${args.budget} reached — stopping at ${perQuestion.length} questions`);
|
||||
break;
|
||||
}
|
||||
|
||||
// 1. Generate no-context answer.
|
||||
const ans = await client.chat({
|
||||
system: '',
|
||||
user: buildAnswerGenerationPrompt(inst.question, []),
|
||||
maxTokens: 400,
|
||||
});
|
||||
acc(ans); answerCalls++;
|
||||
const answer = stripAnswerPrefix(ans.text);
|
||||
|
||||
// 2. Judge nuggets.
|
||||
const { judgement, llmResults } = await judgeQuestion(
|
||||
client,
|
||||
{ question: inst.question, rubric: inst.rubric, memoryAbility: inst.memory_ability, answer },
|
||||
{ computeTau: args.computeTau },
|
||||
);
|
||||
for (const r of llmResults) { acc(r); judgeCalls++; }
|
||||
|
||||
perQuestion.push({
|
||||
instanceId: inst.instance_id,
|
||||
memoryAbility: inst.memory_ability,
|
||||
score: judgement.score,
|
||||
...(judgement.error ? { error: judgement.error } : {}),
|
||||
});
|
||||
records.push({
|
||||
instance_id: inst.instance_id,
|
||||
memory_ability: inst.memory_ability,
|
||||
question: inst.question,
|
||||
answer,
|
||||
answer_failure_mode: ans.failureMode,
|
||||
score: judgement.score,
|
||||
judgment: judgement.judgment,
|
||||
nugget_scores: judgement.nuggetScores,
|
||||
...(judgement.scoreWithTau !== undefined ? { score_with_tau: judgement.scoreWithTau } : {}),
|
||||
...(judgement.eventOrdering ? { event_ordering: judgement.eventOrdering } : {}),
|
||||
n_nuggets: inst.rubric.length,
|
||||
});
|
||||
process.stdout.write(
|
||||
` [${perQuestion.length}/${instances.length}] ${inst.memory_ability.padEnd(24)} ` +
|
||||
`score=${judgement.score.toFixed(2)} (${judgement.judgment}) nuggets=${inst.rubric.length} $${costUsd.toFixed(3)}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
const metrics = computeBeamMetrics(perQuestion);
|
||||
const finishedAt = new Date();
|
||||
|
||||
// Write outputs.
|
||||
const here = url.fileURLToPath(import.meta.url);
|
||||
const repoRoot = path.resolve(path.dirname(here), '..', '..', '..');
|
||||
const outDir = path.join(repoRoot, 'benchmarks', 'results', 'beam');
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
const ts = startedAt.toISOString().replace(/[:.]/g, '-');
|
||||
const jsonlPath = path.join(outDir, `beam-1m-smoke-nocontext-${ts}.jsonl`);
|
||||
const summaryPath = path.join(outDir, `beam-1m-smoke-nocontext-${ts}.summary.json`);
|
||||
fs.writeFileSync(jsonlPath, records.map(r => JSON.stringify(r)).join('\n') + '\n', 'utf-8');
|
||||
|
||||
const summary = {
|
||||
run: {
|
||||
cell: 'no-context',
|
||||
dataset: 'beam-1m',
|
||||
model: args.model,
|
||||
judge_model: args.model,
|
||||
protocol: 'mem0-nugget-graded (0/0.5/1 avg-score, pass>=0.5)',
|
||||
per_ability: args.perAbility,
|
||||
compute_tau: args.computeTau,
|
||||
startedAt: startedAt.toISOString(),
|
||||
finishedAt: finishedAt.toISOString(),
|
||||
budgetStopped,
|
||||
},
|
||||
metrics: {
|
||||
overall_avg_score: metrics.overall.avgScore,
|
||||
overall_pass_rate_pct: metrics.overall.accuracy,
|
||||
total: metrics.overall.total,
|
||||
correct: metrics.overall.correct,
|
||||
errors: metrics.overall.errors,
|
||||
by_ability: metrics.byAbility,
|
||||
},
|
||||
cost: {
|
||||
total_usd: costUsd,
|
||||
answer_calls: answerCalls,
|
||||
judge_calls: judgeCalls,
|
||||
input_tokens: inputTokens,
|
||||
output_tokens: outputTokens,
|
||||
},
|
||||
};
|
||||
fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2) + '\n', 'utf-8');
|
||||
|
||||
console.log('\n════════ BEAM 1M smoke (no-context) — metric plumbing ════════');
|
||||
console.log(formatBeamMetrics(metrics));
|
||||
console.log('──────────────────────────────────────────────────────────────');
|
||||
console.log(`cost=$${costUsd.toFixed(4)} answer_calls=${answerCalls} judge_calls=${judgeCalls} ` +
|
||||
`in=${inputTokens} out=${outputTokens}`);
|
||||
console.log(`jsonl: ${jsonlPath}`);
|
||||
console.log(`summary: ${summaryPath}`);
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('[beam-smoke] FATAL:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
611
benchmarks/harness/scripts/build-beam-canonical.ts
Normal file
611
benchmarks/harness/scripts/build-beam-canonical.ts
Normal file
@@ -0,0 +1,611 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* Canonical BEAM archive builder — Track A (manifest-v8.2-final.md).
|
||||
*
|
||||
* BEAM data ships inside the mohammadtavakoli78/BEAM GitHub repo under chats/.
|
||||
* No separate download step is required if you have cloned the repo.
|
||||
*
|
||||
* Actual directory layout (discovered by inspection):
|
||||
* <beam-repo>/chats/<size>/ 100K | 500K | 1M | 10M
|
||||
* <N>/ numbered conversation directories (1-based)
|
||||
* chat.json [{batch_number, time_anchor, turns: [[{role,id,time_anchor,index,question_type,content}, ...], ...]}]
|
||||
* probing_questions/
|
||||
* probing_questions.json {<category>: [{question, <answer_field>, difficulty, ...}, ...], ...}
|
||||
* topic.json {topic, description, ...}
|
||||
*
|
||||
* Chat-size alias: repo uses "100K" for what we call "128K" (~130K tokens each).
|
||||
*
|
||||
* Reads: <beam-chats-path>/<chat-size-dir>/
|
||||
* Writes: benchmarks/data/beam/beam-<chat-size>.jsonl
|
||||
* benchmarks/data/beam/beam-<chat-size>.meta.json
|
||||
*
|
||||
* Canonicalisation guarantees (required for dataset_version hash determinism):
|
||||
* 1. Include every (conversation × memory_ability × question) triple.
|
||||
* 2. Sort by instance_id ascending.
|
||||
* 3. JSON.stringify each row (no spaces, explicit key order) + '\n'. No BOM.
|
||||
* 4. SHA-256 of the final byte stream.
|
||||
*
|
||||
* Per-instance JSONL schema:
|
||||
* {
|
||||
* "instance_id": "beam_<chatSize>_<convId>_<ability>_q<idx>",
|
||||
* "conversation_id": "beam_<convId>",
|
||||
* "question": "<probing question>",
|
||||
* "expected": ["<reference answer>"],
|
||||
* "context": "<flat role: content lines>",
|
||||
* "memory_ability": "<category>",
|
||||
* "chat_size": "<128K|500K|1M|10M>",
|
||||
* "conversation_index": <int>
|
||||
* }
|
||||
*
|
||||
* Source: mohammadtavakoli78/BEAM (GitHub)
|
||||
* Paper: Tavakoli et al. 2024 "Beyond a Million Tokens: Benchmarking and
|
||||
* Enhancing Long-Term Memory in LLMs" (arXiv:2510.27246, ICLR 2026).
|
||||
*
|
||||
* Zero LLM calls. Zero npm packages beyond Node.js built-ins.
|
||||
*
|
||||
* Usage:
|
||||
* tsx build-beam-canonical.ts --beam-chats-path /path/to/BEAM/chats [--chat-size 128K]
|
||||
* # --beam-chats-path defaults to <repo-root>/benchmarks/harness/scripts/../../../BEAM/chats
|
||||
* # i.e. a sibling BEAM clone next to waggle-os
|
||||
*/
|
||||
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import process from 'node:process';
|
||||
import url from 'node:url';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Constants
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const VALID_CHAT_SIZES = ['128K', '500K', '1M', '10M'] as const;
|
||||
type ChatSize = (typeof VALID_CHAT_SIZES)[number];
|
||||
|
||||
/**
|
||||
* Map our canonical chat-size names to the directory names used in the BEAM repo.
|
||||
* 128K ≈ 100K (actual token count ~130K).
|
||||
*/
|
||||
const CHAT_SIZE_DIR_MAP: Record<ChatSize, string[]> = {
|
||||
'128K': ['100K'],
|
||||
'500K': ['500K'],
|
||||
'1M': ['1M'],
|
||||
'10M': ['10M'],
|
||||
};
|
||||
|
||||
/**
|
||||
* All 10 BEAM memory ability categories.
|
||||
* Source: Table 1 in arXiv:2510.27246 and repo README.
|
||||
*/
|
||||
const MEMORY_ABILITIES = [
|
||||
'abstention',
|
||||
'contradiction_resolution',
|
||||
'event_ordering',
|
||||
'information_extraction',
|
||||
'instruction_following',
|
||||
'knowledge_update',
|
||||
'multi_session_reasoning',
|
||||
'preference_following',
|
||||
'summarization',
|
||||
'temporal_reasoning',
|
||||
] as const;
|
||||
type MemoryAbility = (typeof MEMORY_ABILITIES)[number];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// BEAM data schema (verified by inspection of actual BEAM repo files)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** One message inside a BEAM conversation turn. */
|
||||
interface BeamMessage {
|
||||
role: string;
|
||||
content: string;
|
||||
id?: number;
|
||||
time_anchor?: string | null;
|
||||
index?: string;
|
||||
question_type?: string;
|
||||
}
|
||||
|
||||
/** One batch (session) in chat.json. */
|
||||
interface BeamBatch {
|
||||
batch_number?: number;
|
||||
time_anchor?: string | null;
|
||||
turns: BeamMessage[][]; // list of turn groups; each group is list of msgs
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-category probing question.
|
||||
* Answer field varies by category — we try all known variants.
|
||||
*/
|
||||
interface BeamProbingQuestion {
|
||||
question?: string;
|
||||
// Category-specific answer fields (verified by inspection):
|
||||
answer?: string; // event_ordering, information_extraction, knowledge_update, multi_session_reasoning, temporal_reasoning
|
||||
ideal_response?: string; // abstention
|
||||
ideal_answer?: string; // contradiction_resolution
|
||||
expected_compliance?: string; // instruction_following, preference_following
|
||||
ideal_summary?: string; // summarization
|
||||
// Extra fields (stored for provenance, not used in eval directly):
|
||||
difficulty?: string;
|
||||
rubric?: string[];
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Output schema
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface CanonicalInstance {
|
||||
instance_id: string;
|
||||
conversation_id: string;
|
||||
question: string;
|
||||
expected: string[];
|
||||
context: string;
|
||||
memory_ability: string;
|
||||
chat_size: string;
|
||||
conversation_index: number;
|
||||
/**
|
||||
* schema v2 (2026-07-06): the ordered list of rubric "nuggets" for this
|
||||
* probing question. This is the criterion set the OFFICIAL BEAM metric
|
||||
* scores — each nugget is judged 0 / 0.5 / 1 by the graded LLM judge and
|
||||
* the per-question score is their mean (see beam-nugget-judge.ts). Carried
|
||||
* additively; v1 archives (e.g. the committed beam-128K.jsonl, dataset
|
||||
* hash 9311bba4…) do not have it. Appended LAST in FIELD_ORDER so the
|
||||
* leading columns are byte-identical to v1 for a human diff — note that
|
||||
* ANY added field changes the SHA-256 dataset_version, so a v1 archive
|
||||
* rebuilt with this code becomes a v2 hash. We do not rebuild 128K here.
|
||||
*/
|
||||
rubric: string[];
|
||||
}
|
||||
|
||||
const FIELD_ORDER: readonly (keyof CanonicalInstance)[] = [
|
||||
'instance_id',
|
||||
'conversation_id',
|
||||
'question',
|
||||
'expected',
|
||||
'context',
|
||||
'memory_ability',
|
||||
'chat_size',
|
||||
'conversation_index',
|
||||
'rubric',
|
||||
];
|
||||
|
||||
/** Canonical schema version. Bumped to 2 when `rubric` nuggets were added. */
|
||||
const SCHEMA_VERSION = 2;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Turn flattening
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Flatten BEAM's nested batch/turn-group structure into a flat list of
|
||||
* {role, content} messages, preserving temporal order.
|
||||
*
|
||||
* chat.json structure:
|
||||
* [ {batch_number, time_anchor, turns: [ [msg, msg, ...], [msg, ...] ]} ]
|
||||
*
|
||||
* We flatten: batches → turn groups → individual messages.
|
||||
* We only emit messages with non-empty content.
|
||||
*/
|
||||
function flattenBeamTurns(batches: BeamBatch[]): Array<{ role: string; content: string }> {
|
||||
const out: Array<{ role: string; content: string }> = [];
|
||||
for (const batch of batches) {
|
||||
if (!Array.isArray(batch.turns)) continue;
|
||||
for (const turnGroup of batch.turns) {
|
||||
if (!Array.isArray(turnGroup)) continue;
|
||||
for (const msg of turnGroup) {
|
||||
const role = String(msg.role ?? 'unknown').toLowerCase();
|
||||
const content = String(msg.content ?? '').trim();
|
||||
if (content) {
|
||||
out.push({ role, content });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert flat message list to context string.
|
||||
* Format: "user: ...\nassistant: ...\n"
|
||||
*/
|
||||
function buildContext(msgs: Array<{ role: string; content: string }>): string {
|
||||
return msgs.map(m => `${m.role}: ${m.content}`).join('\n');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Answer normalisation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Extract the reference answer from a probing question, trying all known
|
||||
* per-category answer field names.
|
||||
* Returns null if no answer field is found.
|
||||
*/
|
||||
function normaliseAnswer(pq: BeamProbingQuestion): string | null {
|
||||
return (
|
||||
pq.answer ??
|
||||
pq.ideal_response ??
|
||||
pq.ideal_answer ??
|
||||
pq.expected_compliance ??
|
||||
pq.ideal_summary ??
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the ordered list of rubric "nuggets" from a probing question.
|
||||
* Ported verbatim from mem0's `extract_rubric_nuggets` (benchmarks/beam/run.py):
|
||||
* the `rubric` field may be a list[str] (the BEAM 1M/10M shape), a dict with a
|
||||
* `nuggets` list, or a bare scalar. Empty/whitespace nuggets are dropped.
|
||||
*/
|
||||
function extractRubricNuggets(pq: BeamProbingQuestion): string[] {
|
||||
const raw = (pq as Record<string, unknown>).rubric;
|
||||
const clean = (arr: unknown[]): string[] =>
|
||||
arr
|
||||
.map(n =>
|
||||
n !== null && typeof n === 'object'
|
||||
? String((n as Record<string, unknown>).description ??
|
||||
(n as Record<string, unknown>).text ??
|
||||
JSON.stringify(n))
|
||||
: String(n),
|
||||
)
|
||||
.map(s => s.trim())
|
||||
.filter(s => s.length > 0);
|
||||
|
||||
if (Array.isArray(raw)) return clean(raw);
|
||||
if (raw !== null && typeof raw === 'object') {
|
||||
const nuggets = (raw as Record<string, unknown>).nuggets;
|
||||
if (Array.isArray(nuggets)) return clean(nuggets);
|
||||
}
|
||||
if (raw !== undefined && raw !== null && String(raw).trim().length > 0) {
|
||||
return [String(raw).trim()];
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Serialisation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function serializeCanonical(inst: CanonicalInstance): string {
|
||||
const ordered: Record<string, unknown> = {};
|
||||
for (const key of FIELD_ORDER) {
|
||||
ordered[key] = inst[key];
|
||||
}
|
||||
return JSON.stringify(ordered);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Directory discovery
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Locate the chat-size directory under beamChatsPath.
|
||||
* Maps our canonical size name to the BEAM repo directory name.
|
||||
*/
|
||||
function locateChatSizeDir(beamChatsPath: string, chatSize: ChatSize): string | null {
|
||||
const candidates = CHAT_SIZE_DIR_MAP[chatSize] ?? [chatSize];
|
||||
for (const dirName of candidates) {
|
||||
const full = path.join(beamChatsPath, dirName);
|
||||
if (fs.existsSync(full) && fs.statSync(full).isDirectory()) {
|
||||
return full;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return all numbered conversation directories inside chatSizeDir.
|
||||
* These are directories whose names are numeric strings (1, 2, 3, ...).
|
||||
*/
|
||||
function discoverConversationDirs(chatSizeDir: string): string[] {
|
||||
const entries = fs.readdirSync(chatSizeDir, { withFileTypes: true });
|
||||
return entries
|
||||
.filter(e => e.isDirectory() && /^\d+$/.test(e.name))
|
||||
.sort((a, b) => Number(a.name) - Number(b.name))
|
||||
.map(e => path.join(chatSizeDir, e.name));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CLI arg parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function parseArgs(): { beamChatsPath: string; chatSize: ChatSize } {
|
||||
const argv = process.argv.slice(2);
|
||||
let beamChatsPath = '';
|
||||
let chatSize: ChatSize = '128K';
|
||||
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const flag = argv[i];
|
||||
const next = argv[i + 1];
|
||||
if ((flag === '--beam-chats-path' || flag === '--beam-data-path') && next) {
|
||||
// Accept both --beam-chats-path (new) and --beam-data-path (old compat)
|
||||
// If user passes BEAM root (contains chats/ subdir), auto-append chats/
|
||||
let p = next;
|
||||
if (fs.existsSync(path.join(p, 'chats'))) {
|
||||
p = path.join(p, 'chats');
|
||||
}
|
||||
beamChatsPath = p;
|
||||
i++;
|
||||
} else if (flag === '--chat-size' && next) {
|
||||
const val = next as ChatSize;
|
||||
if (!VALID_CHAT_SIZES.includes(val)) {
|
||||
console.error(
|
||||
`[build-beam-canonical] unknown --chat-size "${val}". Valid: ${VALID_CHAT_SIZES.join(', ')}`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
chatSize = val;
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
return { beamChatsPath, chatSize };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function main(): void {
|
||||
const { beamChatsPath: beamChatsPathArg, chatSize } = parseArgs();
|
||||
|
||||
const here = url.fileURLToPath(import.meta.url);
|
||||
// Script lives at benchmarks/harness/scripts/build-beam-canonical.ts
|
||||
// repo root = 3 levels up: scripts/ → harness/ → benchmarks/ → repo root
|
||||
const scriptDir = path.dirname(here);
|
||||
const repoRoot = path.resolve(scriptDir, '..', '..', '..');
|
||||
const dataDir = path.resolve(repoRoot, 'benchmarks', 'data');
|
||||
|
||||
// Auto-discover beamChatsPath if not provided:
|
||||
// Try <repo-root>/../BEAM/chats (sibling clone convention)
|
||||
let beamChatsPath = beamChatsPathArg;
|
||||
if (!beamChatsPath) {
|
||||
const siblingGuess = path.resolve(repoRoot, '..', 'BEAM', 'chats');
|
||||
if (fs.existsSync(siblingGuess)) {
|
||||
beamChatsPath = siblingGuess;
|
||||
console.log(`[build-beam-canonical] auto-discovered BEAM chats at ${beamChatsPath}`);
|
||||
} else {
|
||||
console.error('[build-beam-canonical] --beam-chats-path is required (or clone BEAM as sibling of waggle-os).\n');
|
||||
console.error('Clone with: git clone https://github.com/mohammadtavakoli78/BEAM.git');
|
||||
console.error('Then re-run: tsx build-beam-canonical.ts --beam-chats-path /path/to/BEAM/chats');
|
||||
process.exit(2);
|
||||
}
|
||||
}
|
||||
|
||||
if (!fs.existsSync(beamChatsPath)) {
|
||||
console.error(`[build-beam-canonical] BEAM chats path not found: ${beamChatsPath}`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Step 1: locate chat-size directory
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
const chatSizeDir = locateChatSizeDir(beamChatsPath, chatSize);
|
||||
if (!chatSizeDir) {
|
||||
const tried = (CHAT_SIZE_DIR_MAP[chatSize] ?? [chatSize]).map(d => path.join(beamChatsPath, d));
|
||||
console.error(
|
||||
`[build-beam-canonical] could not find chat-size directory for "${chatSize}" under ${beamChatsPath}.`,
|
||||
);
|
||||
console.error(`Tried: ${tried.join(', ')}`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
console.log(`[build-beam-canonical] chat-size directory: ${chatSizeDir}`);
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Step 2: scan conversation directories
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
const convDirs = discoverConversationDirs(chatSizeDir);
|
||||
if (convDirs.length === 0) {
|
||||
console.error(
|
||||
`[build-beam-canonical] no numbered conversation directories found under ${chatSizeDir}.`,
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
console.log(`[build-beam-canonical] found ${convDirs.length} conversation directories`);
|
||||
|
||||
const all: CanonicalInstance[] = [];
|
||||
const skipStats = {
|
||||
missingChat: 0,
|
||||
missingProbing: 0,
|
||||
missingQuestion: 0,
|
||||
missingAnswer: 0,
|
||||
noTurns: 0,
|
||||
};
|
||||
|
||||
for (let convIdx = 0; convIdx < convDirs.length; convIdx++) {
|
||||
const convDir = convDirs[convIdx];
|
||||
const convId = path.basename(convDir); // e.g. "1", "2", ...
|
||||
|
||||
// ── Load chat.json ──────────────────────────────────────────────
|
||||
const chatJsonPath = path.join(convDir, 'chat.json');
|
||||
if (!fs.existsSync(chatJsonPath)) {
|
||||
console.warn(`[build-beam-canonical] skipping ${convDir}: no chat.json`);
|
||||
skipStats.missingChat++;
|
||||
continue;
|
||||
}
|
||||
|
||||
let batches: BeamBatch[];
|
||||
try {
|
||||
batches = JSON.parse(fs.readFileSync(chatJsonPath, 'utf-8')) as BeamBatch[];
|
||||
} catch (err) {
|
||||
console.warn(`[build-beam-canonical] skipping ${chatJsonPath}: ${String(err)}`);
|
||||
skipStats.missingChat++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const flatMsgs = flattenBeamTurns(batches);
|
||||
if (flatMsgs.length === 0) {
|
||||
console.warn(`[build-beam-canonical] skipping ${convDir}: 0 messages after flatten`);
|
||||
skipStats.noTurns++;
|
||||
continue;
|
||||
}
|
||||
const context = buildContext(flatMsgs);
|
||||
|
||||
// ── Load probing_questions/probing_questions.json ───────────────
|
||||
const pqPath = path.join(convDir, 'probing_questions', 'probing_questions.json');
|
||||
if (!fs.existsSync(pqPath)) {
|
||||
console.warn(`[build-beam-canonical] skipping ${convDir}: no probing_questions.json`);
|
||||
skipStats.missingProbing++;
|
||||
continue;
|
||||
}
|
||||
|
||||
let pqData: Record<string, BeamProbingQuestion[]>;
|
||||
try {
|
||||
pqData = JSON.parse(fs.readFileSync(pqPath, 'utf-8')) as Record<string, BeamProbingQuestion[]>;
|
||||
} catch (err) {
|
||||
console.warn(`[build-beam-canonical] skipping ${pqPath}: ${String(err)}`);
|
||||
skipStats.missingProbing++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const conversationId = `beam_${convId}`;
|
||||
const safeChatSize = chatSize.replace(/[^a-zA-Z0-9]/g, '');
|
||||
|
||||
// ── Iterate categories ──────────────────────────────────────────
|
||||
for (const [category, questions] of Object.entries(pqData)) {
|
||||
if (!Array.isArray(questions)) continue;
|
||||
|
||||
for (let qi = 0; qi < questions.length; qi++) {
|
||||
const pq = questions[qi];
|
||||
const questionText = pq.question ?? null;
|
||||
if (!questionText) {
|
||||
skipStats.missingQuestion++;
|
||||
continue;
|
||||
}
|
||||
const rubric = extractRubricNuggets(pq);
|
||||
// `expected` keeps the single normalised reference answer for the
|
||||
// legacy substring scorer. If a question has no single-answer field
|
||||
// but does carry rubric nuggets, fall back to the joined rubric
|
||||
// (mem0's ground_truth_answer convention) rather than dropping it.
|
||||
let answerText = normaliseAnswer(pq);
|
||||
if (answerText === null) {
|
||||
if (rubric.length > 0) {
|
||||
answerText = rubric.join(' | ');
|
||||
} else {
|
||||
skipStats.missingAnswer++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const instanceId = `beam_${safeChatSize}_${convId}_${category}_q${qi}`;
|
||||
|
||||
all.push({
|
||||
instance_id: instanceId,
|
||||
conversation_id: conversationId,
|
||||
question: questionText,
|
||||
expected: [answerText],
|
||||
context,
|
||||
memory_ability: category,
|
||||
chat_size: chatSize,
|
||||
conversation_index: convIdx,
|
||||
rubric,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Step 3: validate
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
if (all.length === 0) {
|
||||
console.error('[build-beam-canonical] extracted 0 instances.');
|
||||
console.error(`Scanned ${convDirs.length} conversation dirs. Skip stats: ${JSON.stringify(skipStats)}`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Step 4: sort + distribution
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
all.sort((a, b) => a.instance_id.localeCompare(b.instance_id));
|
||||
|
||||
const byAbility: Record<string, number> = {};
|
||||
for (const ma of MEMORY_ABILITIES) byAbility[ma] = 0;
|
||||
for (const inst of all) {
|
||||
byAbility[inst.memory_ability] = (byAbility[inst.memory_ability] ?? 0) + 1;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Step 5: write outputs
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
const outDir = path.join(dataDir, 'beam');
|
||||
if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true });
|
||||
|
||||
const safeChatSize = chatSize.replace(/[^a-zA-Z0-9]/g, '');
|
||||
const jsonlPath = path.join(outDir, `beam-${safeChatSize}.jsonl`);
|
||||
|
||||
// Stream the write + hash incrementally, one line at a time. The full body
|
||||
// for the 1M/10M tracks (~3 GB for 1M, since the ~4 MB context is repeated
|
||||
// per question) exceeds V8's max string length (~512 MB), so it can never
|
||||
// be materialised as a single `join()`ed string. Writing `line + '\n'` for
|
||||
// each row in sorted order produces byte-identical output to the old
|
||||
// `all.map(serializeCanonical).join('\n') + '\n'` (a trailing newline after
|
||||
// the final row), so the SHA-256 dataset_version stays deterministic and
|
||||
// matches what the join-based path would have produced.
|
||||
const hasher = crypto.createHash('sha256');
|
||||
const fd = fs.openSync(jsonlPath, 'w');
|
||||
try {
|
||||
for (const inst of all) {
|
||||
const line = serializeCanonical(inst) + '\n';
|
||||
fs.writeSync(fd, line, null, 'utf-8');
|
||||
hasher.update(line, 'utf-8');
|
||||
}
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
const hash = hasher.digest('hex');
|
||||
|
||||
const metaPath = path.join(outDir, `beam-${safeChatSize}.meta.json`);
|
||||
const withRubric = all.filter(i => i.rubric.length > 0).length;
|
||||
const totalNuggets = all.reduce((s, i) => s + i.rubric.length, 0);
|
||||
|
||||
const meta = {
|
||||
dataset_version: hash,
|
||||
schema_version: SCHEMA_VERSION,
|
||||
instances_with_rubric: withRubric,
|
||||
total_nuggets: totalNuggets,
|
||||
instance_count: all.length,
|
||||
chat_size: chatSize,
|
||||
built_at: new Date().toISOString(),
|
||||
source: 'mohammadtavakoli78/BEAM (GitHub)',
|
||||
source_reference:
|
||||
'Tavakoli, Salemi, Ye, Abdalla, Zamani, Mitchell 2024, "Beyond a Million Tokens: ' +
|
||||
'Benchmarking and Enhancing Long-Term Memory in LLMs" (arXiv:2510.27246, ICLR 2026)',
|
||||
beam_chats_path: beamChatsPath,
|
||||
conversations_processed: convDirs.length,
|
||||
chat_size_dir_alias: CHAT_SIZE_DIR_MAP[chatSize]?.[0] ?? chatSize,
|
||||
canonicalisation: {
|
||||
sort_order: 'instance_id ascending',
|
||||
field_order: FIELD_ORDER,
|
||||
line_terminator: '\\n',
|
||||
trailing_newline: true,
|
||||
encoding: 'utf-8',
|
||||
no_bom: true,
|
||||
},
|
||||
distribution_by_memory_ability: byAbility,
|
||||
skip_stats: skipStats,
|
||||
};
|
||||
fs.writeFileSync(metaPath, JSON.stringify(meta, null, 2) + '\n', 'utf-8');
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Step 6: report
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
console.log('[build-beam-canonical] distribution by memory_ability:');
|
||||
for (const [k, v] of Object.entries(byAbility)) {
|
||||
console.log(` ${k}: ${v}`);
|
||||
}
|
||||
console.log('[build-beam-canonical] skip_stats:', skipStats);
|
||||
console.log(`[build-beam-canonical] wrote ${jsonlPath} (${all.length} instances)`);
|
||||
console.log(`[build-beam-canonical] wrote ${metaPath}`);
|
||||
console.log(`[build-beam-canonical] dataset_version (SHA-256): ${hash}`);
|
||||
}
|
||||
|
||||
main();
|
||||
301
benchmarks/harness/scripts/build-locomo-canonical.ts
Normal file
301
benchmarks/harness/scripts/build-locomo-canonical.ts
Normal file
@@ -0,0 +1,301 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* Canonical LoCoMo archive builder — Sprint 12 Task 1 Blocker #1.
|
||||
*
|
||||
* Reads: benchmarks/data/locomo10.json (snap-research/locomo, gitignored)
|
||||
* Writes: benchmarks/data/locomo/locomo-1540.jsonl (canonical eval set)
|
||||
* benchmarks/data/locomo/locomo-1540.meta.json (SHA-256 + count)
|
||||
*
|
||||
* Canonicalisation guarantees (required for dataset_version hash determinism):
|
||||
* 1. Include every non-adversarial QA entry (category ≠ 5) from every
|
||||
* conversation, with evidence (empty-evidence entries dropped — same rule
|
||||
* build-preflight-samples.ts applies). Adversarial is excluded per paper
|
||||
* §4.1 because it has no factual ground-truth answer.
|
||||
* 2. Sort by instance_id ascending — stable regardless of JSON key order
|
||||
* in the source file.
|
||||
* 3. Serialize each record with JSON.stringify (no spaces, explicit key
|
||||
* iteration order) and join with `\n` + trailing newline. No BOM.
|
||||
* 4. Compute SHA-256 of the final byte stream. Any drift in the source
|
||||
* or the extraction logic changes the hash and fails H-AUDIT-2
|
||||
* replication checks downstream.
|
||||
*
|
||||
* Per-instance JSONL row schema (flat, downstream-parseable):
|
||||
* {
|
||||
* "instance_id": "locomo_<sample_id>_q<3-digit>",
|
||||
* "conversation_id": "<sample_id>",
|
||||
* "question": "...",
|
||||
* "gold_answer": "...",
|
||||
* "expected": ["..."], // generic DatasetInstance contract
|
||||
* "category": "single-hop" | "multi-hop" | "temporal" | "open-ended",
|
||||
* "context": "<session-grouped evidence block>",
|
||||
* "locomo_metadata": {
|
||||
* "sample_id": "...",
|
||||
* "qa_index": <int>,
|
||||
* "locomo_category": <1|2|3|4>,
|
||||
* "evidence": ["D1:3", ...],
|
||||
* "speaker_a": "...",
|
||||
* "speaker_b": "..."
|
||||
* }
|
||||
* }
|
||||
*
|
||||
* Zero LLM calls. Zero network after locomo10.json is present. Re-running is
|
||||
* deterministic — committed archive must remain stable.
|
||||
*/
|
||||
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import url from 'node:url';
|
||||
|
||||
const CATEGORY_LABEL: Record<number, string> = {
|
||||
1: 'multi-hop',
|
||||
2: 'temporal',
|
||||
3: 'open-ended',
|
||||
4: 'single-hop',
|
||||
5: 'adversarial',
|
||||
};
|
||||
|
||||
interface LocomoTurn {
|
||||
speaker: string;
|
||||
dia_id: string;
|
||||
text: string;
|
||||
img_url?: string[];
|
||||
blip_caption?: string;
|
||||
query?: string;
|
||||
}
|
||||
|
||||
interface LocomoConversation {
|
||||
speaker_a: string;
|
||||
speaker_b: string;
|
||||
[sessionKey: string]: string | LocomoTurn[];
|
||||
}
|
||||
|
||||
interface LocomoQa {
|
||||
question: string;
|
||||
answer: string | number;
|
||||
evidence?: string[];
|
||||
category: number;
|
||||
}
|
||||
|
||||
interface LocomoSample {
|
||||
sample_id: string;
|
||||
conversation: LocomoConversation;
|
||||
qa: LocomoQa[];
|
||||
}
|
||||
|
||||
interface CanonicalInstance {
|
||||
instance_id: string;
|
||||
conversation_id: string;
|
||||
question: string;
|
||||
gold_answer: string;
|
||||
expected: string[];
|
||||
category: 'single-hop' | 'multi-hop' | 'temporal' | 'open-ended';
|
||||
context: string;
|
||||
locomo_metadata: {
|
||||
sample_id: string;
|
||||
qa_index: number;
|
||||
locomo_category: number;
|
||||
evidence: string[];
|
||||
speaker_a: string;
|
||||
speaker_b: string;
|
||||
};
|
||||
}
|
||||
|
||||
function parseDiaId(eid: string): { session: number; turn: number } | null {
|
||||
const m = eid.match(/^D(\d+):(\d+)$/);
|
||||
return m ? { session: Number(m[1]), turn: Number(m[2]) } : null;
|
||||
}
|
||||
|
||||
function buildContext(sample: LocomoSample, evidence: string[]): string {
|
||||
const bySession = new Map<number, { date: string; turns: LocomoTurn[] }>();
|
||||
for (const eid of evidence) {
|
||||
const parsed = parseDiaId(eid);
|
||||
if (!parsed) continue;
|
||||
const sessionKey = `session_${parsed.session}`;
|
||||
const dateKey = `session_${parsed.session}_date_time`;
|
||||
const session = sample.conversation[sessionKey] as LocomoTurn[] | undefined;
|
||||
const dateRaw = sample.conversation[dateKey];
|
||||
const date = typeof dateRaw === 'string' ? dateRaw : '';
|
||||
if (!session) continue;
|
||||
const turn = session.find(t => t.dia_id === eid);
|
||||
if (!turn) continue;
|
||||
if (!bySession.has(parsed.session)) {
|
||||
bySession.set(parsed.session, { date, turns: [] });
|
||||
}
|
||||
bySession.get(parsed.session)!.turns.push(turn);
|
||||
}
|
||||
const sessionNums = Array.from(bySession.keys()).sort((a, b) => a - b);
|
||||
const blocks: string[] = [];
|
||||
for (const n of sessionNums) {
|
||||
const entry = bySession.get(n)!;
|
||||
const header = entry.date ? `Session ${n} (${entry.date}):` : `Session ${n}:`;
|
||||
const lines = entry.turns.map(t => {
|
||||
const caption = t.blip_caption ? ` [image: ${t.blip_caption}]` : '';
|
||||
return `${t.speaker}: ${t.text}${caption}`;
|
||||
});
|
||||
blocks.push([header, ...lines].join('\n'));
|
||||
}
|
||||
return blocks.join('\n\n');
|
||||
}
|
||||
|
||||
function toCanonicalInstance(
|
||||
sample: LocomoSample,
|
||||
qaIndex: number,
|
||||
qa: LocomoQa,
|
||||
): CanonicalInstance | null {
|
||||
const categoryLabel = CATEGORY_LABEL[qa.category];
|
||||
if (!categoryLabel || categoryLabel === 'adversarial') return null;
|
||||
const evidence = qa.evidence ?? [];
|
||||
if (evidence.length === 0) return null;
|
||||
const context = buildContext(sample, evidence);
|
||||
if (!context) return null;
|
||||
const padded = String(qaIndex).padStart(3, '0');
|
||||
const answer = String(qa.answer);
|
||||
return {
|
||||
instance_id: `locomo_${sample.sample_id}_q${padded}`,
|
||||
conversation_id: sample.sample_id,
|
||||
question: qa.question,
|
||||
gold_answer: answer,
|
||||
expected: [answer],
|
||||
category: categoryLabel as CanonicalInstance['category'],
|
||||
context,
|
||||
locomo_metadata: {
|
||||
sample_id: sample.sample_id,
|
||||
qa_index: qaIndex,
|
||||
locomo_category: qa.category,
|
||||
evidence,
|
||||
speaker_a: sample.conversation.speaker_a,
|
||||
speaker_b: sample.conversation.speaker_b,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonical field order enforced by the serializer below. Keeps the output
|
||||
* stable even if upstream code re-orders fields on an object literal — a
|
||||
* source of silent hash drift we want to eliminate.
|
||||
*/
|
||||
const FIELD_ORDER: readonly (keyof CanonicalInstance)[] = [
|
||||
'instance_id',
|
||||
'conversation_id',
|
||||
'question',
|
||||
'gold_answer',
|
||||
'expected',
|
||||
'category',
|
||||
'context',
|
||||
'locomo_metadata',
|
||||
];
|
||||
|
||||
function serializeCanonical(inst: CanonicalInstance): string {
|
||||
const ordered: Record<string, unknown> = {};
|
||||
for (const key of FIELD_ORDER) {
|
||||
ordered[key] = inst[key];
|
||||
}
|
||||
return JSON.stringify(ordered);
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const here = url.fileURLToPath(import.meta.url);
|
||||
const harnessRoot = path.resolve(path.dirname(here), '..');
|
||||
const dataDir = path.resolve(harnessRoot, '..', 'data');
|
||||
const sourcePath = path.join(dataDir, 'locomo10.json');
|
||||
|
||||
if (!fs.existsSync(sourcePath)) {
|
||||
console.error(
|
||||
`[build-locomo-canonical] missing ${sourcePath}\n` +
|
||||
'Download with:\n' +
|
||||
' curl -sL -o benchmarks/data/locomo10.json ' +
|
||||
'https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json',
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const raw = fs.readFileSync(sourcePath, 'utf-8');
|
||||
const samples = JSON.parse(raw) as LocomoSample[];
|
||||
|
||||
const all: CanonicalInstance[] = [];
|
||||
const skipStats = { adversarial: 0, noEvidence: 0, unresolved: 0, unknownCat: 0 };
|
||||
|
||||
for (const sample of samples) {
|
||||
for (let i = 0; i < sample.qa.length; i++) {
|
||||
const qa = sample.qa[i];
|
||||
const categoryLabel = CATEGORY_LABEL[qa.category];
|
||||
if (!categoryLabel) {
|
||||
skipStats.unknownCat++;
|
||||
continue;
|
||||
}
|
||||
if (categoryLabel === 'adversarial') {
|
||||
skipStats.adversarial++;
|
||||
continue;
|
||||
}
|
||||
const evidence = qa.evidence ?? [];
|
||||
if (evidence.length === 0) {
|
||||
skipStats.noEvidence++;
|
||||
continue;
|
||||
}
|
||||
const inst = toCanonicalInstance(sample, i, qa);
|
||||
if (!inst) {
|
||||
skipStats.unresolved++;
|
||||
continue;
|
||||
}
|
||||
all.push(inst);
|
||||
}
|
||||
}
|
||||
|
||||
all.sort((a, b) => a.instance_id.localeCompare(b.instance_id));
|
||||
|
||||
const byCategory: Record<string, number> = {
|
||||
'single-hop': 0, 'multi-hop': 0, 'temporal': 0, 'open-ended': 0,
|
||||
};
|
||||
for (const inst of all) byCategory[inst.category]++;
|
||||
|
||||
const outDir = path.join(dataDir, 'locomo');
|
||||
if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true });
|
||||
|
||||
const outPath = path.join(outDir, 'locomo-1540.jsonl');
|
||||
const body = all.map(serializeCanonical).join('\n') + '\n';
|
||||
fs.writeFileSync(outPath, body, 'utf-8');
|
||||
|
||||
const hash = crypto.createHash('sha256').update(body, 'utf-8').digest('hex');
|
||||
|
||||
const metaPath = path.join(outDir, 'locomo-1540.meta.json');
|
||||
const meta = {
|
||||
dataset_version: hash,
|
||||
instance_count: all.length,
|
||||
built_at: new Date().toISOString(),
|
||||
source: 'https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json',
|
||||
source_reference:
|
||||
'Maharana et al. 2024, ACL-2024, "Evaluating Very Long-Term Conversational Memory of LLM Agents"',
|
||||
canonicalisation: {
|
||||
adversarial_excluded: true,
|
||||
no_evidence_excluded: true,
|
||||
sort_order: 'instance_id ascending',
|
||||
field_order: FIELD_ORDER,
|
||||
line_terminator: '\\n',
|
||||
trailing_newline: true,
|
||||
encoding: 'utf-8',
|
||||
no_bom: true,
|
||||
},
|
||||
distribution: byCategory,
|
||||
skip_stats: skipStats,
|
||||
paper_total_claim: 1540,
|
||||
actual_count: all.length,
|
||||
count_matches_paper: all.length === 1540,
|
||||
};
|
||||
fs.writeFileSync(metaPath, JSON.stringify(meta, null, 2) + '\n', 'utf-8');
|
||||
|
||||
console.log('[build-locomo-canonical] distribution:');
|
||||
for (const [k, v] of Object.entries(byCategory)) console.log(` ${k}: ${v}`);
|
||||
console.log('[build-locomo-canonical] skipped:', skipStats);
|
||||
console.log(`[build-locomo-canonical] wrote ${outPath} (${all.length} instances)`);
|
||||
console.log(`[build-locomo-canonical] wrote ${metaPath}`);
|
||||
console.log(`[build-locomo-canonical] dataset_version (SHA-256): ${hash}`);
|
||||
if (all.length !== 1540) {
|
||||
console.warn(
|
||||
`[build-locomo-canonical] NOTE: actual count ${all.length} differs from paper claim 1540. ` +
|
||||
'Filename retained per brief; see meta.json for provenance.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
445
benchmarks/harness/scripts/build-longmemeval-canonical.ts
Normal file
445
benchmarks/harness/scripts/build-longmemeval-canonical.ts
Normal file
@@ -0,0 +1,445 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* Canonical LongMemEval V1 archive builder — Track A0 (manifest-v8.2-final.md).
|
||||
*
|
||||
* Reads: benchmarks/data/longmemeval_s_cleaned.json (gitignored, _s default)
|
||||
* or benchmarks/data/longmemeval_m_cleaned.json (--variant m)
|
||||
* Writes: benchmarks/data/longmemeval/longmemeval.jsonl (canonical JSONL)
|
||||
* benchmarks/data/longmemeval/longmemeval.meta.json (SHA-256 + count + distribution)
|
||||
* benchmarks/data/longmemeval/longmemeval_s_cleaned.json (raw cache copy)
|
||||
*
|
||||
* Canonicalisation guarantees (required for dataset_version hash determinism):
|
||||
* 1. Include every question from the cleaned dataset (all question_types).
|
||||
* 2. Sort by instance_id ascending — stable regardless of JSON key order in source.
|
||||
* 3. Serialize each record with JSON.stringify (no spaces, explicit key iteration
|
||||
* order) and join with `\n` + trailing newline. No BOM.
|
||||
* 4. Compute SHA-256 of the final byte stream. Any drift in the source or
|
||||
* extraction logic changes the hash and fails replication checks downstream.
|
||||
* 5. Abstention questions (question_id ending in '_abs') are tagged but included.
|
||||
*
|
||||
* Per-instance JSONL row schema (flat, downstream-parseable by DatasetInstance):
|
||||
* {
|
||||
* "instance_id": "longmemeval_<question_id>",
|
||||
* "conversation_id": "<question_id>",
|
||||
* "question": "...",
|
||||
* "expected": ["<answer>"],
|
||||
* "context": "<sessions concatenated as formatted text>",
|
||||
* "question_type": "knowledge-update" | "temporal-reasoning" | ...,
|
||||
* "is_abstention": false
|
||||
* }
|
||||
*
|
||||
* Source: xiaowu0162/longmemeval-cleaned on Hugging Face (Apache 2.0 or CC-BY)
|
||||
* Paper: Wu et al. 2024, "LongMemEval: Benchmarking Chat Assistants on Long-Term
|
||||
* Interactive Memory" (arXiv:2410.10813).
|
||||
*
|
||||
* Zero LLM calls. Zero npm packages beyond Node.js built-ins.
|
||||
*
|
||||
* Usage:
|
||||
* tsx build-longmemeval-canonical.ts [--variant s|m] [--skip-download]
|
||||
*/
|
||||
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import https from 'node:https';
|
||||
import path from 'node:path';
|
||||
import process from 'node:process';
|
||||
import url from 'node:url';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Source URLs and paths
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const VARIANT_URLS: Record<string, string> = {
|
||||
s: 'https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/longmemeval_s_cleaned.json',
|
||||
m: 'https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/longmemeval_m_cleaned.json',
|
||||
};
|
||||
|
||||
const QUESTION_TYPES = [
|
||||
'single-session-user',
|
||||
'single-session-assistant',
|
||||
'single-session-preference',
|
||||
'temporal-reasoning',
|
||||
'knowledge-update',
|
||||
'multi-session',
|
||||
] as const;
|
||||
type QuestionType = (typeof QUESTION_TYPES)[number];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Source schema (from paper/repo xiaowu0162/longmemeval-cleaned)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface LongMemEvalMessage {
|
||||
role: 'user' | 'assistant';
|
||||
content: string;
|
||||
}
|
||||
|
||||
interface LongMemEvalSession {
|
||||
session_id: string;
|
||||
date?: string;
|
||||
messages: LongMemEvalMessage[];
|
||||
}
|
||||
|
||||
interface LongMemEvalQuestion {
|
||||
question_id: string;
|
||||
question: string;
|
||||
answer: string;
|
||||
question_type: QuestionType;
|
||||
// Abstention variant: question_id ends with '_abs'
|
||||
// Primary schema (cleaned HuggingFace variant):
|
||||
sessions?: LongMemEvalSession[];
|
||||
// Actual cleaned-variant schema:
|
||||
// haystack_sessions: list[list[{role, content}]]
|
||||
// haystack_dates: list[str]
|
||||
// haystack_session_ids: list[str]
|
||||
haystack_sessions?: LongMemEvalMessage[][];
|
||||
haystack_dates?: string[];
|
||||
haystack_session_ids?: string[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Output schema
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface CanonicalInstance {
|
||||
instance_id: string;
|
||||
conversation_id: string;
|
||||
question: string;
|
||||
expected: string[];
|
||||
context: string;
|
||||
question_type: QuestionType;
|
||||
is_abstention: boolean;
|
||||
}
|
||||
|
||||
const FIELD_ORDER: readonly (keyof CanonicalInstance)[] = [
|
||||
'instance_id',
|
||||
'conversation_id',
|
||||
'question',
|
||||
'expected',
|
||||
'context',
|
||||
'question_type',
|
||||
'is_abstention',
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Context assembly
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Concatenate all sessions into a single context string.
|
||||
*
|
||||
* Format per session:
|
||||
* Session N (YYYY-MM-DD):
|
||||
* user: ...
|
||||
* assistant: ...
|
||||
*
|
||||
* Sessions without a date omit the parenthetical. Separated by double newline.
|
||||
*/
|
||||
function buildContext(sessions: LongMemEvalSession[]): string {
|
||||
const blocks: string[] = [];
|
||||
for (let i = 0; i < sessions.length; i++) {
|
||||
const s = sessions[i];
|
||||
const n = i + 1;
|
||||
const header = s.date ? `Session ${n} (${s.date}):` : `Session ${n}:`;
|
||||
const lines = s.messages.map(m => `${m.role}: ${m.content}`);
|
||||
blocks.push([header, ...lines].join('\n'));
|
||||
}
|
||||
return blocks.join('\n\n');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Serialisation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function serializeCanonical(inst: CanonicalInstance): string {
|
||||
const ordered: Record<string, unknown> = {};
|
||||
for (const key of FIELD_ORDER) {
|
||||
ordered[key] = inst[key];
|
||||
}
|
||||
return JSON.stringify(ordered);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Download
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function downloadFile(remoteUrl: string, destPath: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
console.log(`[build-longmemeval-canonical] downloading ${remoteUrl}`);
|
||||
console.log(`[build-longmemeval-canonical] → ${destPath}`);
|
||||
|
||||
const file = fs.createWriteStream(destPath);
|
||||
let received = 0;
|
||||
let total = 0;
|
||||
let lastPct = -1;
|
||||
|
||||
function doGet(requestUrl: string): void {
|
||||
https
|
||||
.get(requestUrl, res => {
|
||||
if (res.statusCode === 301 || res.statusCode === 302 || res.statusCode === 307 || res.statusCode === 308) {
|
||||
const location = res.headers.location;
|
||||
if (!location) {
|
||||
reject(new Error(`Redirect with no Location header (${res.statusCode})`));
|
||||
return;
|
||||
}
|
||||
doGet(location);
|
||||
return;
|
||||
}
|
||||
if (res.statusCode !== 200) {
|
||||
reject(new Error(`HTTP ${res.statusCode} for ${requestUrl}`));
|
||||
return;
|
||||
}
|
||||
total = parseInt(res.headers['content-length'] ?? '0', 10);
|
||||
res.on('data', (chunk: Buffer) => {
|
||||
received += chunk.length;
|
||||
if (total > 0) {
|
||||
const pct = Math.floor((received / total) * 100);
|
||||
if (pct !== lastPct && pct % 10 === 0) {
|
||||
process.stdout.write(` ${pct}% (${(received / 1024 / 1024).toFixed(1)} MB)\r`);
|
||||
lastPct = pct;
|
||||
}
|
||||
}
|
||||
});
|
||||
res.pipe(file);
|
||||
res.on('end', () => {
|
||||
file.end();
|
||||
});
|
||||
})
|
||||
.on('error', reject);
|
||||
}
|
||||
|
||||
file.on('finish', () => {
|
||||
process.stdout.write('\n');
|
||||
console.log(
|
||||
`[build-longmemeval-canonical] download complete (${(received / 1024 / 1024).toFixed(2)} MB)`,
|
||||
);
|
||||
resolve();
|
||||
});
|
||||
file.on('error', reject);
|
||||
|
||||
doGet(remoteUrl);
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CLI arg parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function parseArgs(): { variant: string; skipDownload: boolean } {
|
||||
const argv = process.argv.slice(2);
|
||||
let variant = 's';
|
||||
let skipDownload = false;
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
if (argv[i] === '--variant' && argv[i + 1]) {
|
||||
variant = argv[++i];
|
||||
} else if (argv[i] === '--skip-download') {
|
||||
skipDownload = true;
|
||||
}
|
||||
}
|
||||
if (variant !== 's' && variant !== 'm') {
|
||||
console.error(`[build-longmemeval-canonical] unknown --variant "${variant}". Use s or m.`);
|
||||
process.exit(1);
|
||||
}
|
||||
return { variant, skipDownload };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const { variant, skipDownload } = parseArgs();
|
||||
|
||||
const here = url.fileURLToPath(import.meta.url);
|
||||
// Script lives at benchmarks/harness/scripts/build-longmemeval-canonical.ts
|
||||
// Resolve repo root by going 3 levels up: scripts/ → harness/ → benchmarks/ → repo root
|
||||
const scriptDir = path.dirname(here);
|
||||
const repoRoot = path.resolve(scriptDir, '..', '..', '..');
|
||||
const dataDir = path.resolve(repoRoot, 'benchmarks', 'data');
|
||||
|
||||
const rawFilename = `longmemeval_${variant}_cleaned.json`;
|
||||
const rawPath = path.join(dataDir, rawFilename);
|
||||
const remoteUrl = VARIANT_URLS[variant];
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Step 1: acquire raw file
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
if (!skipDownload && !fs.existsSync(rawPath)) {
|
||||
if (!fs.existsSync(dataDir)) fs.mkdirSync(dataDir, { recursive: true });
|
||||
try {
|
||||
await downloadFile(remoteUrl, rawPath);
|
||||
} catch (err) {
|
||||
console.error(`[build-longmemeval-canonical] download failed: ${String(err)}`);
|
||||
console.error('');
|
||||
console.error('Retry manually with:');
|
||||
console.error(` curl -sL -o ${rawPath} '${remoteUrl}'`);
|
||||
process.exit(2);
|
||||
}
|
||||
} else if (skipDownload && !fs.existsSync(rawPath)) {
|
||||
console.error(
|
||||
`[build-longmemeval-canonical] --skip-download set but raw file missing: ${rawPath}`,
|
||||
);
|
||||
console.error('Download with:');
|
||||
console.error(` curl -sL -o ${rawPath} '${remoteUrl}'`);
|
||||
process.exit(2);
|
||||
} else {
|
||||
console.log(`[build-longmemeval-canonical] using cached raw file: ${rawPath}`);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Step 2: parse source JSON
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
console.log('[build-longmemeval-canonical] parsing source JSON …');
|
||||
const raw = fs.readFileSync(rawPath, 'utf-8');
|
||||
let questions: LongMemEvalQuestion[];
|
||||
try {
|
||||
questions = JSON.parse(raw) as LongMemEvalQuestion[];
|
||||
} catch (err) {
|
||||
console.error(`[build-longmemeval-canonical] JSON parse error: ${String(err)}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!Array.isArray(questions)) {
|
||||
console.error('[build-longmemeval-canonical] expected top-level JSON array, got something else.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`[build-longmemeval-canonical] loaded ${questions.length} questions from source`);
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Step 3: convert to canonical instances
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
const all: CanonicalInstance[] = [];
|
||||
const skipStats = { missingFields: 0, noSessions: 0 };
|
||||
|
||||
for (const q of questions) {
|
||||
if (!q.question_id || !q.question || q.answer === undefined || q.answer === null) {
|
||||
skipStats.missingFields++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Normalise to LongMemEvalSession[]: handle both schema variants.
|
||||
// Variant A (original): sessions: [{session_id, date?, messages: [{role, content}]}]
|
||||
// Variant B (cleaned HF): haystack_sessions: list[list[{role,content}]],
|
||||
// haystack_dates: list[str], haystack_session_ids: list[str]
|
||||
let normalisedSessions: LongMemEvalSession[] | null = null;
|
||||
|
||||
if (Array.isArray(q.sessions) && q.sessions.length > 0) {
|
||||
normalisedSessions = q.sessions;
|
||||
} else if (Array.isArray(q.haystack_sessions) && q.haystack_sessions.length > 0) {
|
||||
normalisedSessions = q.haystack_sessions.map((msgs, i) => ({
|
||||
session_id: q.haystack_session_ids?.[i] ?? `session_${i}`,
|
||||
date: q.haystack_dates?.[i],
|
||||
messages: msgs.filter(m => m && typeof m.content === 'string'),
|
||||
}));
|
||||
}
|
||||
|
||||
if (!normalisedSessions || normalisedSessions.length === 0) {
|
||||
skipStats.noSessions++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const isAbstention = q.question_id.endsWith('_abs');
|
||||
const context = buildContext(normalisedSessions);
|
||||
|
||||
all.push({
|
||||
instance_id: `longmemeval_${q.question_id}`,
|
||||
conversation_id: q.question_id,
|
||||
question: q.question,
|
||||
expected: [q.answer],
|
||||
context,
|
||||
question_type: q.question_type,
|
||||
is_abstention: isAbstention,
|
||||
});
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Step 4: sort + distribution
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
all.sort((a, b) => a.instance_id.localeCompare(b.instance_id));
|
||||
|
||||
const byType: Record<string, number> = {};
|
||||
for (const qt of QUESTION_TYPES) byType[qt] = 0;
|
||||
for (const inst of all) {
|
||||
byType[inst.question_type] = (byType[inst.question_type] ?? 0) + 1;
|
||||
}
|
||||
const abstentionCount = all.filter(i => i.is_abstention).length;
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Step 5: write outputs
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
const outDir = path.join(dataDir, 'longmemeval');
|
||||
if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true });
|
||||
|
||||
// 5a. Canonical JSONL
|
||||
const jsonlPath = path.join(outDir, 'longmemeval.jsonl');
|
||||
const body = all.map(serializeCanonical).join('\n') + '\n';
|
||||
fs.writeFileSync(jsonlPath, body, 'utf-8');
|
||||
|
||||
// 5b. SHA-256
|
||||
const hash = crypto.createHash('sha256').update(body, 'utf-8').digest('hex');
|
||||
|
||||
// 5c. meta.json
|
||||
const metaPath = path.join(outDir, 'longmemeval.meta.json');
|
||||
const meta = {
|
||||
dataset_version: hash,
|
||||
instance_count: all.length,
|
||||
variant,
|
||||
built_at: new Date().toISOString(),
|
||||
source: remoteUrl,
|
||||
source_reference:
|
||||
'Wu et al. 2024, "LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory" (arXiv:2410.10813)',
|
||||
hf_repo: 'xiaowu0162/longmemeval-cleaned',
|
||||
canonicalisation: {
|
||||
abstention_included: true,
|
||||
no_sessions_excluded: true,
|
||||
sort_order: 'instance_id ascending',
|
||||
field_order: FIELD_ORDER,
|
||||
line_terminator: '\\n',
|
||||
trailing_newline: true,
|
||||
encoding: 'utf-8',
|
||||
no_bom: true,
|
||||
},
|
||||
distribution_by_question_type: byType,
|
||||
abstention_count: abstentionCount,
|
||||
skip_stats: skipStats,
|
||||
expected_count: variant === 's' ? 500 : null,
|
||||
count_matches_expected: variant === 's' ? all.length === 500 : null,
|
||||
};
|
||||
fs.writeFileSync(metaPath, JSON.stringify(meta, null, 2) + '\n', 'utf-8');
|
||||
|
||||
// 5d. Raw cache copy alongside canonical outputs
|
||||
const rawCachePath = path.join(outDir, rawFilename);
|
||||
if (!fs.existsSync(rawCachePath)) {
|
||||
fs.copyFileSync(rawPath, rawCachePath);
|
||||
console.log(`[build-longmemeval-canonical] cached raw → ${rawCachePath}`);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Step 6: report
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
console.log('[build-longmemeval-canonical] distribution by question_type:');
|
||||
for (const [k, v] of Object.entries(byType)) console.log(` ${k}: ${v}`);
|
||||
console.log(`[build-longmemeval-canonical] abstention questions: ${abstentionCount}`);
|
||||
console.log('[build-longmemeval-canonical] skipped:', skipStats);
|
||||
console.log(`[build-longmemeval-canonical] wrote ${jsonlPath} (${all.length} instances)`);
|
||||
console.log(`[build-longmemeval-canonical] wrote ${metaPath}`);
|
||||
console.log(`[build-longmemeval-canonical] dataset_version (SHA-256): ${hash}`);
|
||||
|
||||
if (variant === 's' && all.length !== 500) {
|
||||
console.warn(
|
||||
`[build-longmemeval-canonical] NOTE: expected 500 instances for _s variant, got ${all.length}. ` +
|
||||
'Check source file integrity.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('[build-longmemeval-canonical] fatal:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
354
benchmarks/harness/scripts/build-preflight-samples.ts
Normal file
354
benchmarks/harness/scripts/build-preflight-samples.ts
Normal file
@@ -0,0 +1,354 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* Sample-lock builder for the Stage 2 preflight gate + failure-mode calibration.
|
||||
*
|
||||
* Reads: benchmarks/data/locomo10.json (snap-research/locomo, gitignored)
|
||||
* Writes: benchmarks/data/preflight-locomo-50.json (Task 1 — seed=42)
|
||||
* benchmarks/data/failure-mode-calibration-10.jsonl (Task 2 — seed=43)
|
||||
*
|
||||
* Selection algorithm (deterministic):
|
||||
* 1. Walk all 10 LoCoMo conversations; for each QA entry, mint a stable
|
||||
* `instance_id` of the form `locomo_<sample_id>_q<3-digit-index>` where
|
||||
* index is the 0-based position within that sample's `qa` array.
|
||||
* 2. Bucket by `category` (1=multi-hop, 2=temporal, 3=open-domain,
|
||||
* 4=single-hop, 5=adversarial — verified against LoCoMo evaluation.py
|
||||
* line 208-217 + ACL-2024 paper §4.1). Skip category 5 (adversarial,
|
||||
* out of scope for 4-way MECE split).
|
||||
* 3. Sort each bucket by instance_id ascending (canonical order).
|
||||
* 4. Fisher-Yates shuffle each bucket with xorshift32(seed). Same PRNG
|
||||
* family as benchmarks/harness/src/datasets.ts → one shuffle convention
|
||||
* across the harness.
|
||||
* 5. Take first N per category per task's distribution.
|
||||
* 6. Build context from evidence dia_ids, grouped by session (with session
|
||||
* date) so the preserved metadata is faithful to what the model needs.
|
||||
*
|
||||
* Non-overlap guarantee: Task 2 (seed=43) removes Task 1's instance_ids from
|
||||
* each bucket BEFORE the shuffle, so the two samples are provably disjoint
|
||||
* regardless of PRNG state.
|
||||
*
|
||||
* Zero LLM calls. Zero network after locomo10.json is present. Re-running is
|
||||
* deterministic — committed lock files are stable.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import url from 'node:url';
|
||||
|
||||
// ── LoCoMo category mapping (verified against evaluation.py + ACL paper) ──
|
||||
const CATEGORY_LABEL: Record<number, string> = {
|
||||
1: 'multi-hop', // eval.py line 213: `elif line['category'] in [1]`: multi-hop
|
||||
2: 'temporal', // all "When did X..." questions with date answers
|
||||
3: 'open-ended', // open-domain / commonsense / inferential (paper §4.1)
|
||||
4: 'single-hop', // simple factoid from single evidence turn
|
||||
5: 'adversarial', // unanswerable — excluded from the 4-way split
|
||||
};
|
||||
|
||||
interface LocomoTurn {
|
||||
speaker: string;
|
||||
dia_id: string;
|
||||
text: string;
|
||||
img_url?: string[];
|
||||
blip_caption?: string;
|
||||
query?: string;
|
||||
}
|
||||
|
||||
interface LocomoConversation {
|
||||
speaker_a: string;
|
||||
speaker_b: string;
|
||||
[sessionKey: string]: string | LocomoTurn[];
|
||||
}
|
||||
|
||||
interface LocomoQa {
|
||||
question: string;
|
||||
answer: string | number;
|
||||
evidence?: string[];
|
||||
category: number;
|
||||
}
|
||||
|
||||
interface LocomoSample {
|
||||
sample_id: string;
|
||||
conversation: LocomoConversation;
|
||||
qa: LocomoQa[];
|
||||
event_summary?: unknown;
|
||||
observation?: unknown;
|
||||
session_summary?: unknown;
|
||||
}
|
||||
|
||||
interface PreflightInstance {
|
||||
id: string;
|
||||
category: 'single-hop' | 'multi-hop' | 'temporal' | 'open-ended';
|
||||
context: string;
|
||||
question: string;
|
||||
ground_truth_answer: string;
|
||||
locomo_metadata: {
|
||||
sample_id: string;
|
||||
qa_index: number;
|
||||
locomo_category: number;
|
||||
evidence: string[];
|
||||
speaker_a: string;
|
||||
speaker_b: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface CalibrationInstance extends PreflightInstance {
|
||||
human_label: {
|
||||
verdict: null;
|
||||
failure_mode: null;
|
||||
rationale: null;
|
||||
};
|
||||
}
|
||||
|
||||
// xorshift32 — same PRNG family as benchmarks/harness/src/datasets.ts.
|
||||
function makeRng(seed: number): () => number {
|
||||
let state = (seed || 1) >>> 0;
|
||||
return () => {
|
||||
state ^= state << 13;
|
||||
state ^= state >>> 17;
|
||||
state ^= state << 5;
|
||||
return (state >>> 0) / 0x100000000;
|
||||
};
|
||||
}
|
||||
|
||||
function fisherYates<T>(items: readonly T[], rand: () => number): T[] {
|
||||
const out = items.slice();
|
||||
for (let i = out.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rand() * (i + 1));
|
||||
[out[i], out[j]] = [out[j], out[i]];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function parseDiaId(eid: string): { session: number; turn: number } | null {
|
||||
const m = eid.match(/^D(\d+):(\d+)$/);
|
||||
if (!m) return null;
|
||||
return { session: Number(m[1]), turn: Number(m[2]) };
|
||||
}
|
||||
|
||||
function buildContext(sample: LocomoSample, evidence: string[]): string {
|
||||
// Group evidence turns by session so the temporal anchor (session_N_date_time)
|
||||
// can be emitted once per session. Preserves the minimum information needed
|
||||
// to answer the question without dumping the whole conversation.
|
||||
const bySession = new Map<number, { date: string; turns: LocomoTurn[] }>();
|
||||
|
||||
for (const eid of evidence) {
|
||||
const parsed = parseDiaId(eid);
|
||||
if (!parsed) continue;
|
||||
const sessionKey = `session_${parsed.session}`;
|
||||
const dateKey = `session_${parsed.session}_date_time`;
|
||||
const session = sample.conversation[sessionKey] as LocomoTurn[] | undefined;
|
||||
const dateRaw = sample.conversation[dateKey];
|
||||
const date = typeof dateRaw === 'string' ? dateRaw : '';
|
||||
if (!session) continue;
|
||||
const turn = session.find(t => t.dia_id === eid);
|
||||
if (!turn) continue;
|
||||
if (!bySession.has(parsed.session)) {
|
||||
bySession.set(parsed.session, { date, turns: [] });
|
||||
}
|
||||
bySession.get(parsed.session)!.turns.push(turn);
|
||||
}
|
||||
|
||||
const sessionNums = Array.from(bySession.keys()).sort((a, b) => a - b);
|
||||
const blocks: string[] = [];
|
||||
for (const n of sessionNums) {
|
||||
const entry = bySession.get(n)!;
|
||||
const header = entry.date ? `Session ${n} (${entry.date}):` : `Session ${n}:`;
|
||||
const lines = entry.turns.map(t => {
|
||||
const caption = t.blip_caption ? ` [image: ${t.blip_caption}]` : '';
|
||||
return `${t.speaker}: ${t.text}${caption}`;
|
||||
});
|
||||
blocks.push([header, ...lines].join('\n'));
|
||||
}
|
||||
return blocks.join('\n\n');
|
||||
}
|
||||
|
||||
function toPreflightInstance(sample: LocomoSample, qaIndex: number, qa: LocomoQa): PreflightInstance | null {
|
||||
const category = CATEGORY_LABEL[qa.category];
|
||||
if (!category || category === 'adversarial') return null;
|
||||
const evidence = qa.evidence ?? [];
|
||||
if (evidence.length === 0) return null; // defensive: no evidence → no context
|
||||
const context = buildContext(sample, evidence);
|
||||
if (!context) return null; // evidence points to turns we can't resolve
|
||||
const padded = String(qaIndex).padStart(3, '0');
|
||||
return {
|
||||
id: `locomo_${sample.sample_id}_q${padded}`,
|
||||
category: category as PreflightInstance['category'],
|
||||
context,
|
||||
question: qa.question,
|
||||
ground_truth_answer: String(qa.answer),
|
||||
locomo_metadata: {
|
||||
sample_id: sample.sample_id,
|
||||
qa_index: qaIndex,
|
||||
locomo_category: qa.category,
|
||||
evidence,
|
||||
speaker_a: sample.conversation.speaker_a,
|
||||
speaker_b: sample.conversation.speaker_b,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function bucketByCategory(instances: PreflightInstance[]): Record<string, PreflightInstance[]> {
|
||||
const buckets: Record<string, PreflightInstance[]> = {
|
||||
'single-hop': [],
|
||||
'multi-hop': [],
|
||||
'temporal': [],
|
||||
'open-ended': [],
|
||||
};
|
||||
for (const inst of instances) buckets[inst.category].push(inst);
|
||||
for (const key of Object.keys(buckets)) {
|
||||
buckets[key].sort((a, b) => a.id.localeCompare(b.id));
|
||||
}
|
||||
return buckets;
|
||||
}
|
||||
|
||||
function pickStratified(
|
||||
buckets: Record<string, PreflightInstance[]>,
|
||||
distribution: Record<string, number>,
|
||||
seed: number,
|
||||
exclude: Set<string>,
|
||||
): PreflightInstance[] {
|
||||
const rand = makeRng(seed);
|
||||
const out: PreflightInstance[] = [];
|
||||
// Stable key order so the same seed always consumes the RNG in the same way.
|
||||
for (const key of ['single-hop', 'multi-hop', 'temporal', 'open-ended']) {
|
||||
const pool = buckets[key].filter(i => !exclude.has(i.id));
|
||||
const shuffled = fisherYates(pool, rand);
|
||||
const need = distribution[key];
|
||||
if (shuffled.length < need) {
|
||||
throw new Error(
|
||||
`category ${key} has ${shuffled.length} usable instances after exclusions, need ${need}`,
|
||||
);
|
||||
}
|
||||
out.push(...shuffled.slice(0, need));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const here = url.fileURLToPath(import.meta.url);
|
||||
const harnessRoot = path.resolve(path.dirname(here), '..');
|
||||
const dataDir = path.resolve(harnessRoot, '..', 'data');
|
||||
const sourcePath = path.join(dataDir, 'locomo10.json');
|
||||
|
||||
if (!fs.existsSync(sourcePath)) {
|
||||
console.error(
|
||||
`[build-preflight-samples] missing ${sourcePath}\n` +
|
||||
'Download with:\n' +
|
||||
' curl -sL -o benchmarks/data/locomo10.json ' +
|
||||
'https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json',
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const raw = fs.readFileSync(sourcePath, 'utf-8');
|
||||
const samples = JSON.parse(raw) as LocomoSample[];
|
||||
|
||||
const allInstances: PreflightInstance[] = [];
|
||||
for (const sample of samples) {
|
||||
for (let i = 0; i < sample.qa.length; i++) {
|
||||
const inst = toPreflightInstance(sample, i, sample.qa[i]);
|
||||
if (inst) allInstances.push(inst);
|
||||
}
|
||||
}
|
||||
|
||||
const buckets = bucketByCategory(allInstances);
|
||||
console.log('[build-preflight-samples] bucket sizes (after excluding adversarial + evidence-less):');
|
||||
for (const key of ['single-hop', 'multi-hop', 'temporal', 'open-ended']) {
|
||||
console.log(` ${key}: ${buckets[key].length}`);
|
||||
}
|
||||
|
||||
// Task 1 — Stage 2 sample lock (seed=42, 13/13/12/12)
|
||||
const stage2 = pickStratified(
|
||||
buckets,
|
||||
{ 'single-hop': 13, 'multi-hop': 13, 'temporal': 12, 'open-ended': 12 },
|
||||
42,
|
||||
new Set(),
|
||||
);
|
||||
const stage2Ids = new Set(stage2.map(i => i.id));
|
||||
|
||||
// Task 2 — failure-mode calibration (seed=43, 3/3/2/2, non-overlapping with Task 1)
|
||||
const calibration = pickStratified(
|
||||
buckets,
|
||||
{ 'single-hop': 3, 'multi-hop': 3, 'temporal': 2, 'open-ended': 2 },
|
||||
43,
|
||||
stage2Ids,
|
||||
);
|
||||
for (const inst of calibration) {
|
||||
if (stage2Ids.has(inst.id)) {
|
||||
throw new Error(`calibration set overlaps stage-2 lock: ${inst.id}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Write Task 1: preflight-locomo-50.json ───────────────────────────
|
||||
const stage2Output = {
|
||||
_meta: {
|
||||
description:
|
||||
'Stage 2 preflight 4-cell sample lock. Istih 50 LoCoMo instanci preko ' +
|
||||
'sva 4 ćelije (raw / memory-only / evolve-only / full-stack).',
|
||||
brief: 'PM-Waggle-OS/briefs/2026-04-20-cc-preflight-prep-tasks.md Task 1',
|
||||
locked_decision: 'decisions/2026-04-20-preflight-oq-resolutions-locked.md §OQ-PF-1',
|
||||
source: 'https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json',
|
||||
source_reference: 'Maharana et al. 2024, ACL-2024, "Evaluating Very Long-Term Conversational Memory of LLM Agents"',
|
||||
seed: 42,
|
||||
selection_algorithm:
|
||||
'Bucket LoCoMo qa entries by category (mapping verified via task_eval/evaluation.py + paper §4.1); ' +
|
||||
'within each category sort by instance_id ascending (canonical order), apply Fisher-Yates shuffle ' +
|
||||
'with xorshift32(seed=42), then take first N per category. Fisher-Yates PRNG is shared across ' +
|
||||
'buckets — key iteration order is fixed (single-hop, multi-hop, temporal, open-ended) to keep the ' +
|
||||
'selection stable against re-runs. instance_id = locomo_<sample_id>_q<3-digit qa-array index>.',
|
||||
distribution: { 'single-hop': 13, 'multi-hop': 13, 'temporal': 12, 'open-ended': 12 },
|
||||
total: 50,
|
||||
locomo_category_map: {
|
||||
'1': 'multi-hop',
|
||||
'2': 'temporal',
|
||||
'3': 'open-ended',
|
||||
'4': 'single-hop',
|
||||
'5': 'adversarial (excluded)',
|
||||
},
|
||||
context_assembly:
|
||||
'Evidence dia_ids are grouped by session, prefixed with session_N_date_time for temporal ' +
|
||||
'anchoring, and rendered as "speaker: text" lines. Images are preserved via blip_caption tags.',
|
||||
},
|
||||
instances: stage2,
|
||||
};
|
||||
const stage2Path = path.join(dataDir, 'preflight-locomo-50.json');
|
||||
fs.writeFileSync(stage2Path, JSON.stringify(stage2Output, null, 2) + '\n', 'utf-8');
|
||||
console.log(`[build-preflight-samples] wrote ${stage2Path} (${stage2.length} instances)`);
|
||||
|
||||
// ── Write Task 2: failure-mode-calibration-10.jsonl ──────────────────
|
||||
const calibrationPath = path.join(dataDir, 'failure-mode-calibration-10.jsonl');
|
||||
const header = [
|
||||
'# Failure-mode judge calibration set',
|
||||
'# brief: PM-Waggle-OS/briefs/2026-04-20-cc-preflight-prep-tasks.md Task 2',
|
||||
'# locked: decisions/2026-04-20-failure-mode-oq-resolutions-locked.md §OQ-FM-3',
|
||||
'# source: https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json',
|
||||
'# seed=43 (non-overlapping with preflight-locomo-50.json seed=42)',
|
||||
'# distribution: single-hop=3, multi-hop=3, temporal=2, open-ended=2 (total=10)',
|
||||
'# human_label.{verdict,failure_mode,rationale} are left null. PM labels first pass; CC validates second pass.',
|
||||
'# Judge activates Stage 1 only after ≥8/10 match against human_label.',
|
||||
'',
|
||||
].join('\n');
|
||||
|
||||
const lines: string[] = [];
|
||||
for (const inst of calibration) {
|
||||
const withLabel: CalibrationInstance = {
|
||||
...inst,
|
||||
human_label: { verdict: null, failure_mode: null, rationale: null },
|
||||
};
|
||||
lines.push(JSON.stringify(withLabel));
|
||||
}
|
||||
fs.writeFileSync(calibrationPath, header + lines.join('\n') + '\n', 'utf-8');
|
||||
console.log(`[build-preflight-samples] wrote ${calibrationPath} (${calibration.length} instances)`);
|
||||
|
||||
// Distribution + overlap summary.
|
||||
const countByCat = (items: PreflightInstance[]): Record<string, number> => {
|
||||
const out: Record<string, number> = {};
|
||||
for (const i of items) out[i.category] = (out[i.category] ?? 0) + 1;
|
||||
return out;
|
||||
};
|
||||
console.log('[build-preflight-samples] stage-2 distribution:', countByCat(stage2));
|
||||
console.log('[build-preflight-samples] calibration distribution:', countByCat(calibration));
|
||||
const overlap = calibration.filter(i => stage2Ids.has(i.id)).length;
|
||||
console.log(`[build-preflight-samples] overlap (must be 0): ${overlap}`);
|
||||
}
|
||||
|
||||
main();
|
||||
781
benchmarks/harness/scripts/run-v8.ts
Normal file
781
benchmarks/harness/scripts/run-v8.ts
Normal file
@@ -0,0 +1,781 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* v8 Multi-Benchmark Runner — manifest-v8.2-final.md
|
||||
*
|
||||
* Executes the 4-track v8 ablation programme in strict order:
|
||||
*
|
||||
* Track A0 — LongMemEval V1 (500 questions, 4 cells, $20 budget)
|
||||
* Track A — BEAM 128K (~300 questions, 4 cells, $50 budget)
|
||||
* Track B — GAIA 2 (BLOCKED: requires WSL2 / SIGALRM fix)
|
||||
* Track D — Terminal-Bench (external infra, zero cost)
|
||||
*
|
||||
* 4-cell ablation grid per track:
|
||||
* no-context → zero-memory baseline
|
||||
* retrieval → HybridSearch recall only
|
||||
* hive_mind_ipb → HybridSearch + I/P/B frame writes (primary treatment)
|
||||
* hive_mind_ipb_strong → same as hive_mind_ipb, Opus 4.x model
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx benchmarks/harness/scripts/run-v8.ts [options]
|
||||
*
|
||||
* Options:
|
||||
* --track a0|a|b|d|all Which track(s) to run (default: all)
|
||||
* --limit N Instance count cap per cell (default: full)
|
||||
* --budget-a0 USD Hard USD cap for Track A0 (default: 20)
|
||||
* --budget-a USD Hard USD cap for Track A (default: 50)
|
||||
* --model-primary ID Primary subject model (default: qwen3.6-35b-a3b)
|
||||
* --model-strong ID Strong subject model (default: claude-opus-4-x)
|
||||
* --lme-data-path P Path to longmemeval.jsonl (default: auto-discover)
|
||||
* --beam-data-path P Path to beam-128K.jsonl (default: auto-discover)
|
||||
* --dry-run Stub LLM calls
|
||||
* --no-ipb-strong Skip hive_mind_ipb_strong cell (saves Opus spend)
|
||||
* --judge ID Enable per-instance judge (default: none)
|
||||
* --seed N PRNG seed (default: 42)
|
||||
*
|
||||
* Env:
|
||||
* LITELLM_URL default http://localhost:4000
|
||||
* LITELLM_API_KEY default sk-waggle-dev
|
||||
*
|
||||
* Output:
|
||||
* benchmarks/results/v8/<track>/<cell>-<dataset>-<ts>.jsonl
|
||||
* benchmarks/results/v8/<track>/<cell>-<dataset>-<ts>.summary.json
|
||||
* benchmarks/results/v8/run-v8-<ts>.log (full run log)
|
||||
*/
|
||||
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import url from 'node:url';
|
||||
import process from 'node:process';
|
||||
|
||||
// @waggle/agent stubs — avoids pulling in heavy agent package during harness runs.
|
||||
// generateTurnId: UUID v4 via crypto. logTurnEvent: no-op (observability only in prod).
|
||||
function generateTurnId(): string {
|
||||
return crypto.randomUUID();
|
||||
}
|
||||
function logTurnEvent(_turnId: string, _event: Record<string, unknown>): void {
|
||||
// no-op in harness context; real agent logging wired in full waggle runtime
|
||||
}
|
||||
import type {
|
||||
CellName, DatasetSpec, JsonlRecord, ModelSpec, RunConfig,
|
||||
} from '../src/types.js';
|
||||
import { loadDataset, getDatasetVersion, sampleInstances } from '../src/datasets.js';
|
||||
import { createLlmClient } from '../src/llm.js';
|
||||
import { JsonlWriter, buildAggregate, scoreAccuracy, percentile } from '../src/metrics.js';
|
||||
// Cells are imported dynamically to avoid @waggle/agent package resolution at startup.
|
||||
// (cells.ts → @waggle/agent → docx/exceljs/etc. which aren't installed in harness-only envs)
|
||||
// These are resolved lazily on first actual cell invocation.
|
||||
// Lazy-loaded cell modules (resolved on first invocation, not at import time)
|
||||
|
||||
async function loadCells(): Promise<{
|
||||
cells: typeof import('../src/cells.js').cells;
|
||||
isCellName: typeof import('../src/cells.js').isCellName;
|
||||
hiveMindIpbCell: typeof import('../src/cells-ipb.js').hiveMindIpbCell;
|
||||
}> {
|
||||
const [cellsMod, ipbMod] = await Promise.all([
|
||||
import('../src/cells.js'),
|
||||
import('../src/cells-ipb.js'),
|
||||
]);
|
||||
return {
|
||||
cells: cellsMod.cells,
|
||||
isCellName: cellsMod.isCellName,
|
||||
hiveMindIpbCell: ipbMod.hiveMindIpbCell,
|
||||
};
|
||||
}
|
||||
import { createSubstrate } from '../src/substrate.js';
|
||||
import type { Substrate } from '../src/substrate.js';
|
||||
import { extractTurnsFromLongMemEval, ingestLongMemEvalCorpus } from '../src/ingest-longmemeval.js';
|
||||
import { extractTurnsFromBeam, ingestBeamCorpus } from '../src/ingest-beam.js';
|
||||
import { StreakTracker } from '../src/streak-tracker.js';
|
||||
import { preCellHealthCheck } from '../src/health-check.js';
|
||||
import { acquireRunnerLock } from '../src/runner-lock.js';
|
||||
import type { LockHandle } from '../src/runner-lock.js';
|
||||
import { createJudgeLlmClient } from '../src/judge-client.js';
|
||||
import { runJudge } from '../src/judge-runner.js';
|
||||
import type { JudgeConfig, JudgePayload } from '../src/judge-runner.js';
|
||||
import type { JudgeClientCostEntry } from '../src/judge-client.js';
|
||||
|
||||
// ── Extended cell name (adds hive_mind_ipb) ────────────────────────────────
|
||||
|
||||
type V8CellName = CellName | 'hive_mind_ipb';
|
||||
|
||||
// ── Constants ─────────────────────────────────────────────────────────────────
|
||||
|
||||
const DEFAULT_SEED = 42;
|
||||
const DEFAULT_BUDGET_A0 = 20; // Track A0: LME V1 ($20 hard halt)
|
||||
const DEFAULT_BUDGET_A = 50; // Track A: BEAM ($50 hard halt)
|
||||
|
||||
// v8 4-cell ablation grid (in run order)
|
||||
const V8_CELLS: readonly V8CellName[] = [
|
||||
'no-context',
|
||||
'retrieval',
|
||||
'hive_mind_ipb',
|
||||
// hive_mind_ipb_strong is added at runtime when --model-strong is set and
|
||||
// --no-ipb-strong is NOT passed. It runs as a separate hive_mind_ipb cell
|
||||
// invocation with the strong model id.
|
||||
];
|
||||
|
||||
// ── Path helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
function harnessRoot(): string {
|
||||
const here = url.fileURLToPath(import.meta.url);
|
||||
// scripts/ → harness root
|
||||
return path.resolve(path.dirname(here), '..');
|
||||
}
|
||||
|
||||
function benchRoot(): string {
|
||||
return path.resolve(harnessRoot(), '..');
|
||||
}
|
||||
|
||||
function defaultOutputDir(track: string): string {
|
||||
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
const dir = path.join(benchRoot(), 'results', 'v8', track);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
return dir;
|
||||
}
|
||||
|
||||
function outputPath(dir: string, cell: string, dataset: string): string {
|
||||
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
return path.join(dir, `${cell}-${dataset}-${ts}.jsonl`);
|
||||
}
|
||||
|
||||
// ── Config loaders ────────────────────────────────────────────────────────────
|
||||
|
||||
function loadModels(): Record<string, ModelSpec> {
|
||||
const cfg = path.join(harnessRoot(), 'config', 'models.json');
|
||||
return JSON.parse(fs.readFileSync(cfg, 'utf-8')) as Record<string, ModelSpec>;
|
||||
}
|
||||
|
||||
function loadDatasets(): Record<string, DatasetSpec> {
|
||||
const cfg = path.join(harnessRoot(), 'config', 'datasets.json');
|
||||
return JSON.parse(fs.readFileSync(cfg, 'utf-8')) as Record<string, DatasetSpec>;
|
||||
}
|
||||
|
||||
// ── CLI arg parsing ────────────────────────────────────────────────────────────
|
||||
|
||||
interface V8Args {
|
||||
tracks: Array<'a0' | 'a' | 'b' | 'd'>;
|
||||
limit: number;
|
||||
budgetA0: number;
|
||||
budgetA: number;
|
||||
modelPrimary: string;
|
||||
modelStrong?: string;
|
||||
runIpbStrong: boolean;
|
||||
lmeDataPath?: string;
|
||||
beamDataPath?: string;
|
||||
dryRun?: boolean;
|
||||
judge?: string;
|
||||
seed: number;
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): V8Args {
|
||||
const out: V8Args = {
|
||||
tracks: ['a0', 'a'], // default: A0 + A (B blocked, D is external)
|
||||
limit: Number.POSITIVE_INFINITY,
|
||||
budgetA0: DEFAULT_BUDGET_A0,
|
||||
budgetA: DEFAULT_BUDGET_A,
|
||||
modelPrimary: 'qwen3.6-35b-a3b',
|
||||
runIpbStrong: true,
|
||||
seed: DEFAULT_SEED,
|
||||
};
|
||||
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const flag = argv[i];
|
||||
const next = argv[i + 1];
|
||||
switch (flag) {
|
||||
case '--track':
|
||||
if (next === 'all') {
|
||||
out.tracks = ['a0', 'a']; // b and d are external
|
||||
} else if (next === 'a0' || next === 'a' || next === 'b' || next === 'd') {
|
||||
out.tracks = [next];
|
||||
} else {
|
||||
console.error(`[run-v8] Unknown --track value: ${next}. Use a0|a|b|d|all`);
|
||||
process.exit(1);
|
||||
}
|
||||
i++;
|
||||
break;
|
||||
case '--limit': out.limit = Number(next); i++; break;
|
||||
case '--budget-a0': out.budgetA0 = Number(next); i++; break;
|
||||
case '--budget-a': out.budgetA = Number(next); i++; break;
|
||||
case '--model-primary': out.modelPrimary = next; i++; break;
|
||||
case '--model-strong': out.modelStrong = next; i++; break;
|
||||
case '--no-ipb-strong': out.runIpbStrong = false; break;
|
||||
case '--lme-data-path': out.lmeDataPath = next; i++; break;
|
||||
case '--beam-data-path': out.beamDataPath = next; i++; break;
|
||||
case '--dry-run': out.dryRun = true; break;
|
||||
case '--live': out.dryRun = false; break;
|
||||
case '--judge': out.judge = next; i++; break;
|
||||
case '--seed': out.seed = Number(next); i++; break;
|
||||
case '--help':
|
||||
case '-h':
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function printHelp(): void {
|
||||
console.log(`
|
||||
run-v8.ts — v8 Multi-Benchmark Runner
|
||||
|
||||
Usage:
|
||||
npx tsx benchmarks/harness/scripts/run-v8.ts [options]
|
||||
|
||||
Tracks (run in order):
|
||||
--track a0 Track A0: LongMemEval V1 (500q, $20 budget)
|
||||
--track a Track A: BEAM 128K (~300q, $50 budget)
|
||||
--track all Run A0 then A (default)
|
||||
|
||||
Options:
|
||||
--limit N Instance cap per cell
|
||||
--budget-a0 USD Track A0 hard halt (default: $20)
|
||||
--budget-a USD Track A hard halt (default: $50)
|
||||
--model-primary ID Primary model (default: qwen3.6-35b-a3b)
|
||||
--model-strong ID Strong model for ipb_strong cell (default: claude-opus-4-x)
|
||||
--no-ipb-strong Skip hive_mind_ipb_strong cell
|
||||
--lme-data-path P Path to longmemeval.jsonl
|
||||
--beam-data-path P Path to beam-128K.jsonl
|
||||
--dry-run Stub LLM calls
|
||||
--judge MODEL Enable per-instance judge
|
||||
--seed N PRNG seed (default: 42)
|
||||
`);
|
||||
}
|
||||
|
||||
// ── Per-instance runner (handles both canonical cells + hive_mind_ipb) ──────
|
||||
|
||||
async function round(n: number, decimals: number): Promise<number> {
|
||||
const f = Math.pow(10, decimals);
|
||||
return Math.round(n * f) / f;
|
||||
}
|
||||
|
||||
function computeFileHash(p: string): string {
|
||||
return crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex');
|
||||
}
|
||||
|
||||
interface V8RunOneConfig {
|
||||
cellName: V8CellName;
|
||||
/** When cellName is 'hive_mind_ipb' and this is set, uses strongModel. */
|
||||
useStrongModel?: boolean;
|
||||
dataset: DatasetSpec;
|
||||
model: ModelSpec;
|
||||
strongModel?: ModelSpec;
|
||||
litellmUrl: string;
|
||||
litellmApiKey: string;
|
||||
dryRun: boolean;
|
||||
limit: number;
|
||||
seed: number;
|
||||
budgetUsd: number;
|
||||
outputFilePath: string;
|
||||
substrate?: Substrate;
|
||||
judgeConfig?: JudgeConfig;
|
||||
judgeCosts: JudgeClientCostEntry[];
|
||||
}
|
||||
|
||||
async function runOneV8(config: V8RunOneConfig): Promise<void> {
|
||||
const {
|
||||
cellName, useStrongModel, dataset, model, strongModel,
|
||||
litellmUrl, litellmApiKey, dryRun, limit, seed,
|
||||
budgetUsd, outputFilePath, substrate, judgeConfig, judgeCosts,
|
||||
} = config;
|
||||
|
||||
const activeModel = useStrongModel && strongModel ? strongModel : model;
|
||||
const dataRoot = path.join(harnessRoot(), '..', 'data');
|
||||
const all = loadDataset(dataset, dataRoot);
|
||||
const datasetVersion = getDatasetVersion(dataset, dataRoot);
|
||||
const sampled = sampleInstances(all, seed, limit);
|
||||
|
||||
const writer = new JsonlWriter(outputFilePath);
|
||||
const llm = createLlmClient({ dryRun, litellmUrl, litellmApiKey });
|
||||
|
||||
const startedAt = new Date().toISOString();
|
||||
let totalCost = 0;
|
||||
let budgetStoppedAt: number | null = null;
|
||||
const latencies: number[] = [];
|
||||
const streakTracker = new StreakTracker();
|
||||
let streakHaltAt: number | null = null;
|
||||
let streakHaltSummary: string | null = null;
|
||||
|
||||
const displayCell = useStrongModel ? 'hive_mind_ipb_strong' : cellName;
|
||||
console.log(`[v8:run] cell=${displayCell} dataset=${dataset.id} model=${activeModel.id} n=${sampled.length} budget=$${budgetUsd}`);
|
||||
|
||||
for (let i = 0; i < sampled.length; i++) {
|
||||
if (totalCost >= budgetUsd) {
|
||||
budgetStoppedAt = i;
|
||||
console.log(`[v8:budget] halted at instance ${i} (cost=$${totalCost.toFixed(4)} >= $${budgetUsd})`);
|
||||
break;
|
||||
}
|
||||
|
||||
const instance = sampled[i];
|
||||
const turnId = generateTurnId();
|
||||
|
||||
// Lazy-load cell modules on first iteration (avoids @waggle/agent at import time)
|
||||
const { cells, isCellName, hiveMindIpbCell } = await loadCells();
|
||||
|
||||
let result;
|
||||
if (cellName === 'hive_mind_ipb') {
|
||||
result = await hiveMindIpbCell({
|
||||
instance,
|
||||
model: activeModel,
|
||||
llm,
|
||||
turnId,
|
||||
substrate,
|
||||
retrievalTopK: 20,
|
||||
});
|
||||
} else if (isCellName(cellName)) {
|
||||
result = await cells[cellName]({
|
||||
instance,
|
||||
model: activeModel,
|
||||
llm,
|
||||
turnId,
|
||||
substrate,
|
||||
litellm: { url: litellmUrl, apiKey: litellmApiKey },
|
||||
retrievalTopK: 20,
|
||||
});
|
||||
} else {
|
||||
throw new Error(`[run-v8] Unknown cell: ${cellName}`);
|
||||
}
|
||||
|
||||
latencies.push(result.latencyMs);
|
||||
const accuracy = result.failureMode ? 0 : scoreAccuracy(result.text, instance.expected);
|
||||
totalCost += result.costUsd;
|
||||
|
||||
logTurnEvent(turnId, {
|
||||
stage: 'llm.response',
|
||||
cell: displayCell,
|
||||
model: activeModel.id,
|
||||
textChars: result.text.length,
|
||||
latencyMs: result.latencyMs,
|
||||
costUsd: result.costUsd,
|
||||
failureMode: result.failureMode,
|
||||
reasoningShape: result.reasoningShape ?? 'none',
|
||||
reasoningChars: result.reasoningContent?.length ?? 0,
|
||||
});
|
||||
|
||||
// Per-instance judge
|
||||
let judgePayload: JudgePayload | null = null;
|
||||
if (judgeConfig && !result.failureMode) {
|
||||
judgePayload = await runJudge(
|
||||
{
|
||||
question: instance.question,
|
||||
groundTruth: instance.expected[0] ?? '',
|
||||
contextExcerpt: instance.context,
|
||||
modelAnswer: result.text,
|
||||
},
|
||||
judgeConfig,
|
||||
);
|
||||
}
|
||||
|
||||
const record: JsonlRecord = {
|
||||
turnId,
|
||||
cell: displayCell as CellName, // cast: JSONL schema, hive_mind_ipb_strong stored as variant
|
||||
instance_id: instance.instance_id,
|
||||
model: activeModel.id,
|
||||
seed,
|
||||
accuracy,
|
||||
p50_latency_ms: percentile(latencies, 50),
|
||||
p95_latency_ms: percentile(latencies, 95),
|
||||
usd_per_query: Math.round(result.costUsd * 1_000_000) / 1_000_000,
|
||||
failure_mode: result.failureMode,
|
||||
dataset_version: datasetVersion,
|
||||
...(judgePayload && {
|
||||
model_answer: judgePayload.model_answer,
|
||||
judge_verdict: judgePayload.judge_verdict,
|
||||
judge_failure_mode: judgePayload.judge_failure_mode,
|
||||
judge_rationale: judgePayload.judge_rationale,
|
||||
judge_model: judgePayload.judge_model,
|
||||
judge_timestamp: judgePayload.judge_timestamp,
|
||||
judge_ensemble: judgePayload.judge_ensemble,
|
||||
}),
|
||||
...(result.reasoningContent !== undefined && {
|
||||
reasoning_content: result.reasoningContent,
|
||||
reasoning_content_chars: result.reasoningContent.length,
|
||||
}),
|
||||
...(result.reasoningShape !== undefined && {
|
||||
reasoning_shape: result.reasoningShape,
|
||||
}),
|
||||
...(activeModel.pinning_surface !== undefined && {
|
||||
model_pinning_surface: activeModel.pinning_surface,
|
||||
model_pinning_carve_out_reason: activeModel.pinning_surface_carve_out_reason ?? null,
|
||||
}),
|
||||
model_revision_hash: null,
|
||||
};
|
||||
writer.write(record);
|
||||
|
||||
// Progress log every 10 instances
|
||||
if ((i + 1) % 10 === 0 || i === sampled.length - 1) {
|
||||
const runningAcc = writer.all().reduce((sum, r) => sum + r.accuracy, 0) / writer.all().length;
|
||||
console.log(
|
||||
`[v8:progress] cell=${displayCell} ${i + 1}/${sampled.length} ` +
|
||||
`acc=${(runningAcc * 100).toFixed(1)}% cost=$${totalCost.toFixed(4)}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (streakTracker.record(result.failureMode)) {
|
||||
streakHaltAt = i + 1;
|
||||
streakHaltSummary = streakTracker.summary();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
await writer.close();
|
||||
const finishedAt = new Date().toISOString();
|
||||
|
||||
const runConfig = {
|
||||
run: { kind: 'cell' as const, name: displayCell as CellName },
|
||||
dataset,
|
||||
model: activeModel,
|
||||
limit,
|
||||
seed,
|
||||
budgetUsd,
|
||||
outputPath: outputFilePath,
|
||||
dryRun,
|
||||
litellmUrl,
|
||||
litellmApiKey,
|
||||
} as RunConfig;
|
||||
const summary = buildAggregate(runConfig, writer.all(), startedAt, finishedAt, budgetStoppedAt);
|
||||
const summaryPath = outputFilePath.replace(/\.jsonl$/, '.summary.json');
|
||||
fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2), 'utf-8');
|
||||
|
||||
console.log(
|
||||
`[v8:summary] cell=${displayCell} dataset=${dataset.id} ` +
|
||||
`n=${summary.counts.total} completed=${summary.counts.completed} ` +
|
||||
`failed=${summary.counts.failed} ` +
|
||||
`accuracy=${(summary.metrics.meanAccuracy * 100).toFixed(2)}% ` +
|
||||
`cost=$${summary.metrics.totalUsd.toFixed(4)} ` +
|
||||
`jsonl=${outputFilePath}`,
|
||||
);
|
||||
|
||||
if (streakHaltAt !== null) {
|
||||
throw new Error(
|
||||
`[v8:halt] cell '${displayCell}' aborted at instance ${streakHaltAt}/${sampled.length} ` +
|
||||
`due to consecutive transport failures (${streakHaltSummary}).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Track runner ──────────────────────────────────────────────────────────────
|
||||
|
||||
interface TrackConfig {
|
||||
track: 'a0' | 'a';
|
||||
datasetId: string;
|
||||
budgetUsd: number;
|
||||
ingestDataset: boolean;
|
||||
ingestFn: 'lme' | 'beam';
|
||||
dataPathOverride?: string;
|
||||
}
|
||||
|
||||
async function runTrack(
|
||||
config: TrackConfig,
|
||||
args: V8Args,
|
||||
primaryModel: ModelSpec,
|
||||
strongModel: ModelSpec | undefined,
|
||||
allDatasets: Record<string, DatasetSpec>,
|
||||
litellmUrl: string,
|
||||
litellmApiKey: string,
|
||||
dryRun: boolean,
|
||||
judgeConfig: JudgeConfig | undefined,
|
||||
judgeCosts: JudgeClientCostEntry[],
|
||||
): Promise<void> {
|
||||
const { track, datasetId, budgetUsd, ingestFn, dataPathOverride } = config;
|
||||
const dataset = allDatasets[datasetId];
|
||||
if (!dataset) {
|
||||
throw new Error(`[run-v8] Dataset not found: ${datasetId}. Check config/datasets.json.`);
|
||||
}
|
||||
|
||||
console.log(`\n${'='.repeat(70)}`);
|
||||
console.log(`[v8] TRACK ${track.toUpperCase()} — ${dataset.displayName}`);
|
||||
console.log(`${'='.repeat(70)}`);
|
||||
|
||||
const outDir = defaultOutputDir(track);
|
||||
|
||||
// ── Build substrate + ingest corpus ──────────────────────────────────────
|
||||
let substrate: Substrate | null = null;
|
||||
|
||||
if (!dryRun) {
|
||||
console.log(`[v8:substrate] building ephemeral MindDB substrate for ${datasetId} …`);
|
||||
substrate = createSubstrate();
|
||||
const dataRoot = path.join(harnessRoot(), '..', 'data');
|
||||
const jsonlPath = dataPathOverride ?? path.join(dataRoot, dataset.dataPath);
|
||||
|
||||
if (!fs.existsSync(jsonlPath)) {
|
||||
throw new Error(
|
||||
`[run-v8] Dataset JSONL not found at ${jsonlPath}. ` +
|
||||
(ingestFn === 'lme'
|
||||
? 'Run: npx tsx benchmarks/harness/scripts/build-longmemeval-canonical.ts'
|
||||
: 'Run: npx tsx benchmarks/harness/scripts/build-beam-canonical.ts --beam-data-path /path/to/BEAM/data --chat-size 128K'),
|
||||
);
|
||||
}
|
||||
|
||||
const ingestStart = Date.now();
|
||||
if (ingestFn === 'lme') {
|
||||
const turns = extractTurnsFromLongMemEval(jsonlPath);
|
||||
console.log(`[v8:substrate] extracted ${turns.length} LME V1 turns from ${jsonlPath}`);
|
||||
const stats = await ingestLongMemEvalCorpus(
|
||||
substrate.db, substrate.search, substrate.frames, substrate.sessions, turns,
|
||||
);
|
||||
console.log(
|
||||
`[v8:substrate] LME ingest complete: frames=${stats.count} ` +
|
||||
`ingest_ms=${stats.ingestMs} index_ms=${stats.indexMs} ` +
|
||||
`total_ms=${Date.now() - ingestStart}`,
|
||||
);
|
||||
} else {
|
||||
const turns = extractTurnsFromBeam(jsonlPath);
|
||||
console.log(`[v8:substrate] extracted ${turns.length} BEAM turns from ${jsonlPath}`);
|
||||
const stats = await ingestBeamCorpus(
|
||||
substrate.db, substrate.search, substrate.frames, substrate.sessions, turns,
|
||||
);
|
||||
console.log(
|
||||
`[v8:substrate] BEAM ingest complete: frames=${stats.count} ` +
|
||||
`ingest_ms=${stats.ingestMs} index_ms=${stats.indexMs} ` +
|
||||
`total_ms=${Date.now() - ingestStart}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
console.log(`[v8:substrate] dry-run — skipping substrate ingest`);
|
||||
}
|
||||
|
||||
// Build the per-track cell list
|
||||
const trackCells: Array<{ cellName: V8CellName; useStrong: boolean }> = [
|
||||
{ cellName: 'no-context', useStrong: false },
|
||||
{ cellName: 'retrieval', useStrong: false },
|
||||
{ cellName: 'hive_mind_ipb', useStrong: false },
|
||||
];
|
||||
if (args.runIpbStrong && strongModel) {
|
||||
trackCells.push({ cellName: 'hive_mind_ipb', useStrong: true });
|
||||
}
|
||||
|
||||
// Budget per-cell (split total track budget evenly so any single cell can't exhaust)
|
||||
// Each cell gets the full budget; the track budget is checked across cells
|
||||
// by the caller. Individual cell budgets are capped at budgetUsd / 4 * 1.5
|
||||
// to ensure all 4 cells get a fair share with some slack.
|
||||
const cellBudget = budgetUsd; // individual hard-halt per cell
|
||||
|
||||
let trackTotalCost = 0;
|
||||
|
||||
try {
|
||||
for (const { cellName, useStrong } of trackCells) {
|
||||
const displayCell = useStrong ? 'hive_mind_ipb_strong' : cellName;
|
||||
|
||||
// Skip retrieval/hive_mind_ipb cells in dry-run (need live embedder)
|
||||
if (dryRun && (cellName === 'retrieval' || cellName === 'hive_mind_ipb')) {
|
||||
console.log(`[v8:skip] ${displayCell} — skipped in dry-run mode (needs substrate)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const outPath = outputPath(outDir, displayCell, datasetId);
|
||||
await runOneV8({
|
||||
cellName,
|
||||
useStrongModel: useStrong,
|
||||
dataset,
|
||||
model: primaryModel,
|
||||
strongModel,
|
||||
litellmUrl,
|
||||
litellmApiKey,
|
||||
dryRun,
|
||||
limit: args.limit,
|
||||
seed: args.seed,
|
||||
budgetUsd: cellBudget,
|
||||
outputFilePath: outPath,
|
||||
substrate: substrate ?? undefined,
|
||||
judgeConfig,
|
||||
judgeCosts,
|
||||
});
|
||||
|
||||
// Read cost from summary file for track total
|
||||
const summaryPath = outPath.replace(/\.jsonl$/, '.summary.json');
|
||||
if (fs.existsSync(summaryPath)) {
|
||||
const summ = JSON.parse(fs.readFileSync(summaryPath, 'utf-8')) as {
|
||||
metrics?: { totalUsd?: number };
|
||||
};
|
||||
const cellCost = summ.metrics?.totalUsd ?? 0;
|
||||
trackTotalCost += cellCost;
|
||||
console.log(
|
||||
`[v8:track-cost] ${track.toUpperCase()} track running total: $${trackTotalCost.toFixed(4)}`,
|
||||
);
|
||||
if (trackTotalCost >= budgetUsd) {
|
||||
console.warn(
|
||||
`[v8:track-budget] Track ${track.toUpperCase()} track budget ($${budgetUsd}) reached ` +
|
||||
`after cell ${displayCell}. Stopping remaining cells.`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (substrate) {
|
||||
substrate.close();
|
||||
console.log(`[v8:substrate] closed substrate for ${datasetId}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[v8:track-done] Track ${track.toUpperCase()} complete. ` +
|
||||
`Total cost: $${trackTotalCost.toFixed(4)} ` +
|
||||
`Results in: ${outDir}`,
|
||||
);
|
||||
}
|
||||
|
||||
// ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const allModels = loadModels();
|
||||
const allDatasets = loadDatasets();
|
||||
|
||||
const primaryModel = allModels[args.modelPrimary];
|
||||
if (!primaryModel) {
|
||||
throw new Error(
|
||||
`[run-v8] Unknown primary model: ${args.modelPrimary}. ` +
|
||||
`Valid ids: ${Object.keys(allModels).join(', ')}`,
|
||||
);
|
||||
}
|
||||
|
||||
let strongModel: ModelSpec | undefined;
|
||||
if (args.modelStrong) {
|
||||
strongModel = allModels[args.modelStrong];
|
||||
if (!strongModel) {
|
||||
throw new Error(
|
||||
`[run-v8] Unknown strong model: ${args.modelStrong}. ` +
|
||||
`Valid ids: ${Object.keys(allModels).join(', ')}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// Try to find claude-opus-4-x automatically
|
||||
strongModel = allModels['claude-opus-4-x'] ?? allModels['claude-opus-4-8'];
|
||||
if (strongModel) {
|
||||
console.log(`[v8] strong model auto-resolved: ${strongModel.id}`);
|
||||
} else if (args.runIpbStrong) {
|
||||
console.warn('[v8] No strong model found in models.json (tried claude-opus-4-x, claude-opus-4-8). Skipping hive_mind_ipb_strong cell.');
|
||||
}
|
||||
}
|
||||
|
||||
const dryRun = args.dryRun ?? !process.env.LITELLM_URL;
|
||||
const litellmUrl = process.env.LITELLM_URL ?? 'http://localhost:4000';
|
||||
const litellmApiKey = process.env.LITELLM_API_KEY ?? 'sk-waggle-dev';
|
||||
|
||||
if (dryRun) {
|
||||
console.log('[v8] DRY RUN mode — LLM calls will be stubbed');
|
||||
}
|
||||
|
||||
// Judge wiring
|
||||
const judgeCosts: JudgeClientCostEntry[] = [];
|
||||
let judgeConfig: JudgeConfig | undefined;
|
||||
if (!dryRun && args.judge) {
|
||||
judgeConfig = {
|
||||
kind: 'single',
|
||||
model: args.judge,
|
||||
client: createJudgeLlmClient({
|
||||
litellmUrl, litellmApiKey,
|
||||
model: args.judge,
|
||||
onCall: entry => judgeCosts.push(entry),
|
||||
}),
|
||||
};
|
||||
console.log(`[v8] judge model: ${args.judge}`);
|
||||
}
|
||||
|
||||
// Health check
|
||||
if (!dryRun) {
|
||||
const judgePingModels = args.judge ? [args.judge] : [];
|
||||
const hc = await preCellHealthCheck({
|
||||
litellmUrl, litellmApiKey,
|
||||
subjectModel: primaryModel.litellmModel,
|
||||
judgeModels: judgePingModels,
|
||||
});
|
||||
if (!hc.ok) {
|
||||
const summary = hc.failures.map(f => `${f.endpoint} → ${f.error}`).join('; ');
|
||||
throw new Error(
|
||||
`[v8:health-check] FAILED: ${summary}. ` +
|
||||
`Check LiteLLM proxy + provider keys before spending budget.`,
|
||||
);
|
||||
}
|
||||
console.log(`[v8:health-check] OK in ${hc.durationMs}ms`);
|
||||
}
|
||||
|
||||
// Acquire single-runner lock
|
||||
const lockSentinel = path.join(harnessRoot(), '..', 'results', '.benchmark-runner');
|
||||
const runnerLock: LockHandle = acquireRunnerLock(lockSentinel);
|
||||
console.log(`[v8:lock] acquired runner lock (pid=${process.pid})`);
|
||||
|
||||
const runStart = Date.now();
|
||||
console.log(`\n[v8] Starting v8 multi-benchmark run`);
|
||||
console.log(`[v8] Primary: ${primaryModel.id}${strongModel ? ` Strong: ${strongModel.id}` : ''}`);
|
||||
console.log(`[v8] Tracks: ${args.tracks.join(', ')}`);
|
||||
console.log(`[v8] Limit per cell: ${Number.isFinite(args.limit) ? args.limit : 'full'}`);
|
||||
console.log(`[v8] Seed: ${args.seed}`);
|
||||
|
||||
try {
|
||||
// ── Track A0: LongMemEval V1 ──────────────────────────────────────────
|
||||
if (args.tracks.includes('a0')) {
|
||||
const lmeJsonlPath = args.lmeDataPath ??
|
||||
path.join(harnessRoot(), '..', 'data', 'longmemeval', 'longmemeval.jsonl');
|
||||
|
||||
await runTrack(
|
||||
{
|
||||
track: 'a0',
|
||||
datasetId: 'longmemeval',
|
||||
budgetUsd: args.budgetA0,
|
||||
ingestDataset: true,
|
||||
ingestFn: 'lme',
|
||||
dataPathOverride: lmeJsonlPath,
|
||||
},
|
||||
args, primaryModel, strongModel, allDatasets,
|
||||
litellmUrl, litellmApiKey, dryRun, judgeConfig, judgeCosts,
|
||||
);
|
||||
}
|
||||
|
||||
// ── Track A: BEAM 128K ────────────────────────────────────────────────
|
||||
if (args.tracks.includes('a')) {
|
||||
const beamJsonlPath = args.beamDataPath ??
|
||||
path.join(harnessRoot(), '..', 'data', 'beam', 'beam-128K.jsonl');
|
||||
|
||||
await runTrack(
|
||||
{
|
||||
track: 'a',
|
||||
datasetId: 'beam-128k',
|
||||
budgetUsd: args.budgetA,
|
||||
ingestDataset: true,
|
||||
ingestFn: 'beam',
|
||||
dataPathOverride: beamJsonlPath,
|
||||
},
|
||||
args, primaryModel, strongModel, allDatasets,
|
||||
litellmUrl, litellmApiKey, dryRun, judgeConfig, judgeCosts,
|
||||
);
|
||||
}
|
||||
|
||||
// ── Track B: GAIA 2 ───────────────────────────────────────────────────
|
||||
if (args.tracks.includes('b')) {
|
||||
console.log('\n[v8:track-b] BLOCKED — SIGALRM issue on Windows/WSL1.');
|
||||
console.log('[v8:track-b] Fix: run under WSL2 or Docker. Gate: PM-RATIFY-V8-PHASE1.');
|
||||
}
|
||||
|
||||
// ── Track D: Terminal-Bench ───────────────────────────────────────────
|
||||
if (args.tracks.includes('d')) {
|
||||
console.log('\n[v8:track-d] Terminal-Bench is external infra.');
|
||||
console.log('[v8:track-d] Submit waggle scaffold to harborframework/terminal-bench-2-leaderboard.');
|
||||
console.log('[v8:track-d] Baseline: little-coder #118/#123 = 24.6% ± 3.2% (Qwen3.6-35B, 2026-05-14).');
|
||||
}
|
||||
|
||||
} finally {
|
||||
runnerLock.release();
|
||||
}
|
||||
|
||||
// Judge summary
|
||||
if (judgeCosts.length > 0) {
|
||||
const judgeTotalUsd = judgeCosts.reduce((sum, e) => sum + e.usd, 0);
|
||||
const judgeOk = judgeCosts.filter(e => e.ok).length;
|
||||
console.log(
|
||||
`\n[v8:judge-total] calls=${judgeCosts.length} ok=${judgeOk} ` +
|
||||
`failed=${judgeCosts.length - judgeOk} total_usd=$${judgeTotalUsd.toFixed(6)}`,
|
||||
);
|
||||
}
|
||||
|
||||
const totalMs = Date.now() - runStart;
|
||||
console.log(`\n[v8:done] Total wall time: ${(totalMs / 1000).toFixed(1)}s`);
|
||||
console.log(`[v8:done] Results in: ${path.join(benchRoot(), 'results', 'v8')}/`);
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('[v8:fatal]', err?.message ?? err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user