299 lines
17 KiB
TypeScript
299 lines
17 KiB
TypeScript
#!/usr/bin/env tsx
|
|
/**
|
|
* P1 — Evidence Ledger builder (E6). Per BEAM 1M conversation, re-distill the
|
|
* WHOLE raw chat.json into an exhaustive, DATED, QUOTE-GROUNDED evidence ledger.
|
|
*
|
|
* WHY (see BEAM-PARADOX-DEEP-DIVE): the entire 18pp gap to Eywa sits in 4
|
|
* whole-history STATE abilities (abstention / contradiction / event_ordering /
|
|
* temporal). Top-k retrieval covers <3% of a 1M-token conversation, so no
|
|
* excerpt can answer them. The fix is a complete DERIVED store, in Eywa's record
|
|
* format: each line carries a DATE + a VERBATIM SOURCE QUOTE (anti-lossiness —
|
|
* the prior minds-1M-obs distillation stripped dates/numbers and collapsed
|
|
* temporal 0.60→0.15). We rebuild from scratch; we do NOT reuse minds-1M-obs.
|
|
*
|
|
* UNIT: a "session" = a contiguous same-date block (BEAM anchors each session's
|
|
* opening main_question with a time_anchor that propagates forward — identical
|
|
* semantics to src/beam-date-map.ts::buildConvDateMap). Each session (~100K tok)
|
|
* is chunked into small message windows for EXHAUSTIVE extraction; every chunk
|
|
* inherits the session date. Both USER and ASSISTANT messages are mined (known
|
|
* gate facts such as the DeepL/Google pricing live in assistant turns).
|
|
*
|
|
* LINE FORMAT (one atomic fact per line, date-ordered):
|
|
* [YYYY-MM-DD] <fact> ("<verbatim source quote, <=25 words>") {entity1; entity2}
|
|
*
|
|
* RESUMABLE: per-chunk cache (data/beam/ledgers-1M/.cache/) + per-conv
|
|
* .done.json. Rerun with --resume skips finished convs; cached chunks are reused
|
|
* even for an unfinished conv. --budget hard-caps spend.
|
|
*
|
|
* Usage:
|
|
* tsx scripts/beam-build-ledger.ts --convs 1 --budget 5 # P1 gate: conv 1 first
|
|
* tsx scripts/beam-build-ledger.ts --convs 1,10,11 --budget 12 # pilot set
|
|
* tsx scripts/beam-build-ledger.ts --convs 1-35 --budget 30 --resume # full
|
|
*/
|
|
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import url from 'node:url';
|
|
import { createBeamOpenAiClient, BeamOpenAiClient, OPENAI_PRICING, loadDotEnv } from '../src/beam-openai-client.js';
|
|
import { normalizeTimeAnchor } from '../src/beam-date-map.js';
|
|
|
|
// ── paths ────────────────────────────────────────────────────────────────────
|
|
const here = url.fileURLToPath(import.meta.url);
|
|
const repoRoot = path.resolve(path.dirname(here), '..', '..', '..'); // waggle-os
|
|
const BEAM_CHATS = path.resolve(repoRoot, '..', 'BEAM', 'chats');
|
|
const OUT_DIR = path.join(repoRoot, 'benchmarks', 'data', 'beam', 'ledgers-1M');
|
|
const CACHE_DIR = path.join(OUT_DIR, '.cache');
|
|
|
|
// ── args ─────────────────────────────────────────────────────────────────────
|
|
interface Args { convs: number[]; budget: number; resume: boolean; model: string; chunkMsgs: number; concurrency: number; }
|
|
|
|
function parseConvSpec(spec: string): number[] {
|
|
const out = new Set<number>();
|
|
for (const part of spec.split(',')) {
|
|
const m = part.match(/^(\d+)-(\d+)$/);
|
|
if (m) { for (let i = +m[1]; i <= +m[2]; i++) out.add(i); }
|
|
else if (/^\d+$/.test(part.trim())) out.add(+part.trim());
|
|
}
|
|
return [...out].sort((a, b) => a - b);
|
|
}
|
|
|
|
function parseArgs(): Args {
|
|
const argv = process.argv.slice(2);
|
|
const a: Args = { convs: parseConvSpec('1-35'), budget: 45, resume: false, model: 'gpt-5-mini', chunkMsgs: 20, concurrency: 10 };
|
|
for (let i = 0; i < argv.length; i++) {
|
|
const f = argv[i]; const next = argv[i + 1];
|
|
if (f === '--convs' && next) { a.convs = parseConvSpec(next); i++; }
|
|
else if (f === '--budget' && next) { a.budget = parseFloat(next); i++; }
|
|
else if (f === '--resume') { a.resume = true; }
|
|
else if (f === '--model' && next) { a.model = next; i++; }
|
|
else if (f === '--chunk-msgs' && next) { a.chunkMsgs = parseInt(next, 10); i++; }
|
|
else if (f === '--concurrency' && next) { a.concurrency = parseInt(next, 10); i++; }
|
|
}
|
|
return a;
|
|
}
|
|
|
|
// ── session extraction from chat.json (mirrors buildConvDateMap propagation) ──
|
|
interface Msg { role: string; content: string; }
|
|
interface Session { date: string; messages: Msg[]; }
|
|
interface RawMsg { role?: string; content?: string; time_anchor?: string }
|
|
interface RawBatch { turns?: RawMsg[][]; time_anchor?: string }
|
|
|
|
function loadSessions(conv: number): Session[] {
|
|
const p = path.join(BEAM_CHATS, '1M', String(conv), 'chat.json');
|
|
const batches = JSON.parse(fs.readFileSync(p, 'utf-8')) as RawBatch[];
|
|
let current: string | null = null;
|
|
const sessions: Session[] = [];
|
|
let cur: Session | null = null;
|
|
for (const batch of batches) {
|
|
const bd = normalizeTimeAnchor(batch.time_anchor);
|
|
if (bd) current = bd;
|
|
if (!Array.isArray(batch.turns)) continue;
|
|
for (const group of batch.turns) {
|
|
if (!Array.isArray(group)) continue;
|
|
for (const m of group) {
|
|
const role = String(m.role ?? 'unknown').toLowerCase();
|
|
const content = String(m.content ?? '').trim();
|
|
const md = normalizeTimeAnchor(m.time_anchor);
|
|
if (md) current = md;
|
|
if (!content || current === null) continue;
|
|
if (!cur || cur.date !== current) { cur = { date: current, messages: [] }; sessions.push(cur); }
|
|
cur.messages.push({ role, content });
|
|
}
|
|
}
|
|
}
|
|
return sessions;
|
|
}
|
|
|
|
function chunk<T>(arr: T[], size: number): T[][] {
|
|
const out: T[][] = [];
|
|
for (let i = 0; i < arr.length; i += size) out.push(arr.slice(i, i + size));
|
|
return out;
|
|
}
|
|
|
|
// ── extraction prompt ─────────────────────────────────────────────────────────
|
|
const SYSTEM_PROMPT =
|
|
'You are a meticulous memory-extraction system. You convert a dated slice of a ' +
|
|
'conversation between a user and an AI assistant into an exhaustive list of atomic ' +
|
|
'evidence facts. You never summarize away detail and you never invent anything.';
|
|
|
|
function buildExtractionPrompt(date: string, msgs: Msg[]): string {
|
|
const slice = msgs.map(m => `${m.role.toUpperCase()}: ${m.content}`).join('\n\n');
|
|
return `The following is a slice of a conversation that took place on ${date}. Distill it into an evidence ledger: one line per SUBSTANTIVE fact.
|
|
|
|
STRICT OUTPUT FORMAT — one fact per line, and NOTHING else:
|
|
[${date}] <the fact, self-contained and specific> ("<verbatim quote copied EXACTLY from the slice, <=25 words>") {entity1; entity2}
|
|
|
|
RULES:
|
|
1. COMPLETE ON INFORMATION, not on sentences. Capture every distinct piece of information that could later be asked about: user decisions, preferences, goals, plans, and instructions; every stated tool, library, framework, version number, configuration value, port, price/cost, date, deadline, name, event, problem/error, and its outcome or resolution. Do NOT drop any fact that carries a number, version, date, price, name, or a decision.
|
|
2. CONSOLIDATE, do not fragment. Put ONE coherent claim on ONE line — fold a recommendation and its rationale, or a setting and its value, into a single line. Do NOT split a bulleted list or a single explanation into many fragment lines. Skip pure pleasantries, filler, restatements, and generic advice that carries no specific detail. Most 20-message slices distill to roughly 25-45 lines, not hundreds.
|
|
3. The parenthetical quote is MANDATORY on every line and must be copied VERBATIM (character-for-character) from a message in the slice, at most 25 words. It is the evidence anchor — never paraphrase it, never fabricate it. Quote the single most load-bearing <=25-word span (containing the key value/name/number when there is one).
|
|
4. Copy numbers, versions, dates, prices, filenames, ports, and identifiers EXACTLY as written in the slice — never round or normalize them.
|
|
5. Attribute the source. Start the fact with "User" or "Assistant" when the source matters (e.g., "User decided to use franc v6.1.0", "Assistant recommended running the backend on port 4000").
|
|
6. {entities}: the key named entities in the fact (technologies, projects, people, files, services, providers), semicolon-separated. Use {} if there are none.
|
|
7. If a value CHANGES within the slice (e.g., a version or decision is revised), emit one line for each stated value — do not collapse them.
|
|
8. Preserve the wording of every stated date ("on March 18", "next Tuesday", "in two weeks") inside the fact so downstream temporal reasoning has the raw reference.
|
|
9. Output ONLY the evidence lines. No headers, no numbering, no commentary, no blank lines, no markdown.
|
|
|
|
CONVERSATION SLICE (date ${date}):
|
|
${slice}`;
|
|
}
|
|
|
|
// ── line parsing / validation ─────────────────────────────────────────────────
|
|
const LINE_RE = /^\[(\d{4}-\d{2}-\d{2})\]\s+(.*?)\s*\("([^"]*)"\)\s*(\{[^}]*\})?\s*$/;
|
|
|
|
function normWS(s: string): string { return s.replace(/\s+/g, ' ').trim().toLowerCase(); }
|
|
|
|
/** Verify the verbatim quote actually appears in the slice (anti-hallucination /
|
|
* anti-lossiness). Ellipsis-split spans each checked as substrings. */
|
|
function quoteInSlice(quote: string, sliceNorm: string): boolean {
|
|
const parts = quote.split(/\.\.\.|…/).map(p => normWS(p)).filter(p => p.length >= 6);
|
|
if (parts.length === 0) return normWS(quote).length > 0 && sliceNorm.includes(normWS(quote));
|
|
return parts.every(p => sliceNorm.includes(p));
|
|
}
|
|
|
|
interface ChunkResult { lines: string[]; verified: number; total: number; costUsd: number; failure: string | null; }
|
|
|
|
async function extractChunk(client: BeamOpenAiClient, date: string, msgs: Msg[]): Promise<ChunkResult> {
|
|
const prompt = buildExtractionPrompt(date, msgs);
|
|
const r = await client.chat({ system: SYSTEM_PROMPT, user: prompt, maxTokens: 16384 });
|
|
if (r.failureMode) return { lines: [], verified: 0, total: 0, costUsd: r.costUsd, failure: r.failureMode };
|
|
const sliceNorm = normWS(msgs.map(m => m.content).join(' '));
|
|
const lines: string[] = [];
|
|
let verified = 0, total = 0;
|
|
for (const rawLine of r.text.split('\n')) {
|
|
const line = rawLine.trim();
|
|
if (!line) continue;
|
|
const m = line.match(LINE_RE);
|
|
if (!m) continue; // drop malformed lines
|
|
total++;
|
|
const quote = m[3];
|
|
const ok = quoteInSlice(quote, sliceNorm);
|
|
if (ok) verified++;
|
|
// normalize entity braces to {} when empty/missing
|
|
const ent = m[4] && m[4] !== '{}' ? ` ${m[4]}` : '';
|
|
lines.push(`[${m[1]}] ${m[2].trim()} ("${quote}")${ent}`);
|
|
}
|
|
return { lines, verified, total, costUsd: r.costUsd, failure: null };
|
|
}
|
|
|
|
// ── per-chunk cache ────────────────────────────────────────────────────────────
|
|
function cachePath(conv: number, si: number, ci: number): string {
|
|
return path.join(CACHE_DIR, `conv${conv}_s${si}_c${ci}.json`);
|
|
}
|
|
|
|
async function mapLimit<T, R>(items: T[], limit: number, fn: (item: T, idx: number) => Promise<R>): Promise<R[]> {
|
|
const results: R[] = new Array(items.length);
|
|
let next = 0;
|
|
async function worker() {
|
|
while (true) {
|
|
const i = next++;
|
|
if (i >= items.length) return;
|
|
results[i] = await fn(items[i], i);
|
|
}
|
|
}
|
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
|
return results;
|
|
}
|
|
|
|
// ── main ────────────────────────────────────────────────────────────────────
|
|
async function main(): Promise<void> {
|
|
const args = parseArgs();
|
|
loadDotEnv();
|
|
fs.mkdirSync(OUT_DIR, { recursive: true });
|
|
fs.mkdirSync(CACHE_DIR, { recursive: true });
|
|
// gpt-5-mini emits many output tokens per extraction chunk → the 60s default
|
|
// timeout aborts mid-generation. Use a long timeout and reasoning_effort:low
|
|
// (extraction is mechanical; hidden reasoning is wasted latency + $ here).
|
|
// Any provider-prefixed model ("openai/*", "anthropic/*") is routed through
|
|
// OpenRouter (the OpenAI-direct account's quota can be exhausted — 429
|
|
// insufficient_quota). Bare model names keep the OpenAI-direct path unchanged.
|
|
const client: BeamOpenAiClient = args.model.includes('/')
|
|
? (() => {
|
|
const key = process.env.OPENROUTER_API_KEY;
|
|
if (!key) throw new Error('OPENROUTER_API_KEY required for provider-routed models');
|
|
const bare = args.model.replace(/^[^/]+\//, '');
|
|
return new BeamOpenAiClient({
|
|
model: args.model, apiKey: key, baseUrl: 'https://openrouter.ai/api/v1',
|
|
pricing: OPENAI_PRICING[args.model] ?? OPENAI_PRICING[bare] ?? { inputPerMillion: 0.25, outputPerMillion: 2.0 },
|
|
timeoutMs: 300_000, maxRetries: 2, reasoningEffort: 'minimal',
|
|
});
|
|
})()
|
|
: createBeamOpenAiClient({
|
|
model: args.model, pricing: OPENAI_PRICING[args.model],
|
|
timeoutMs: 300_000, maxRetries: 2, reasoningEffort: 'minimal',
|
|
});
|
|
|
|
console.log(`[ledger] model=${args.model} chunkMsgs=${args.chunkMsgs} concurrency=${args.concurrency} budget=$${args.budget} convs=${args.convs.length}`);
|
|
let totalCost = 0;
|
|
|
|
for (const conv of args.convs) {
|
|
const ledgerPath = path.join(OUT_DIR, `conv${conv}.ledger.txt`);
|
|
const donePath = path.join(OUT_DIR, `conv${conv}.done.json`);
|
|
if (args.resume && fs.existsSync(donePath) && fs.existsSync(ledgerPath)) {
|
|
console.log(`[ledger] conv ${conv}: done (skip)`);
|
|
continue;
|
|
}
|
|
const chatPath = path.join(BEAM_CHATS, '1M', String(conv), 'chat.json');
|
|
if (!fs.existsSync(chatPath)) { console.warn(`[ledger] conv ${conv}: no chat.json (skip)`); continue; }
|
|
|
|
const sessions = loadSessions(conv);
|
|
// Build the flat list of (sessionIdx, chunkIdx, date, msgs) jobs.
|
|
interface Job { si: number; ci: number; date: string; msgs: Msg[]; }
|
|
const jobs: Job[] = [];
|
|
sessions.forEach((s, si) => chunk(s.messages, args.chunkMsgs).forEach((c, ci) => jobs.push({ si, ci, date: s.date, msgs: c })));
|
|
|
|
console.log(`[ledger] conv ${conv}: ${sessions.length} sessions, ${jobs.length} chunks → extracting...`);
|
|
let convCost = 0, verifiedSum = 0, totalSum = 0, failures = 0;
|
|
|
|
const results = await mapLimit(jobs, args.concurrency, async (job) => {
|
|
const cp = cachePath(conv, job.si, job.ci);
|
|
if (fs.existsSync(cp)) {
|
|
try { return JSON.parse(fs.readFileSync(cp, 'utf-8')) as ChunkResult; } catch { /* re-extract */ }
|
|
}
|
|
if (totalCost + convCost >= args.budget) return { lines: [], verified: 0, total: 0, costUsd: 0, failure: 'budget' } as ChunkResult;
|
|
const res = await extractChunk(client, job.date, job.msgs);
|
|
convCost += res.costUsd;
|
|
if (!res.failure) fs.writeFileSync(cp, JSON.stringify(res));
|
|
return res;
|
|
});
|
|
|
|
// Assemble the ledger in job order (session-ordered, chunk-ordered → date-ordered).
|
|
const allLines: string[] = [];
|
|
for (const r of results) {
|
|
allLines.push(...r.lines);
|
|
verifiedSum += r.verified; totalSum += r.total;
|
|
if (r.failure && r.failure !== 'budget') failures++;
|
|
}
|
|
if (totalCost + convCost >= args.budget && results.some(r => r.failure === 'budget')) {
|
|
console.warn(`[ledger] conv ${conv}: BUDGET hit mid-conv — partial cache written, NOT marking done. Rerun with higher --budget --resume.`);
|
|
totalCost += convCost;
|
|
break;
|
|
}
|
|
// Chunk failures are NOT cached, so a rerun retries only them. Do not mark
|
|
// done and do not overwrite an existing ledger with a partial one — leave the
|
|
// prior good ledger (if any) untouched and let a rerun complete the conv.
|
|
if (failures > 0) {
|
|
console.warn(`[ledger] conv ${conv}: ${failures} chunk failures — NOT marking done; rerun to retry`);
|
|
totalCost += convCost;
|
|
continue;
|
|
}
|
|
|
|
fs.writeFileSync(ledgerPath, allLines.join('\n') + '\n');
|
|
const approxTokens = Math.ceil((allLines.join('\n').length) / 4);
|
|
const stats = {
|
|
conv, sessions: sessions.length, chunks: jobs.length, messages: sessions.reduce((n, s) => n + s.messages.length, 0),
|
|
lines: allLines.length, approx_tokens: approxTokens,
|
|
quote_verified: totalSum ? +(verifiedSum / totalSum).toFixed(4) : 0,
|
|
quote_verified_pct: totalSum ? +(100 * verifiedSum / totalSum).toFixed(1) : 0,
|
|
chunk_failures: failures, cost_usd: +convCost.toFixed(4), model: args.model,
|
|
};
|
|
fs.writeFileSync(donePath, JSON.stringify(stats, null, 2));
|
|
totalCost += convCost;
|
|
console.log(`[ledger] conv ${conv}: ${allLines.length} lines, ~${approxTokens} tok, quote-verified ${stats.quote_verified_pct}% (${verifiedSum}/${totalSum}), ${failures} chunk-failures, $${convCost.toFixed(3)} | running $${totalCost.toFixed(2)}`);
|
|
if (totalCost >= args.budget) { console.warn(`[ledger] budget $${args.budget} reached — stopping.`); break; }
|
|
}
|
|
console.log(`[ledger] DONE. total spend $${totalCost.toFixed(3)}`);
|
|
}
|
|
|
|
main().catch(e => { console.error('[beam-build-ledger] FATAL:', e); process.exit(1); });
|