132 lines
6.6 KiB
TypeScript
132 lines
6.6 KiB
TypeScript
#!/usr/bin/env tsx
|
|
/**
|
|
* BEAM 1M — STANDING-DIRECTIVES extractor (the "personal mind / identity" lane).
|
|
*
|
|
* The three falsified levers (distilled-fact retrieval, additive hybrid,
|
|
* outline preamble) all failed the same way: they COMPRESSED content, and BEAM
|
|
* rewards verbatim detail. This lane does the opposite: it extracts a SMALL
|
|
* set of durable, user-stated standing directives — explicit preferences,
|
|
* standing instructions, dietary/format/tooling rules — kept near-verbatim
|
|
* with their dates. 10-40 lines per conversation, not a summary of anything.
|
|
*
|
|
* Source: the distilled facts already in minds-1M-obs ("[YYYY-MM-DD] fact"),
|
|
* batched through gpt-4o-mini with a strict KEEP-ONLY-DIRECTIVES filter, then
|
|
* a final dedupe/merge pass per conversation. Output:
|
|
* benchmarks/data/beam/directives-1M/beam_1M_<conv>.json
|
|
* { conv, gop_id, directives: [{date, text}], built_at, cost_usd }
|
|
*
|
|
* RESUMABLE per conv (.done.json). No ollama dependency. ~$1 for all 35.
|
|
*
|
|
* Usage: npx tsx benchmarks/harness/scripts/beam-build-directives.ts [--convs 1-35] [--estimate]
|
|
*/
|
|
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import url from 'node:url';
|
|
import process from 'node:process';
|
|
|
|
import { createSubstrate } from '../src/substrate.js';
|
|
import { createBeamOpenAiClient, loadDotEnv } from '../src/beam-openai-client.js';
|
|
|
|
const FILTER_SYSTEM =
|
|
'You filter a list of dated facts about a USER. KEEP ONLY standing directives: ' +
|
|
'explicit preferences ("prefers X", "dislikes Y", "favorite is Z"), standing instructions or rules the user ' +
|
|
'gave the assistant ("always respond with...", "never suggest...", "call me..."), and durable personal ' +
|
|
'constraints that shape future answers (dietary restrictions, accessibility needs, format/tooling/style rules). ' +
|
|
'DISCARD everything else: events, one-off tasks, project status, possessions, plans, numbers that are not rules. ' +
|
|
'Output the kept lines VERBATIM (including their [YYYY-MM-DD] prefix), one per line, no bullets, no preamble. ' +
|
|
'If nothing qualifies, output nothing.';
|
|
|
|
const MERGE_SYSTEM =
|
|
'You deduplicate a list of dated user directives (preferences / standing instructions). Merge duplicates and ' +
|
|
'near-duplicates, KEEPING the most recent date for each distinct directive and its most specific wording. ' +
|
|
'If two directives conflict, keep BOTH (they show a preference change; the reader uses the dates). ' +
|
|
'Output one directive per line as "[YYYY-MM-DD] text", chronologically ordered, no preamble. Maximum 40 lines: ' +
|
|
'if more, keep the most consequential.';
|
|
|
|
const FACT_DATE_RE = /^\[(\d{4}-\d{2}-\d{2})\]\s*/;
|
|
const BATCH_CHARS = 12000;
|
|
|
|
function parseConvSpec(spec: string): number[] {
|
|
const out = new Set<number>();
|
|
for (const part of spec.split(',')) {
|
|
const m = part.match(/^(\d+)-(\d+)$/);
|
|
if (m) { for (let i = +m[1]; i <= +m[2]; i++) out.add(i); }
|
|
else if (/^\d+$/.test(part.trim())) out.add(+part.trim());
|
|
}
|
|
return [...out].sort((a, b) => a - b);
|
|
}
|
|
|
|
async function main(): Promise<void> {
|
|
const argv = process.argv.slice(2);
|
|
const here = url.fileURLToPath(import.meta.url);
|
|
const repoRoot = path.resolve(path.dirname(here), '..', '..', '..');
|
|
let convs = parseConvSpec('1-35');
|
|
let estimate = false;
|
|
for (let i = 0; i < argv.length; i++) {
|
|
if (argv[i] === '--convs' && argv[i + 1]) convs = parseConvSpec(argv[++i]);
|
|
else if (argv[i] === '--estimate') estimate = true;
|
|
}
|
|
const obsDir = path.join(repoRoot, 'benchmarks', 'data', 'beam', 'minds-1M-obs');
|
|
const outDir = path.join(repoRoot, 'benchmarks', 'data', 'beam', 'directives-1M');
|
|
fs.mkdirSync(outDir, { recursive: true });
|
|
|
|
loadDotEnv();
|
|
const client = estimate ? null : createBeamOpenAiClient({ model: 'gpt-4o-mini' });
|
|
|
|
let totalCost = 0; let totalChars = 0; let totalBatches = 0;
|
|
for (const conv of convs) {
|
|
const outPath = path.join(outDir, `beam_1M_${conv}.json`);
|
|
const donePath = path.join(outDir, `beam_1M_${conv}.done.json`);
|
|
if (fs.existsSync(donePath) && fs.existsSync(outPath)) { console.log(`[directives][conv ${conv}] SKIP`); continue; }
|
|
const mindPath = path.join(obsDir, `beam_1M_${conv}.mind`);
|
|
if (!fs.existsSync(mindPath)) { console.error(`[directives][conv ${conv}] missing obs mind`); continue; }
|
|
|
|
const substrate = createSubstrate({ dbPath: mindPath });
|
|
let facts: string[];
|
|
try {
|
|
facts = substrate.frames.getGopFrames(`beam_${conv}`)
|
|
.map(f => f.content)
|
|
.filter(c => FACT_DATE_RE.test(c));
|
|
} finally { substrate.close(); }
|
|
|
|
// Batch the facts through the filter.
|
|
const batches: string[] = [];
|
|
let cur: string[] = []; let curLen = 0;
|
|
for (const f of facts) {
|
|
if (curLen + f.length > BATCH_CHARS && cur.length) { batches.push(cur.join('\n')); cur = []; curLen = 0; }
|
|
cur.push(f); curLen += f.length + 1;
|
|
}
|
|
if (cur.length) batches.push(cur.join('\n'));
|
|
totalChars += facts.reduce((a, b) => a + b.length, 0); totalBatches += batches.length;
|
|
if (estimate) { console.log(`[directives][conv ${conv}] estimate: facts=${facts.length} batches=${batches.length}`); continue; }
|
|
|
|
let convCost = 0; const kept: string[] = [];
|
|
for (const b of batches) {
|
|
const res = await client!.chat({ system: FILTER_SYSTEM, user: b, maxTokens: 700 });
|
|
convCost += res.costUsd;
|
|
for (const line of res.text.split('\n')) { const t = line.trim(); if (t && FACT_DATE_RE.test(t)) kept.push(t); }
|
|
}
|
|
// Merge/dedupe pass.
|
|
let directives: Array<{ date: string; text: string }> = [];
|
|
if (kept.length) {
|
|
const res = await client!.chat({ system: MERGE_SYSTEM, user: kept.join('\n'), maxTokens: 1200 });
|
|
convCost += res.costUsd;
|
|
for (const line of res.text.split('\n')) {
|
|
const m = line.trim().match(FACT_DATE_RE);
|
|
if (m) directives.push({ date: m[1], text: line.trim().slice(m[0].length) });
|
|
}
|
|
}
|
|
fs.writeFileSync(outPath, JSON.stringify({ conv, gop_id: `beam_${conv}`, directives, built_at: new Date().toISOString(), cost_usd: Math.round(convCost * 1e4) / 1e4 }, null, 2));
|
|
fs.writeFileSync(donePath, JSON.stringify({ conv, directives: directives.length, cost_usd: convCost }));
|
|
totalCost += convCost;
|
|
console.log(`[directives][conv ${conv}] DONE raw-kept=${kept.length} merged=${directives.length} cost=$${convCost.toFixed(4)}`);
|
|
}
|
|
if (estimate) {
|
|
const inTok = totalChars / 4;
|
|
console.log(`\nESTIMATE: batches=${totalBatches} input≈${(inTok / 1e6).toFixed(2)}M tok → ≈ $${((inTok / 1e6) * 0.15 + (totalBatches * 300 / 1e6) * 0.6).toFixed(2)}`);
|
|
} else console.log(`\nALL DONE. total=$${totalCost.toFixed(2)}`);
|
|
}
|
|
|
|
main().catch(err => { console.error('[beam-build-directives] FATAL:', err); process.exit(1); });
|