moving

benchmarks/harness/scripts/beam-build-directives.ts

@@ -0,0 +1,131 @@
+#!/usr/bin/env tsx
+/**
+ * BEAM 1M — STANDING-DIRECTIVES extractor (the "personal mind / identity" lane).
+ *
+ * The three falsified levers (distilled-fact retrieval, additive hybrid,
+ * outline preamble) all failed the same way: they COMPRESSED content, and BEAM
+ * rewards verbatim detail. This lane does the opposite: it extracts a SMALL
+ * set of durable, user-stated standing directives — explicit preferences,
+ * standing instructions, dietary/format/tooling rules — kept near-verbatim
+ * with their dates. 10-40 lines per conversation, not a summary of anything.
+ *
+ * Source: the distilled facts already in minds-1M-obs ("[YYYY-MM-DD] fact"),
+ * batched through gpt-4o-mini with a strict KEEP-ONLY-DIRECTIVES filter, then
+ * a final dedupe/merge pass per conversation. Output:
+ *   benchmarks/data/beam/directives-1M/beam_1M_<conv>.json
+ *   { conv, gop_id, directives: [{date, text}], built_at, cost_usd }
+ *
+ * RESUMABLE per conv (.done.json). No ollama dependency. ~$1 for all 35.
+ *
+ * Usage: npx tsx benchmarks/harness/scripts/beam-build-directives.ts [--convs 1-35] [--estimate]
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import url from 'node:url';
+import process from 'node:process';
+
+import { createSubstrate } from '../src/substrate.js';
+import { createBeamOpenAiClient, loadDotEnv } from '../src/beam-openai-client.js';
+
+const FILTER_SYSTEM =
+  'You filter a list of dated facts about a USER. KEEP ONLY standing directives: ' +
+  'explicit preferences ("prefers X", "dislikes Y", "favorite is Z"), standing instructions or rules the user ' +
+  'gave the assistant ("always respond with...", "never suggest...", "call me..."), and durable personal ' +
+  'constraints that shape future answers (dietary restrictions, accessibility needs, format/tooling/style rules). ' +
+  'DISCARD everything else: events, one-off tasks, project status, possessions, plans, numbers that are not rules. ' +
+  'Output the kept lines VERBATIM (including their [YYYY-MM-DD] prefix), one per line, no bullets, no preamble. ' +
+  'If nothing qualifies, output nothing.';
+
+const MERGE_SYSTEM =
+  'You deduplicate a list of dated user directives (preferences / standing instructions). Merge duplicates and ' +
+  'near-duplicates, KEEPING the most recent date for each distinct directive and its most specific wording. ' +
+  'If two directives conflict, keep BOTH (they show a preference change; the reader uses the dates). ' +
+  'Output one directive per line as "[YYYY-MM-DD] text", chronologically ordered, no preamble. Maximum 40 lines: ' +
+  'if more, keep the most consequential.';
+
+const FACT_DATE_RE = /^\[(\d{4}-\d{2}-\d{2})\]\s*/;
+const BATCH_CHARS = 12000;
+
+function parseConvSpec(spec: string): number[] {
+  const out = new Set<number>();
+  for (const part of spec.split(',')) {
+    const m = part.match(/^(\d+)-(\d+)$/);
+    if (m) { for (let i = +m[1]; i <= +m[2]; i++) out.add(i); }
+    else if (/^\d+$/.test(part.trim())) out.add(+part.trim());
+  }
+  return [...out].sort((a, b) => a - b);
+}
+
+async function main(): Promise<void> {
+  const argv = process.argv.slice(2);
+  const here = url.fileURLToPath(import.meta.url);
+  const repoRoot = path.resolve(path.dirname(here), '..', '..', '..');
+  let convs = parseConvSpec('1-35');
+  let estimate = false;
+  for (let i = 0; i < argv.length; i++) {
+    if (argv[i] === '--convs' && argv[i + 1]) convs = parseConvSpec(argv[++i]);
+    else if (argv[i] === '--estimate') estimate = true;
+  }
+  const obsDir = path.join(repoRoot, 'benchmarks', 'data', 'beam', 'minds-1M-obs');
+  const outDir = path.join(repoRoot, 'benchmarks', 'data', 'beam', 'directives-1M');
+  fs.mkdirSync(outDir, { recursive: true });
+
+  loadDotEnv();
+  const client = estimate ? null : createBeamOpenAiClient({ model: 'gpt-4o-mini' });
+
+  let totalCost = 0; let totalChars = 0; let totalBatches = 0;
+  for (const conv of convs) {
+    const outPath = path.join(outDir, `beam_1M_${conv}.json`);
+    const donePath = path.join(outDir, `beam_1M_${conv}.done.json`);
+    if (fs.existsSync(donePath) && fs.existsSync(outPath)) { console.log(`[directives][conv ${conv}] SKIP`); continue; }
+    const mindPath = path.join(obsDir, `beam_1M_${conv}.mind`);
+    if (!fs.existsSync(mindPath)) { console.error(`[directives][conv ${conv}] missing obs mind`); continue; }
+
+    const substrate = createSubstrate({ dbPath: mindPath });
+    let facts: string[];
+    try {
+      facts = substrate.frames.getGopFrames(`beam_${conv}`)
+        .map(f => f.content)
+        .filter(c => FACT_DATE_RE.test(c));
+    } finally { substrate.close(); }
+
+    // Batch the facts through the filter.
+    const batches: string[] = [];
+    let cur: string[] = []; let curLen = 0;
+    for (const f of facts) {
+      if (curLen + f.length > BATCH_CHARS && cur.length) { batches.push(cur.join('\n')); cur = []; curLen = 0; }
+      cur.push(f); curLen += f.length + 1;
+    }
+    if (cur.length) batches.push(cur.join('\n'));
+    totalChars += facts.reduce((a, b) => a + b.length, 0); totalBatches += batches.length;
+    if (estimate) { console.log(`[directives][conv ${conv}] estimate: facts=${facts.length} batches=${batches.length}`); continue; }
+
+    let convCost = 0; const kept: string[] = [];
+    for (const b of batches) {
+      const res = await client!.chat({ system: FILTER_SYSTEM, user: b, maxTokens: 700 });
+      convCost += res.costUsd;
+      for (const line of res.text.split('\n')) { const t = line.trim(); if (t && FACT_DATE_RE.test(t)) kept.push(t); }
+    }
+    // Merge/dedupe pass.
+    let directives: Array<{ date: string; text: string }> = [];
+    if (kept.length) {
+      const res = await client!.chat({ system: MERGE_SYSTEM, user: kept.join('\n'), maxTokens: 1200 });
+      convCost += res.costUsd;
+      for (const line of res.text.split('\n')) {
+        const m = line.trim().match(FACT_DATE_RE);
+        if (m) directives.push({ date: m[1], text: line.trim().slice(m[0].length) });
+      }
+    }
+    fs.writeFileSync(outPath, JSON.stringify({ conv, gop_id: `beam_${conv}`, directives, built_at: new Date().toISOString(), cost_usd: Math.round(convCost * 1e4) / 1e4 }, null, 2));
+    fs.writeFileSync(donePath, JSON.stringify({ conv, directives: directives.length, cost_usd: convCost }));
+    totalCost += convCost;
+    console.log(`[directives][conv ${conv}] DONE raw-kept=${kept.length} merged=${directives.length} cost=$${convCost.toFixed(4)}`);
+  }
+  if (estimate) {
+    const inTok = totalChars / 4;
+    console.log(`\nESTIMATE: batches=${totalBatches} input≈${(inTok / 1e6).toFixed(2)}M tok → ≈ $${((inTok / 1e6) * 0.15 + (totalBatches * 300 / 1e6) * 0.6).toFixed(2)}`);
+  } else console.log(`\nALL DONE. total=$${totalCost.toFixed(2)}`);
+}
+
+main().catch(err => { console.error('[beam-build-directives] FATAL:', err); process.exit(1); });