145 lines
7.9 KiB
JavaScript
145 lines
7.9 KiB
JavaScript
/**
|
||
* D1 chunk-lane eval probe — long-frame needle retrieval (2026-06-12).
|
||
*
|
||
* PRE-REGISTERED DESIGN (before any run):
|
||
* - LoCoMo was REJECTED as the ruler: its frames max at ~1000 chars, below
|
||
* the 2000-char chunk threshold — every frame yields one chunk ≡ the whole
|
||
* frame, so a LoCoMo A/B measures noise by construction.
|
||
* - Honest corpus: a COPY of the real production personal mind
|
||
* (~/.hive-mind/personal.mind: 471 frames, 129 >2k chars, max 25.7k).
|
||
* - Both cells re-embedded from scratch with the SAME local embedder
|
||
* (Ollama nomic-embed-text-8k, 1024-dim) — the original mind's vectors are
|
||
* mock-fingerprinted and unusable; equal footing by construction.
|
||
* - Needle = a verbatim sentence drawn from the 50–90% depth of a long frame
|
||
* (where whole-frame embedding signal dilutes). Verbatim is fair: vector
|
||
* lanes never touch FTS, and both cells get the identical query.
|
||
* - Cell A (control): HybridSearch.vectorSearch (whole-frame vectors)
|
||
* - Cell B (treatment): HybridSearch.vectorSearchChunks (chunk vectors,
|
||
* best-chunk-per-frame) — called directly; no env flag needed.
|
||
* - Metric: hit@1 / hit@5 / hit@10 of the source frame, paired per needle.
|
||
* - Decision rule (pre-registered): flip default-ON only if chunk hit@5
|
||
* beats whole-frame hit@5 by a clear margin (paired; report exact counts).
|
||
*
|
||
* Run: node benchmarks/chunk-probe/run-probe.mjs (from waggle-os-w4 root)
|
||
* Requires: Ollama up with nomic-embed-text-8k; hive-mind-core dist built.
|
||
* Cost: $0 (all local). The source mind is NEVER touched (sqlite backup API).
|
||
*/
|
||
import { pathToFileURL, fileURLToPath } from 'node:url';
|
||
import path from 'node:path';
|
||
import fs from 'node:fs';
|
||
|
||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||
const CORE = pathToFileURL(path.join(ROOT, 'packages/hive-mind-core/dist/index.js')).href;
|
||
const SOURCE_MIND = 'C:/Users/MarkoMarkovic/.hive-mind/personal.mind';
|
||
const PROBE_DIR = path.join(ROOT, 'benchmarks/chunk-probe/data');
|
||
const PROBE_MIND = path.join(PROBE_DIR, 'probe.mind');
|
||
|
||
const MIN_FRAME_LEN = 2500; // only frames where chunking can matter
|
||
const MAX_NEEDLES = 60;
|
||
const K = 10;
|
||
|
||
const { MindDB, HybridSearch, createOllamaEmbedder, rechunkAllFrames } = await import(CORE);
|
||
|
||
// ── 1. Copy the mind (sqlite backup API — consistent snapshot, source untouched)
|
||
fs.mkdirSync(PROBE_DIR, { recursive: true });
|
||
for (const suffix of ['', '-wal', '-shm']) {
|
||
const p = PROBE_MIND + suffix;
|
||
if (fs.existsSync(p)) fs.rmSync(p);
|
||
}
|
||
{
|
||
const BetterSqlite3 = (await import(pathToFileURL(path.join(ROOT, 'node_modules/better-sqlite3/lib/index.js')).href)).default;
|
||
const src = new BetterSqlite3(SOURCE_MIND, { readonly: true });
|
||
await src.backup(PROBE_MIND);
|
||
src.close();
|
||
}
|
||
console.log('[probe] mind copied to', PROBE_MIND);
|
||
|
||
// ── 2. Open with mono substrate + real local embedder
|
||
const db = new MindDB(PROBE_MIND);
|
||
const embedder = createOllamaEmbedder({ model: 'nomic-embed-text-8k', targetDimensions: 1024 });
|
||
const search = new HybridSearch(db, embedder);
|
||
|
||
// ── 3. Wipe mock vectors, re-embed all frames (whole-frame control vectors)
|
||
db.recreateVecTables(1024);
|
||
const raw = db.getDatabase();
|
||
const frames = raw.prepare('SELECT id, content FROM memory_frames ORDER BY id').all();
|
||
console.log(`[probe] re-embedding ${frames.length} frames (whole-frame vectors)…`);
|
||
const BATCH = 16;
|
||
for (let i = 0; i < frames.length; i += BATCH) {
|
||
// Cap at 4,000 chars. LIVE FINDING: the '-8k' model's underlying nomic-bert
|
||
// architecture caps at 2048 TOKENS (ollama show: nomic-bert.context_length
|
||
// 2048) and Ollama enforces it — R5's maxEmbedCharsForModel trusts the
|
||
// model NAME (24k chars) and would 400, then per-text-fallback every long
|
||
// frame to a MOCK vector in production. Whole-frame embedding beyond the
|
||
// real cap is impossible — that blindness is what the chunk lane (≤2k
|
||
// chars ≈ 500 tokens) structurally fixes.
|
||
const batch = frames.slice(i, i + BATCH).map(f => ({ id: f.id, content: f.content.slice(0, 4_000) }));
|
||
await search.indexFramesBatch(batch);
|
||
if ((i / BATCH) % 5 === 0) process.stdout.write(` ${Math.min(i + BATCH, frames.length)}/${frames.length}\r`);
|
||
}
|
||
console.log(`\n[probe] whole-frame vectors: ${raw.prepare('SELECT COUNT(*) n FROM memory_frames_vec').get().n}`);
|
||
|
||
// ── 4. Chunk backfill (treatment vectors; flag-independent helper)
|
||
const rechunk = await rechunkAllFrames(db, search);
|
||
console.log('[probe] rechunk:', JSON.stringify(rechunk));
|
||
const chunkStats = raw.prepare(
|
||
`SELECT COUNT(*) chunks, SUM(cnt > 1) multi FROM (
|
||
SELECT frame_id, COUNT(*) cnt FROM memory_frame_chunks GROUP BY frame_id)`
|
||
).get();
|
||
console.log(`[probe] chunks: ${chunkStats.chunks} total, ${chunkStats.multi} frames with >1 chunk`);
|
||
|
||
// ── 5. Needle extraction: verbatim sentence from 50–90% depth of long frames
|
||
function extractNeedle(content) {
|
||
const start = Math.floor(content.length * 0.5);
|
||
const end = Math.floor(content.length * 0.9);
|
||
const slice = content.slice(start, end);
|
||
const sentences = slice.split(/(?<=[.!?])\s+|\n+/).map(s => s.trim());
|
||
const candidates = sentences.filter(s => s.length >= 80 && s.length <= 250 && s.split(/\s+/).length >= 10);
|
||
if (candidates.length === 0) return null;
|
||
// deterministic pick: the longest candidate (most distinctive)
|
||
return candidates.sort((a, b) => b.length - a.length)[0];
|
||
}
|
||
const longFrames = frames.filter(f => f.content.length >= MIN_FRAME_LEN);
|
||
const needles = [];
|
||
for (const f of longFrames) {
|
||
const n = extractNeedle(f.content);
|
||
// beyondCap: needle text starts past the 8k whole-frame embed cap — content
|
||
// structurally invisible to the whole-frame vector (chunking's strongest case).
|
||
if (n) needles.push({ frameId: f.id, needle: n, frameLen: f.content.length, beyondCap: f.content.indexOf(n) >= 4_000 });
|
||
if (needles.length >= MAX_NEEDLES) break;
|
||
}
|
||
console.log(`[probe] ${needles.length} needles from ${longFrames.length} long frames (>=${MIN_FRAME_LEN} chars)`);
|
||
|
||
// ── 6. Paired cells
|
||
let stats = { a1: 0, a5: 0, a10: 0, b1: 0, b5: 0, b10: 0, bOnly5: 0, aOnly5: 0 };
|
||
let strat = { withinCap: { n: 0, a5: 0, b5: 0 }, beyondCap: { n: 0, a5: 0, b5: 0 } };
|
||
const rows = [];
|
||
for (const { frameId, needle, frameLen, beyondCap } of needles) {
|
||
const a = await search.vectorSearch(needle, K);
|
||
const b = (await search.vectorSearchChunks(needle, K)) ?? [];
|
||
const ra = a.indexOf(frameId);
|
||
const rb = b.indexOf(frameId);
|
||
if (ra === 0) stats.a1++; if (ra >= 0 && ra < 5) stats.a5++; if (ra >= 0) stats.a10++;
|
||
if (rb === 0) stats.b1++; if (rb >= 0 && rb < 5) stats.b5++; if (rb >= 0) stats.b10++;
|
||
const aHit5 = ra >= 0 && ra < 5, bHit5 = rb >= 0 && rb < 5;
|
||
if (bHit5 && !aHit5) stats.bOnly5++;
|
||
if (aHit5 && !bHit5) stats.aOnly5++;
|
||
const s = beyondCap ? strat.beyondCap : strat.withinCap;
|
||
s.n++; if (aHit5) s.a5++; if (bHit5) s.b5++;
|
||
rows.push({ frameId, frameLen, beyondCap, rankWhole: ra, rankChunk: rb, needle: needle.slice(0, 80) });
|
||
}
|
||
|
||
// ── 7. Report
|
||
const n = needles.length;
|
||
console.log('\n========== D1 CHUNK-LANE PROBE RESULT ==========');
|
||
console.log(`corpus: copy of personal.mind | needles: ${n} | embedder: nomic-embed-text-8k/1024`);
|
||
console.log(` hit@1 hit@5 hit@10`);
|
||
console.log(`whole-frame ${stats.a1}/${n} ${stats.a5}/${n} ${stats.a10}/${n}`);
|
||
console.log(`chunk-lane ${stats.b1}/${n} ${stats.b5}/${n} ${stats.b10}/${n}`);
|
||
console.log(`paired hit@5: chunk-only wins ${stats.bOnly5}, whole-only wins ${stats.aOnly5} (discordant pairs)`);
|
||
console.log(`strata hit@5: within-cap n=${strat.withinCap.n} whole ${strat.withinCap.a5} vs chunk ${strat.withinCap.b5}` +
|
||
` | beyond-cap n=${strat.beyondCap.n} whole ${strat.beyondCap.a5} vs chunk ${strat.beyondCap.b5}`);
|
||
fs.writeFileSync(path.join(PROBE_DIR, 'probe-result.json'), JSON.stringify({ n, stats, rows }, null, 2));
|
||
console.log('[probe] per-needle detail → benchmarks/chunk-probe/data/probe-result.json');
|
||
db.close();
|