/** * Search-quality benchmark for memory (P1.5). * * Loads the reference set (`reference-set.json`) into the mem0 store and a * cosine baseline, runs every query against each, and prints recall@1, * recall@5, MRR and latency per engine and per query type. The comparison with * the engine mem0 replaced is in docs/oss/research/memory-search-quality.md. * * Run from the repository root after `npm run build:packages`: * * npx tsx packages/memory-mem0/bench/search-quality.ts [options] * * --engines Comma-separated: mem0, mem0-rerank, cosine. Default: * every engine. `mem0-rerank` re-orders mem0's results * with the in-process cross-encoder. `cosine` is a * baseline: plain cosine similarity over the same * embeddings, with no keyword signal. * --embedder inprocess (default, offline), ollama or openai. * --limit Results per query (default 10). * --out Also write the per-query results as JSON. * * Embedders: * inprocess Xenova/all-MiniLM-L6-v2 padded to 1024 dimensions, as in the * sidecar, loaded from WAGGLE_MODELS_DIR (default ~/.waggle/models). * ollama OLLAMA_HOST (default http://localhost:11434) and * OLLAMA_EMBED_MODEL (default nomic-embed-text). Ollama must * already be running with the model pulled. * openai The user's own key: BENCH_EMBEDDING_API_KEY (required), * BENCH_EMBEDDING_MODEL (default text-embedding-3-small) and * BENCH_EMBEDDING_BASE_URL (default the OpenAI endpoint). The bench * never reads OPENAI_API_KEY, so a key that happens to be set in * the environment is not used. * * Every fetch() to another machine is refused, except the endpoint of the * openai embedder. mem0 telemetry is switched off before mem0 is loaded. */ import fs from 'node:fs'; import { createRequire } from 'node:module'; import os from 'node:os'; import path from 'node:path'; import { performance } from 'node:perf_hooks'; import { fileURLToPath } from 'node:url'; import type { MemoryImportance, MemoryKind } from '@waggle/memory-contract'; // The adapter sets this too; setting it first keeps telemetry off even if // something loads mem0 earlier than the adapter. process.env.MEM0_TELEMETRY = 'false'; const BENCH_DIR = path.dirname(fileURLToPath(import.meta.url)); const ENGINE_NAMES = ['mem0', 'mem0-rerank', 'cosine'] as const; type EngineName = (typeof ENGINE_NAMES)[number]; type BackendName = 'mem0' | 'cosine'; const backendOf = (engine: EngineName): BackendName => engine.replace(/-rerank$/, '') as BackendName; /** Candidates a reranker re-orders. */ const RERANK_POOL = 30; interface MemoryFixture { id: string; scope: string; kind: string; importance: string; text: string } interface QueryFixture { id: string; type: string; scopes: string[]; text: string; relevant: string[] } interface ReferenceSet { memories: MemoryFixture[]; queries: QueryFixture[] } interface Embedder { readonly dimensions: number; embed(text: string): Promise; embedBatch(texts: string[]): Promise; } interface Reranker { scoreBatch(query: string, docs: string[]): Promise } interface Hit { ref: string; text: string; score: number } interface Backend { readonly ingestMs: number; search(text: string, scopes: string[], limit: number, reranker?: Reranker): Promise; close(): Promise; } type RerankerFactory = (config?: { cacheDir?: string }) => Promise; interface Options { engines: EngineName[] | null; embedder: 'inprocess' | 'ollama' | 'openai'; limit: number; out: string | null; } function parseArgs(argv: string[]): Options { const options: Options = { engines: null, embedder: 'inprocess', limit: 10, out: null }; for (let i = 0; i < argv.length; i++) { const value = argv[i + 1]; switch (argv[i]) { case '--engines': options.engines = value.split(',').map((name) => { if (!(ENGINE_NAMES as readonly string[]).includes(name)) throw new Error(`unknown engine: ${name}`); return name as EngineName; }); i++; break; case '--embedder': if (value !== 'inprocess' && value !== 'ollama' && value !== 'openai') { throw new Error(`unknown embedder: ${value}`); } options.embedder = value; i++; break; case '--limit': options.limit = Number(value); if (!Number.isInteger(options.limit) || options.limit < 5) throw new Error('--limit must be an integer >= 5'); i++; break; case '--out': options.out = path.resolve(value); i++; break; default: throw new Error(`unknown argument: ${argv[i]}`); } } return options; } // ── Network guard ──────────────────────────────────────────────────────────── const LOOPBACK = new Set(['localhost', '127.0.0.1', '::1', '[::1]']); const requests = { local: 0, remote: 0, blocked: [] as string[] }; function guardNetwork(allowedRemoteHost: string | null): void { const realFetch = globalThis.fetch; globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url); if (LOOPBACK.has(url.hostname)) requests.local++; else if (url.hostname === allowedRemoteHost) requests.remote++; else { requests.blocked.push(url.host); throw new Error(`search-quality bench: blocked a request to ${url.host}`); } return realFetch(input, init); }) as typeof fetch; } // ── Embedders ──────────────────────────────────────────────────────────────── interface EmbedderPlan { label: string; remoteHost: string | null; create(core: Record): Promise } function planEmbedder(kind: Options['embedder'], modelsDir: string): EmbedderPlan { type Factory = (config: C) => Embedder | Promise; if (kind === 'inprocess') { return { label: 'in-process Xenova/all-MiniLM-L6-v2 (384 dims padded to 1024)', remoteHost: null, create: async (core) => (core.createInProcessEmbedder as Factory)({ model: 'Xenova/all-MiniLM-L6-v2', cacheDir: modelsDir, targetDimensions: 1024, }), }; } if (kind === 'ollama') { const baseUrl = process.env.OLLAMA_HOST ?? 'http://localhost:11434'; const model = process.env.OLLAMA_EMBED_MODEL ?? 'nomic-embed-text'; if (!LOOPBACK.has(new URL(baseUrl).hostname)) throw new Error('OLLAMA_HOST must point at this machine'); return { label: `Ollama ${model} at ${baseUrl} (padded or cut to 1024 dims)`, remoteHost: null, create: async (core) => { const embedder = await (core.createOllamaEmbedder as Factory)({ baseUrl, model, targetDimensions: 1024 }); try { await embedder.embed('probe'); } catch (error) { throw new Error(`Ollama does not answer at ${baseUrl} with model ${model}. Start it and pull the model first; ` + `the bench does neither. (${error instanceof Error ? error.message : String(error)})`); } return embedder; }, }; } const apiKey = process.env.BENCH_EMBEDDING_API_KEY; if (!apiKey) throw new Error('--embedder openai needs BENCH_EMBEDDING_API_KEY (OPENAI_API_KEY is deliberately ignored)'); const model = process.env.BENCH_EMBEDDING_MODEL ?? 'text-embedding-3-small'; const baseUrl = process.env.BENCH_EMBEDDING_BASE_URL ?? 'https://api.openai.com/v1/embeddings'; return { label: `OpenAI-compatible ${model} at ${new URL(baseUrl).host} (padded or cut to 1024 dims)`, remoteHost: new URL(baseUrl).hostname, create: async (core) => (core.createApiEmbedder as Factory)({ provider: 'openai', apiKey, model, baseUrl, targetDimensions: 1024, }), }; } // ── Backends ───────────────────────────────────────────────────────────────── async function createMem0Backend(set: ReferenceSet, embedder: Embedder, dir: string): Promise { const { createMem0MemoryStore } = await import('../src/index.js'); const { scopeFromKey } = await import('@waggle/memory-contract'); const toScope = (key: string) => { const scope = scopeFromKey(key); if (!scope) throw new Error(`bad scope key: ${key}`); return scope; }; const store = await createMem0MemoryStore({ directory: dir, embedder }); const refs = new Map(); const started = performance.now(); for (const memory of set.memories) { const [change] = await store.remember({ scope: toScope(memory.scope), text: memory.text, kind: memory.kind as MemoryKind, importance: memory.importance as MemoryImportance, source: 'user_stated', origin: 'bench', }); refs.set(change.record.id, memory.id); } const ingestMs = performance.now() - started; return { ingestMs, async search(text, scopes, limit, reranker) { const results = await store.search(text, { scopes: scopes.map(toScope), limit: reranker ? RERANK_POOL : limit }); const hits = results.map((r) => ({ ref: refs.get(r.id)!, text: r.text, score: r.score })); return rerankTop(text, hits, limit, reranker); }, close: () => store.close(), }; } /** Baseline: rank every memory of the searched scopes by cosine similarity alone. */ async function createCosineBackend(set: ReferenceSet, embedder: Embedder): Promise { const started = performance.now(); const vectors = await embedder.embedBatch(set.memories.map((m) => m.text)); const ingestMs = performance.now() - started; const cosine = (a: Float32Array, b: Float32Array) => { let dot = 0, na = 0, nb = 0; for (let i = 0; i < a.length; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; } return dot / (Math.sqrt(na) * Math.sqrt(nb) || 1); }; return { ingestMs, async search(text, scopes, limit, reranker) { const query = await embedder.embed(text); const hits = set.memories .map((m, i) => ({ memory: m, score: cosine(query, vectors[i]) })) .filter(({ memory }) => scopes.includes(memory.scope)) .sort((a, b) => b.score - a.score) .slice(0, reranker ? RERANK_POOL : limit) .map(({ memory, score }) => ({ ref: memory.id, text: memory.text, score })); return rerankTop(text, hits, limit, reranker); }, close: async () => {}, }; } async function rerankTop(text: string, hits: Hit[], limit: number, reranker?: Reranker): Promise { if (!reranker || hits.length === 0) return hits.slice(0, limit); const scores = await reranker.scoreBatch(text, hits.map((h) => h.text)); return hits .map((hit, i) => ({ ...hit, score: scores[i] })) .sort((a, b) => b.score - a.score) .slice(0, limit); } // ── Metrics ────────────────────────────────────────────────────────────────── interface QueryResult { query: string; type: string; scopes: string[]; relevant: string[]; ranked: Array<{ ref: string; score: number }>; /** 1-based rank of the first relevant memory within the results, or null. */ firstRank: number | null; recallAt1: number | null; recallAt5: number | null; reciprocalRank: number | null; latencyMs: number; } /** Share of the relevant memories found in the top k, out of the most that fit (min(k, relevant)). */ function recallAt(ranked: string[], relevant: Set, k: number): number { const found = ranked.slice(0, k).filter((ref) => relevant.has(ref)).length; return found / Math.min(k, relevant.size); } function evaluate(query: QueryFixture, hits: Hit[], latencyMs: number): QueryResult { const relevant = new Set(query.relevant); const ranked = hits.map((h) => h.ref); const index = ranked.findIndex((ref) => relevant.has(ref)); const firstRank = index >= 0 ? index + 1 : null; const answerable = relevant.size > 0; return { query: query.id, type: query.type, scopes: query.scopes, relevant: query.relevant, ranked: hits.map((h) => ({ ref: h.ref, score: Number(h.score.toFixed(4)) })), firstRank, recallAt1: answerable ? recallAt(ranked, relevant, 1) : null, recallAt5: answerable ? recallAt(ranked, relevant, 5) : null, reciprocalRank: answerable ? (firstRank ? 1 / firstRank : 0) : null, latencyMs, }; } interface Summary { n: number; recallAt1: number; recallAt5: number; mrr: number } function summarize(results: QueryResult[]): Summary { const answerable = results.filter((r) => r.relevant.length > 0); const mean = (pick: (r: QueryResult) => number | null) => answerable.reduce((sum, r) => sum + (pick(r) ?? 0), 0) / (answerable.length || 1); return { n: answerable.length, recallAt1: mean((r) => r.recallAt1), recallAt5: mean((r) => r.recallAt5), mrr: mean((r) => r.reciprocalRank), }; } function percentile(values: number[], p: number): number { const sorted = [...values].sort((a, b) => a - b); return sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))] ?? 0; } /** * How well the top score separates a correct first hit from a query memory * cannot answer: the chance that a correct top-1 outscores a no-answer top-1 * (ties count half). 1.0 means a score threshold could tell them apart. */ function abstainSeparation(results: QueryResult[]): number | null { const correct = results.filter((r) => r.firstRank === 1).map((r) => r.ranked[0].score); const empty = results.filter((r) => r.relevant.length === 0).map((r) => r.ranked[0]?.score ?? -Infinity); if (correct.length === 0 || empty.length === 0) return null; let wins = 0; for (const c of correct) for (const e of empty) wins += c > e ? 1 : c === e ? 0.5 : 0; return wins / (correct.length * empty.length); } // ── Report ─────────────────────────────────────────────────────────────────── const f2 = (value: number) => value.toFixed(2); const f1 = (value: number) => value.toFixed(1); function table(header: string[], rows: string[][]): string { return [ `| ${header.join(' | ')} |`, `|${header.map(() => '---').join('|')}|`, ...rows.map((row) => `| ${row.join(' | ')} |`), ].join('\n'); } function report(set: ReferenceSet, runs: Map, setup: string[]): string { const engines = [...runs.keys()]; const out: string[] = ['# Memory search quality', '', ...setup.map((line) => `- ${line}`), '']; const cell = (s: Summary) => `${f2(s.recallAt1)} / ${f2(s.recallAt5)} / ${f2(s.mrr)}`; out.push('## Overall (answerable queries)', ''); out.push(table( ['engine', 'n', 'recall@1', 'recall@5', 'MRR', 'median ms', 'p95 ms', 'abstain separation'], engines.map((engine) => { const results = runs.get(engine)!; const s = summarize(results); const latencies = results.map((r) => r.latencyMs); const separation = abstainSeparation(results); return [engine, String(s.n), f2(s.recallAt1), f2(s.recallAt5), f2(s.mrr), f1(percentile(latencies, 0.5)), f1(percentile(latencies, 0.95)), separation === null ? '-' : f2(separation)]; }), ), ''); const types = [...new Set(set.queries.map((q) => q.type))].filter((type) => type !== 'no-answer'); out.push('## By query type (recall@1 / recall@5 / MRR)', ''); out.push(table( ['type', 'n', ...engines], types.map((type) => [ type, String(set.queries.filter((q) => q.type === type).length), ...engines.map((engine) => cell(summarize(runs.get(engine)!.filter((r) => r.type === type)))), ]), ), ''); out.push('## By number of scopes searched (recall@1 / recall@5 / MRR)', ''); out.push(table( ['scopes', 'n', ...engines], [['one', (n: number) => n === 1], ['two or more', (n: number) => n > 1]].map(([label, test]) => { const pick = (r: QueryResult) => (test as (n: number) => boolean)(r.scopes.length) && r.relevant.length > 0; return [ label as string, String(runs.get(engines[0])!.filter(pick).length), ...engines.map((engine) => cell(summarize(runs.get(engine)!.filter(pick)))), ]; }), ), ''); out.push('## Misses', ''); out.push('Answerable queries where at least one engine did not rank a relevant memory first. ' + 'Each engine cell: rank of the first relevant memory ("-" if not in the results), recall@5, and the top result.', ''); const missRows: string[][] = []; for (const query of set.queries.filter((q) => q.relevant.length > 0)) { const per = engines.map((engine) => runs.get(engine)!.find((r) => r.query === query.id)!); if (per.every((r) => r.firstRank === 1 && r.recallAt5 === 1)) continue; missRows.push([ query.id, query.type, query.text, query.relevant.join(' '), ...per.map((r) => `${r.firstRank ?? '-'} · ${f2(r.recallAt5 ?? 0)} · ${r.ranked[0]?.ref ?? '-'}`), ]); } out.push(table(['query', 'type', 'text', 'relevant', ...engines], missRows), ''); out.push('## No-answer queries (top result and its score)', ''); out.push(table( ['query', 'text', ...engines], set.queries.filter((q) => q.relevant.length === 0).map((query) => [ query.id, query.text, ...engines.map((engine) => { const top = runs.get(engine)!.find((r) => r.query === query.id)!.ranked[0]; return top ? `${top.ref} (${top.score.toFixed(3)})` : '-'; }), ]), ), ''); return out.join('\n'); } function packageVersion(fromDir: string, name: string): string { try { let dir = path.dirname(createRequire(path.join(fromDir, 'package.json')).resolve(name)); while (dir !== path.dirname(dir)) { const file = path.join(dir, 'package.json'); if (fs.existsSync(file)) { const pkg = JSON.parse(fs.readFileSync(file, 'utf8')) as { name?: string; version?: string }; if (pkg.name === name) return pkg.version ?? 'unknown'; } dir = path.dirname(dir); } } catch { /* reported as unknown */ } return 'unknown'; } // ── Main ───────────────────────────────────────────────────────────────────── async function main(): Promise { const options = parseArgs(process.argv.slice(2)); const set = JSON.parse(fs.readFileSync(path.join(BENCH_DIR, 'reference-set.json'), 'utf8')) as ReferenceSet; const modelsDir = process.env.WAGGLE_MODELS_DIR ?? path.join(os.homedir(), '.waggle', 'models'); const plan = planEmbedder(options.embedder, modelsDir); guardNetwork(plan.remoteHost); const core = await import('@waggle/core') as unknown as Record; const wanted = options.engines ?? [...ENGINE_NAMES]; const embedder = await plan.create(core); let reranker: Reranker | undefined; if (wanted.some((name) => name.endsWith('-rerank'))) { // The sidecar caches the reranker under /reranker (packages/server/src/local/index.ts). reranker = await (core.createInProcessReranker as RerankerFactory)({ cacheDir: path.join(modelsDir, 'reranker') }); } const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-search-bench-')); const backends = new Map(); try { const needed = new Set(wanted.map(backendOf)); if (needed.has('mem0')) { backends.set('mem0', await createMem0Backend(set, embedder, fs.mkdtempSync(path.join(tempRoot, 'mem0-')))); } if (needed.has('cosine')) backends.set('cosine', await createCosineBackend(set, embedder)); const runs = new Map(); for (const engine of wanted) { const backend = backends.get(backendOf(engine))!; const engineReranker = engine.endsWith('-rerank') ? reranker : undefined; // Warm up so the first timed query does not pay for lazy initialisation. await backend.search('warm-up query', ['personal'], options.limit, engineReranker); const results: QueryResult[] = []; for (const query of set.queries) { const started = performance.now(); const hits = await backend.search(query.text, query.scopes, options.limit, engineReranker); results.push(evaluate(query, hits, performance.now() - started)); } runs.set(engine, results); } const memo = (name: BackendName) => backends.get(name) ? `${f1(backends.get(name)!.ingestMs / 1000)} s` : 'not run'; const setup = [ `Reference set: ${set.memories.length} memories, ${set.queries.length} queries (${set.queries.filter((q) => q.relevant.length === 0).length} without an answer)`, `Embedder: ${plan.label}`, `Results per query: ${options.limit}; reranker pool: ${RERANK_POOL}`, `mem0ai ${packageVersion(path.join(BENCH_DIR, '..'), 'mem0ai')}, @huggingface/transformers ${packageVersion(path.join(BENCH_DIR, '..', '..', 'core'), '@huggingface/transformers')}, Node ${process.version}`, `Machine: ${os.cpus()[0]?.model ?? 'unknown CPU'}, ${os.cpus().length} cores, ${Math.round(os.totalmem() / 2 ** 30)} GB, ${os.platform()} ${os.release()}`, `Load time: mem0 ${memo('mem0')}, cosine ${memo('cosine')} (embedding included)`, `Network: ${requests.local} local requests, ${requests.remote} to the embedding API, ${requests.blocked.length} blocked`, ]; console.log(report(set, runs, setup)); if (options.out) { fs.writeFileSync(options.out, `${JSON.stringify({ setup, results: Object.fromEntries(runs) }, null, 2)}\n`); console.error(`Wrote ${options.out}`); } if (requests.blocked.length > 0) { throw new Error(`blocked network requests: ${requests.blocked.join(', ')}`); } } finally { for (const backend of backends.values()) await backend.close(); fs.rmSync(tempRoot, { recursive: true, force: true }); } } main().catch((error: unknown) => { console.error(error instanceof Error ? error.stack ?? error.message : error); process.exitCode = 1; });