Files
waggle-os/packages/memory-mem0/bench/search-quality.ts
shumeiko.it b275f0bf89 feat: default the data directory to ~/.waggle-community (P1.3c)
The community edition defaulted to ~/.waggle, the data directory of the
commercial Waggle, so both editions would open the same config, vault,
waggle.db, marketplace catalog and skills. Default to
~/.waggle-community instead (ADR-0011). WAGGLE_DATA_DIR still overrides
it wherever it did; nothing is migrated from ~/.waggle.

The name is defined once as DEFAULT_DATA_DIR_NAME and
DEFAULT_DATA_DIR_LABEL in @waggle/shared, with defaultDataDir() in
@waggle/core for the sidecar, the CLI, the agent and the model caches.
The marketplace installs on its own without workspace packages, so it
keeps a copy in its data-dir.ts; the CLI help, which loads no
dependency, spells the path out and its runtime test checks it.

The marketplace security scan also flags skills that reference the new
skills folder. Tests that asserted the old default now use the
constant, and comments name the new directory.
2026-11-04 10:31:20 +01:00

502 lines
22 KiB
TypeScript

/**
* Search-quality benchmark for memory (P1.5).
*
* Loads the reference set (`reference-set.json`) into the mem0 store and a
* cosine baseline, runs every query against each, and prints recall@1,
* recall@5, MRR and latency per engine and per query type. The comparison with
* the engine mem0 replaced is in docs/oss/research/memory-search-quality.md.
*
* Run from the repository root after `npm run build:packages`:
*
* npx tsx packages/memory-mem0/bench/search-quality.ts [options]
*
* --engines <list> Comma-separated: mem0, mem0-rerank, cosine. Default:
* every engine. `mem0-rerank` re-orders mem0's results
* with the in-process cross-encoder. `cosine` is a
* baseline: plain cosine similarity over the same
* embeddings, with no keyword signal.
* --embedder <name> inprocess (default, offline), ollama or openai.
* --limit <n> Results per query (default 10).
* --out <file> Also write the per-query results as JSON.
*
* Embedders:
* inprocess Xenova/all-MiniLM-L6-v2 padded to 1024 dimensions, as in the
* sidecar, loaded from WAGGLE_MODELS_DIR (default ~/.waggle-community/models).
* ollama OLLAMA_HOST (default http://localhost:11434) and
* OLLAMA_EMBED_MODEL (default nomic-embed-text). Ollama must
* already be running with the model pulled.
* openai The user's own key: BENCH_EMBEDDING_API_KEY (required),
* BENCH_EMBEDDING_MODEL (default text-embedding-3-small) and
* BENCH_EMBEDDING_BASE_URL (default the OpenAI endpoint). The bench
* never reads OPENAI_API_KEY, so a key that happens to be set in
* the environment is not used.
*
* Every fetch() to another machine is refused, except the endpoint of the
* openai embedder. mem0 telemetry is switched off before mem0 is loaded.
*/
import fs from 'node:fs';
import { createRequire } from 'node:module';
import os from 'node:os';
import path from 'node:path';
import { performance } from 'node:perf_hooks';
import { fileURLToPath } from 'node:url';
import type { MemoryImportance, MemoryKind } from '@waggle/memory-contract';
import { DEFAULT_DATA_DIR_NAME } from '@waggle/shared';
// The adapter sets this too; setting it first keeps telemetry off even if
// something loads mem0 earlier than the adapter.
process.env.MEM0_TELEMETRY = 'false';
const BENCH_DIR = path.dirname(fileURLToPath(import.meta.url));
const ENGINE_NAMES = ['mem0', 'mem0-rerank', 'cosine'] as const;
type EngineName = (typeof ENGINE_NAMES)[number];
type BackendName = 'mem0' | 'cosine';
const backendOf = (engine: EngineName): BackendName => engine.replace(/-rerank$/, '') as BackendName;
/** Candidates a reranker re-orders. */
const RERANK_POOL = 30;
interface MemoryFixture { id: string; scope: string; kind: string; importance: string; text: string }
interface QueryFixture { id: string; type: string; scopes: string[]; text: string; relevant: string[] }
interface ReferenceSet { memories: MemoryFixture[]; queries: QueryFixture[] }
interface Embedder {
readonly dimensions: number;
embed(text: string): Promise<Float32Array>;
embedBatch(texts: string[]): Promise<Float32Array[]>;
}
interface Reranker { scoreBatch(query: string, docs: string[]): Promise<number[]> }
interface Hit { ref: string; text: string; score: number }
interface Backend {
readonly ingestMs: number;
search(text: string, scopes: string[], limit: number, reranker?: Reranker): Promise<Hit[]>;
close(): Promise<void>;
}
type RerankerFactory = (config?: { cacheDir?: string }) => Promise<Reranker>;
interface Options {
engines: EngineName[] | null;
embedder: 'inprocess' | 'ollama' | 'openai';
limit: number;
out: string | null;
}
function parseArgs(argv: string[]): Options {
const options: Options = { engines: null, embedder: 'inprocess', limit: 10, out: null };
for (let i = 0; i < argv.length; i++) {
const value = argv[i + 1];
switch (argv[i]) {
case '--engines':
options.engines = value.split(',').map((name) => {
if (!(ENGINE_NAMES as readonly string[]).includes(name)) throw new Error(`unknown engine: ${name}`);
return name as EngineName;
});
i++;
break;
case '--embedder':
if (value !== 'inprocess' && value !== 'ollama' && value !== 'openai') {
throw new Error(`unknown embedder: ${value}`);
}
options.embedder = value;
i++;
break;
case '--limit':
options.limit = Number(value);
if (!Number.isInteger(options.limit) || options.limit < 5) throw new Error('--limit must be an integer >= 5');
i++;
break;
case '--out':
options.out = path.resolve(value);
i++;
break;
default:
throw new Error(`unknown argument: ${argv[i]}`);
}
}
return options;
}
// ── Network guard ────────────────────────────────────────────────────────────
const LOOPBACK = new Set(['localhost', '127.0.0.1', '::1', '[::1]']);
const requests = { local: 0, remote: 0, blocked: [] as string[] };
function guardNetwork(allowedRemoteHost: string | null): void {
const realFetch = globalThis.fetch;
globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => {
const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url);
if (LOOPBACK.has(url.hostname)) requests.local++;
else if (url.hostname === allowedRemoteHost) requests.remote++;
else {
requests.blocked.push(url.host);
throw new Error(`search-quality bench: blocked a request to ${url.host}`);
}
return realFetch(input, init);
}) as typeof fetch;
}
// ── Embedders ────────────────────────────────────────────────────────────────
interface EmbedderPlan { label: string; remoteHost: string | null; create(core: Record<string, unknown>): Promise<Embedder> }
function planEmbedder(kind: Options['embedder'], modelsDir: string): EmbedderPlan {
type Factory<C> = (config: C) => Embedder | Promise<Embedder>;
if (kind === 'inprocess') {
return {
label: 'in-process Xenova/all-MiniLM-L6-v2 (384 dims padded to 1024)',
remoteHost: null,
create: async (core) => (core.createInProcessEmbedder as Factory<object>)({
model: 'Xenova/all-MiniLM-L6-v2', cacheDir: modelsDir, targetDimensions: 1024,
}),
};
}
if (kind === 'ollama') {
const baseUrl = process.env.OLLAMA_HOST ?? 'http://localhost:11434';
const model = process.env.OLLAMA_EMBED_MODEL ?? 'nomic-embed-text';
if (!LOOPBACK.has(new URL(baseUrl).hostname)) throw new Error('OLLAMA_HOST must point at this machine');
return {
label: `Ollama ${model} at ${baseUrl} (padded or cut to 1024 dims)`,
remoteHost: null,
create: async (core) => {
const embedder = await (core.createOllamaEmbedder as Factory<object>)({ baseUrl, model, targetDimensions: 1024 });
try {
await embedder.embed('probe');
} catch (error) {
throw new Error(`Ollama does not answer at ${baseUrl} with model ${model}. Start it and pull the model first; `
+ `the bench does neither. (${error instanceof Error ? error.message : String(error)})`);
}
return embedder;
},
};
}
const apiKey = process.env.BENCH_EMBEDDING_API_KEY;
if (!apiKey) throw new Error('--embedder openai needs BENCH_EMBEDDING_API_KEY (OPENAI_API_KEY is deliberately ignored)');
const model = process.env.BENCH_EMBEDDING_MODEL ?? 'text-embedding-3-small';
const baseUrl = process.env.BENCH_EMBEDDING_BASE_URL ?? 'https://api.openai.com/v1/embeddings';
return {
label: `OpenAI-compatible ${model} at ${new URL(baseUrl).host} (padded or cut to 1024 dims)`,
remoteHost: new URL(baseUrl).hostname,
create: async (core) => (core.createApiEmbedder as Factory<object>)({
provider: 'openai', apiKey, model, baseUrl, targetDimensions: 1024,
}),
};
}
// ── Backends ─────────────────────────────────────────────────────────────────
async function createMem0Backend(set: ReferenceSet, embedder: Embedder, dir: string): Promise<Backend> {
const { createMem0MemoryStore } = await import('../src/index.js');
const { scopeFromKey } = await import('@waggle/memory-contract');
const toScope = (key: string) => {
const scope = scopeFromKey(key);
if (!scope) throw new Error(`bad scope key: ${key}`);
return scope;
};
const store = await createMem0MemoryStore({ directory: dir, embedder });
const refs = new Map<string, string>();
const started = performance.now();
for (const memory of set.memories) {
const [change] = await store.remember({
scope: toScope(memory.scope),
text: memory.text,
kind: memory.kind as MemoryKind,
importance: memory.importance as MemoryImportance,
source: 'user_stated',
origin: 'bench',
});
refs.set(change.record.id, memory.id);
}
const ingestMs = performance.now() - started;
return {
ingestMs,
async search(text, scopes, limit, reranker) {
const results = await store.search(text, { scopes: scopes.map(toScope), limit: reranker ? RERANK_POOL : limit });
const hits = results.map((r) => ({ ref: refs.get(r.id)!, text: r.text, score: r.score }));
return rerankTop(text, hits, limit, reranker);
},
close: () => store.close(),
};
}
/** Baseline: rank every memory of the searched scopes by cosine similarity alone. */
async function createCosineBackend(set: ReferenceSet, embedder: Embedder): Promise<Backend> {
const started = performance.now();
const vectors = await embedder.embedBatch(set.memories.map((m) => m.text));
const ingestMs = performance.now() - started;
const cosine = (a: Float32Array, b: Float32Array) => {
let dot = 0, na = 0, nb = 0;
for (let i = 0; i < a.length; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
return dot / (Math.sqrt(na) * Math.sqrt(nb) || 1);
};
return {
ingestMs,
async search(text, scopes, limit, reranker) {
const query = await embedder.embed(text);
const hits = set.memories
.map((m, i) => ({ memory: m, score: cosine(query, vectors[i]) }))
.filter(({ memory }) => scopes.includes(memory.scope))
.sort((a, b) => b.score - a.score)
.slice(0, reranker ? RERANK_POOL : limit)
.map(({ memory, score }) => ({ ref: memory.id, text: memory.text, score }));
return rerankTop(text, hits, limit, reranker);
},
close: async () => {},
};
}
async function rerankTop(text: string, hits: Hit[], limit: number, reranker?: Reranker): Promise<Hit[]> {
if (!reranker || hits.length === 0) return hits.slice(0, limit);
const scores = await reranker.scoreBatch(text, hits.map((h) => h.text));
return hits
.map((hit, i) => ({ ...hit, score: scores[i] }))
.sort((a, b) => b.score - a.score)
.slice(0, limit);
}
// ── Metrics ──────────────────────────────────────────────────────────────────
interface QueryResult {
query: string;
type: string;
scopes: string[];
relevant: string[];
ranked: Array<{ ref: string; score: number }>;
/** 1-based rank of the first relevant memory within the results, or null. */
firstRank: number | null;
recallAt1: number | null;
recallAt5: number | null;
reciprocalRank: number | null;
latencyMs: number;
}
/** Share of the relevant memories found in the top k, out of the most that fit (min(k, relevant)). */
function recallAt(ranked: string[], relevant: Set<string>, k: number): number {
const found = ranked.slice(0, k).filter((ref) => relevant.has(ref)).length;
return found / Math.min(k, relevant.size);
}
function evaluate(query: QueryFixture, hits: Hit[], latencyMs: number): QueryResult {
const relevant = new Set(query.relevant);
const ranked = hits.map((h) => h.ref);
const index = ranked.findIndex((ref) => relevant.has(ref));
const firstRank = index >= 0 ? index + 1 : null;
const answerable = relevant.size > 0;
return {
query: query.id,
type: query.type,
scopes: query.scopes,
relevant: query.relevant,
ranked: hits.map((h) => ({ ref: h.ref, score: Number(h.score.toFixed(4)) })),
firstRank,
recallAt1: answerable ? recallAt(ranked, relevant, 1) : null,
recallAt5: answerable ? recallAt(ranked, relevant, 5) : null,
reciprocalRank: answerable ? (firstRank ? 1 / firstRank : 0) : null,
latencyMs,
};
}
interface Summary { n: number; recallAt1: number; recallAt5: number; mrr: number }
function summarize(results: QueryResult[]): Summary {
const answerable = results.filter((r) => r.relevant.length > 0);
const mean = (pick: (r: QueryResult) => number | null) =>
answerable.reduce((sum, r) => sum + (pick(r) ?? 0), 0) / (answerable.length || 1);
return {
n: answerable.length,
recallAt1: mean((r) => r.recallAt1),
recallAt5: mean((r) => r.recallAt5),
mrr: mean((r) => r.reciprocalRank),
};
}
function percentile(values: number[], p: number): number {
const sorted = [...values].sort((a, b) => a - b);
return sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))] ?? 0;
}
/**
* How well the top score separates a correct first hit from a query memory
* cannot answer: the chance that a correct top-1 outscores a no-answer top-1
* (ties count half). 1.0 means a score threshold could tell them apart.
*/
function abstainSeparation(results: QueryResult[]): number | null {
const correct = results.filter((r) => r.firstRank === 1).map((r) => r.ranked[0].score);
const empty = results.filter((r) => r.relevant.length === 0).map((r) => r.ranked[0]?.score ?? -Infinity);
if (correct.length === 0 || empty.length === 0) return null;
let wins = 0;
for (const c of correct) for (const e of empty) wins += c > e ? 1 : c === e ? 0.5 : 0;
return wins / (correct.length * empty.length);
}
// ── Report ───────────────────────────────────────────────────────────────────
const f2 = (value: number) => value.toFixed(2);
const f1 = (value: number) => value.toFixed(1);
function table(header: string[], rows: string[][]): string {
return [
`| ${header.join(' | ')} |`,
`|${header.map(() => '---').join('|')}|`,
...rows.map((row) => `| ${row.join(' | ')} |`),
].join('\n');
}
function report(set: ReferenceSet, runs: Map<EngineName, QueryResult[]>, setup: string[]): string {
const engines = [...runs.keys()];
const out: string[] = ['# Memory search quality', '', ...setup.map((line) => `- ${line}`), ''];
const cell = (s: Summary) => `${f2(s.recallAt1)} / ${f2(s.recallAt5)} / ${f2(s.mrr)}`;
out.push('## Overall (answerable queries)', '');
out.push(table(
['engine', 'n', 'recall@1', 'recall@5', 'MRR', 'median ms', 'p95 ms', 'abstain separation'],
engines.map((engine) => {
const results = runs.get(engine)!;
const s = summarize(results);
const latencies = results.map((r) => r.latencyMs);
const separation = abstainSeparation(results);
return [engine, String(s.n), f2(s.recallAt1), f2(s.recallAt5), f2(s.mrr),
f1(percentile(latencies, 0.5)), f1(percentile(latencies, 0.95)), separation === null ? '-' : f2(separation)];
}),
), '');
const types = [...new Set(set.queries.map((q) => q.type))].filter((type) => type !== 'no-answer');
out.push('## By query type (recall@1 / recall@5 / MRR)', '');
out.push(table(
['type', 'n', ...engines],
types.map((type) => [
type,
String(set.queries.filter((q) => q.type === type).length),
...engines.map((engine) => cell(summarize(runs.get(engine)!.filter((r) => r.type === type)))),
]),
), '');
out.push('## By number of scopes searched (recall@1 / recall@5 / MRR)', '');
out.push(table(
['scopes', 'n', ...engines],
[['one', (n: number) => n === 1], ['two or more', (n: number) => n > 1]].map(([label, test]) => {
const pick = (r: QueryResult) => (test as (n: number) => boolean)(r.scopes.length) && r.relevant.length > 0;
return [
label as string,
String(runs.get(engines[0])!.filter(pick).length),
...engines.map((engine) => cell(summarize(runs.get(engine)!.filter(pick)))),
];
}),
), '');
out.push('## Misses', '');
out.push('Answerable queries where at least one engine did not rank a relevant memory first. '
+ 'Each engine cell: rank of the first relevant memory ("-" if not in the results), recall@5, and the top result.', '');
const missRows: string[][] = [];
for (const query of set.queries.filter((q) => q.relevant.length > 0)) {
const per = engines.map((engine) => runs.get(engine)!.find((r) => r.query === query.id)!);
if (per.every((r) => r.firstRank === 1 && r.recallAt5 === 1)) continue;
missRows.push([
query.id, query.type, query.text, query.relevant.join(' '),
...per.map((r) => `${r.firstRank ?? '-'} · ${f2(r.recallAt5 ?? 0)} · ${r.ranked[0]?.ref ?? '-'}`),
]);
}
out.push(table(['query', 'type', 'text', 'relevant', ...engines], missRows), '');
out.push('## No-answer queries (top result and its score)', '');
out.push(table(
['query', 'text', ...engines],
set.queries.filter((q) => q.relevant.length === 0).map((query) => [
query.id, query.text,
...engines.map((engine) => {
const top = runs.get(engine)!.find((r) => r.query === query.id)!.ranked[0];
return top ? `${top.ref} (${top.score.toFixed(3)})` : '-';
}),
]),
), '');
return out.join('\n');
}
function packageVersion(fromDir: string, name: string): string {
try {
let dir = path.dirname(createRequire(path.join(fromDir, 'package.json')).resolve(name));
while (dir !== path.dirname(dir)) {
const file = path.join(dir, 'package.json');
if (fs.existsSync(file)) {
const pkg = JSON.parse(fs.readFileSync(file, 'utf8')) as { name?: string; version?: string };
if (pkg.name === name) return pkg.version ?? 'unknown';
}
dir = path.dirname(dir);
}
} catch { /* reported as unknown */ }
return 'unknown';
}
// ── Main ─────────────────────────────────────────────────────────────────────
async function main(): Promise<void> {
const options = parseArgs(process.argv.slice(2));
const set = JSON.parse(fs.readFileSync(path.join(BENCH_DIR, 'reference-set.json'), 'utf8')) as ReferenceSet;
const modelsDir = process.env.WAGGLE_MODELS_DIR ?? path.join(os.homedir(), DEFAULT_DATA_DIR_NAME, 'models');
const plan = planEmbedder(options.embedder, modelsDir);
guardNetwork(plan.remoteHost);
const core = await import('@waggle/core') as unknown as Record<string, unknown>;
const wanted = options.engines ?? [...ENGINE_NAMES];
const embedder = await plan.create(core);
let reranker: Reranker | undefined;
if (wanted.some((name) => name.endsWith('-rerank'))) {
// The sidecar caches the reranker under <models>/reranker (packages/server/src/local/index.ts).
reranker = await (core.createInProcessReranker as RerankerFactory)({ cacheDir: path.join(modelsDir, 'reranker') });
}
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-search-bench-'));
const backends = new Map<BackendName, Backend>();
try {
const needed = new Set(wanted.map(backendOf));
if (needed.has('mem0')) {
backends.set('mem0', await createMem0Backend(set, embedder, fs.mkdtempSync(path.join(tempRoot, 'mem0-'))));
}
if (needed.has('cosine')) backends.set('cosine', await createCosineBackend(set, embedder));
const runs = new Map<EngineName, QueryResult[]>();
for (const engine of wanted) {
const backend = backends.get(backendOf(engine))!;
const engineReranker = engine.endsWith('-rerank') ? reranker : undefined;
// Warm up so the first timed query does not pay for lazy initialisation.
await backend.search('warm-up query', ['personal'], options.limit, engineReranker);
const results: QueryResult[] = [];
for (const query of set.queries) {
const started = performance.now();
const hits = await backend.search(query.text, query.scopes, options.limit, engineReranker);
results.push(evaluate(query, hits, performance.now() - started));
}
runs.set(engine, results);
}
const memo = (name: BackendName) => backends.get(name) ? `${f1(backends.get(name)!.ingestMs / 1000)} s` : 'not run';
const setup = [
`Reference set: ${set.memories.length} memories, ${set.queries.length} queries (${set.queries.filter((q) => q.relevant.length === 0).length} without an answer)`,
`Embedder: ${plan.label}`,
`Results per query: ${options.limit}; reranker pool: ${RERANK_POOL}`,
`mem0ai ${packageVersion(path.join(BENCH_DIR, '..'), 'mem0ai')}, @huggingface/transformers ${packageVersion(path.join(BENCH_DIR, '..', '..', 'core'), '@huggingface/transformers')}, Node ${process.version}`,
`Machine: ${os.cpus()[0]?.model ?? 'unknown CPU'}, ${os.cpus().length} cores, ${Math.round(os.totalmem() / 2 ** 30)} GB, ${os.platform()} ${os.release()}`,
`Load time: mem0 ${memo('mem0')}, cosine ${memo('cosine')} (embedding included)`,
`Network: ${requests.local} local requests, ${requests.remote} to the embedding API, ${requests.blocked.length} blocked`,
];
console.log(report(set, runs, setup));
if (options.out) {
fs.writeFileSync(options.out, `${JSON.stringify({ setup, results: Object.fromEntries(runs) }, null, 2)}\n`);
console.error(`Wrote ${options.out}`);
}
if (requests.blocked.length > 0) {
throw new Error(`blocked network requests: ${requests.blocked.join(', ')}`);
}
} finally {
for (const backend of backends.values()) await backend.close();
fs.rmSync(tempRoot, { recursive: true, force: true });
}
}
main().catch((error: unknown) => {
console.error(error instanceof Error ? error.stack ?? error.message : error);
process.exitCode = 1;
});