The bench compared mem0 with MindDB, FrameStore and HybridSearch from the hive-mind substrate, which @waggle/core no longer exports and which left the repository. It now runs mem0, mem0-rerank and cosine; the recorded comparison stays in docs/oss/research/memory-search-quality.md.
501 lines
22 KiB
TypeScript
501 lines
22 KiB
TypeScript
/**
|
|
* Search-quality benchmark for memory (P1.5).
|
|
*
|
|
* Loads the reference set (`reference-set.json`) into the mem0 store and a
|
|
* cosine baseline, runs every query against each, and prints recall@1,
|
|
* recall@5, MRR and latency per engine and per query type. The comparison with
|
|
* the engine mem0 replaced is in docs/oss/research/memory-search-quality.md.
|
|
*
|
|
* Run from the repository root after `npm run build:packages`:
|
|
*
|
|
* npx tsx packages/memory-mem0/bench/search-quality.ts [options]
|
|
*
|
|
* --engines <list> Comma-separated: mem0, mem0-rerank, cosine. Default:
|
|
* every engine. `mem0-rerank` re-orders mem0's results
|
|
* with the in-process cross-encoder. `cosine` is a
|
|
* baseline: plain cosine similarity over the same
|
|
* embeddings, with no keyword signal.
|
|
* --embedder <name> inprocess (default, offline), ollama or openai.
|
|
* --limit <n> Results per query (default 10).
|
|
* --out <file> Also write the per-query results as JSON.
|
|
*
|
|
* Embedders:
|
|
* inprocess Xenova/all-MiniLM-L6-v2 padded to 1024 dimensions, as in the
|
|
* sidecar, loaded from WAGGLE_MODELS_DIR (default ~/.waggle/models).
|
|
* ollama OLLAMA_HOST (default http://localhost:11434) and
|
|
* OLLAMA_EMBED_MODEL (default nomic-embed-text). Ollama must
|
|
* already be running with the model pulled.
|
|
* openai The user's own key: BENCH_EMBEDDING_API_KEY (required),
|
|
* BENCH_EMBEDDING_MODEL (default text-embedding-3-small) and
|
|
* BENCH_EMBEDDING_BASE_URL (default the OpenAI endpoint). The bench
|
|
* never reads OPENAI_API_KEY, so a key that happens to be set in
|
|
* the environment is not used.
|
|
*
|
|
* Every fetch() to another machine is refused, except the endpoint of the
|
|
* openai embedder. mem0 telemetry is switched off before mem0 is loaded.
|
|
*/
|
|
import fs from 'node:fs';
|
|
import { createRequire } from 'node:module';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { performance } from 'node:perf_hooks';
|
|
import { fileURLToPath } from 'node:url';
|
|
import type { MemoryImportance, MemoryKind } from '@waggle/memory-contract';
|
|
|
|
// The adapter sets this too; setting it first keeps telemetry off even if
|
|
// something loads mem0 earlier than the adapter.
|
|
process.env.MEM0_TELEMETRY = 'false';
|
|
|
|
const BENCH_DIR = path.dirname(fileURLToPath(import.meta.url));
|
|
const ENGINE_NAMES = ['mem0', 'mem0-rerank', 'cosine'] as const;
|
|
type EngineName = (typeof ENGINE_NAMES)[number];
|
|
type BackendName = 'mem0' | 'cosine';
|
|
const backendOf = (engine: EngineName): BackendName => engine.replace(/-rerank$/, '') as BackendName;
|
|
/** Candidates a reranker re-orders. */
|
|
const RERANK_POOL = 30;
|
|
|
|
interface MemoryFixture { id: string; scope: string; kind: string; importance: string; text: string }
|
|
interface QueryFixture { id: string; type: string; scopes: string[]; text: string; relevant: string[] }
|
|
interface ReferenceSet { memories: MemoryFixture[]; queries: QueryFixture[] }
|
|
|
|
interface Embedder {
|
|
readonly dimensions: number;
|
|
embed(text: string): Promise<Float32Array>;
|
|
embedBatch(texts: string[]): Promise<Float32Array[]>;
|
|
}
|
|
interface Reranker { scoreBatch(query: string, docs: string[]): Promise<number[]> }
|
|
interface Hit { ref: string; text: string; score: number }
|
|
interface Backend {
|
|
readonly ingestMs: number;
|
|
search(text: string, scopes: string[], limit: number, reranker?: Reranker): Promise<Hit[]>;
|
|
close(): Promise<void>;
|
|
}
|
|
|
|
type RerankerFactory = (config?: { cacheDir?: string }) => Promise<Reranker>;
|
|
|
|
interface Options {
|
|
engines: EngineName[] | null;
|
|
embedder: 'inprocess' | 'ollama' | 'openai';
|
|
limit: number;
|
|
out: string | null;
|
|
}
|
|
|
|
function parseArgs(argv: string[]): Options {
|
|
const options: Options = { engines: null, embedder: 'inprocess', limit: 10, out: null };
|
|
for (let i = 0; i < argv.length; i++) {
|
|
const value = argv[i + 1];
|
|
switch (argv[i]) {
|
|
case '--engines':
|
|
options.engines = value.split(',').map((name) => {
|
|
if (!(ENGINE_NAMES as readonly string[]).includes(name)) throw new Error(`unknown engine: ${name}`);
|
|
return name as EngineName;
|
|
});
|
|
i++;
|
|
break;
|
|
case '--embedder':
|
|
if (value !== 'inprocess' && value !== 'ollama' && value !== 'openai') {
|
|
throw new Error(`unknown embedder: ${value}`);
|
|
}
|
|
options.embedder = value;
|
|
i++;
|
|
break;
|
|
case '--limit':
|
|
options.limit = Number(value);
|
|
if (!Number.isInteger(options.limit) || options.limit < 5) throw new Error('--limit must be an integer >= 5');
|
|
i++;
|
|
break;
|
|
case '--out':
|
|
options.out = path.resolve(value);
|
|
i++;
|
|
break;
|
|
default:
|
|
throw new Error(`unknown argument: ${argv[i]}`);
|
|
}
|
|
}
|
|
return options;
|
|
}
|
|
|
|
// ── Network guard ────────────────────────────────────────────────────────────
|
|
|
|
const LOOPBACK = new Set(['localhost', '127.0.0.1', '::1', '[::1]']);
|
|
const requests = { local: 0, remote: 0, blocked: [] as string[] };
|
|
|
|
function guardNetwork(allowedRemoteHost: string | null): void {
|
|
const realFetch = globalThis.fetch;
|
|
globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => {
|
|
const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url);
|
|
if (LOOPBACK.has(url.hostname)) requests.local++;
|
|
else if (url.hostname === allowedRemoteHost) requests.remote++;
|
|
else {
|
|
requests.blocked.push(url.host);
|
|
throw new Error(`search-quality bench: blocked a request to ${url.host}`);
|
|
}
|
|
return realFetch(input, init);
|
|
}) as typeof fetch;
|
|
}
|
|
|
|
// ── Embedders ────────────────────────────────────────────────────────────────
|
|
|
|
interface EmbedderPlan { label: string; remoteHost: string | null; create(core: Record<string, unknown>): Promise<Embedder> }
|
|
|
|
function planEmbedder(kind: Options['embedder'], modelsDir: string): EmbedderPlan {
|
|
type Factory<C> = (config: C) => Embedder | Promise<Embedder>;
|
|
if (kind === 'inprocess') {
|
|
return {
|
|
label: 'in-process Xenova/all-MiniLM-L6-v2 (384 dims padded to 1024)',
|
|
remoteHost: null,
|
|
create: async (core) => (core.createInProcessEmbedder as Factory<object>)({
|
|
model: 'Xenova/all-MiniLM-L6-v2', cacheDir: modelsDir, targetDimensions: 1024,
|
|
}),
|
|
};
|
|
}
|
|
if (kind === 'ollama') {
|
|
const baseUrl = process.env.OLLAMA_HOST ?? 'http://localhost:11434';
|
|
const model = process.env.OLLAMA_EMBED_MODEL ?? 'nomic-embed-text';
|
|
if (!LOOPBACK.has(new URL(baseUrl).hostname)) throw new Error('OLLAMA_HOST must point at this machine');
|
|
return {
|
|
label: `Ollama ${model} at ${baseUrl} (padded or cut to 1024 dims)`,
|
|
remoteHost: null,
|
|
create: async (core) => {
|
|
const embedder = await (core.createOllamaEmbedder as Factory<object>)({ baseUrl, model, targetDimensions: 1024 });
|
|
try {
|
|
await embedder.embed('probe');
|
|
} catch (error) {
|
|
throw new Error(`Ollama does not answer at ${baseUrl} with model ${model}. Start it and pull the model first; `
|
|
+ `the bench does neither. (${error instanceof Error ? error.message : String(error)})`);
|
|
}
|
|
return embedder;
|
|
},
|
|
};
|
|
}
|
|
const apiKey = process.env.BENCH_EMBEDDING_API_KEY;
|
|
if (!apiKey) throw new Error('--embedder openai needs BENCH_EMBEDDING_API_KEY (OPENAI_API_KEY is deliberately ignored)');
|
|
const model = process.env.BENCH_EMBEDDING_MODEL ?? 'text-embedding-3-small';
|
|
const baseUrl = process.env.BENCH_EMBEDDING_BASE_URL ?? 'https://api.openai.com/v1/embeddings';
|
|
return {
|
|
label: `OpenAI-compatible ${model} at ${new URL(baseUrl).host} (padded or cut to 1024 dims)`,
|
|
remoteHost: new URL(baseUrl).hostname,
|
|
create: async (core) => (core.createApiEmbedder as Factory<object>)({
|
|
provider: 'openai', apiKey, model, baseUrl, targetDimensions: 1024,
|
|
}),
|
|
};
|
|
}
|
|
|
|
// ── Backends ─────────────────────────────────────────────────────────────────
|
|
|
|
async function createMem0Backend(set: ReferenceSet, embedder: Embedder, dir: string): Promise<Backend> {
|
|
const { createMem0MemoryStore } = await import('../src/index.js');
|
|
const { scopeFromKey } = await import('@waggle/memory-contract');
|
|
const toScope = (key: string) => {
|
|
const scope = scopeFromKey(key);
|
|
if (!scope) throw new Error(`bad scope key: ${key}`);
|
|
return scope;
|
|
};
|
|
const store = await createMem0MemoryStore({ directory: dir, embedder });
|
|
const refs = new Map<string, string>();
|
|
const started = performance.now();
|
|
for (const memory of set.memories) {
|
|
const [change] = await store.remember({
|
|
scope: toScope(memory.scope),
|
|
text: memory.text,
|
|
kind: memory.kind as MemoryKind,
|
|
importance: memory.importance as MemoryImportance,
|
|
source: 'user_stated',
|
|
origin: 'bench',
|
|
});
|
|
refs.set(change.record.id, memory.id);
|
|
}
|
|
const ingestMs = performance.now() - started;
|
|
|
|
return {
|
|
ingestMs,
|
|
async search(text, scopes, limit, reranker) {
|
|
const results = await store.search(text, { scopes: scopes.map(toScope), limit: reranker ? RERANK_POOL : limit });
|
|
const hits = results.map((r) => ({ ref: refs.get(r.id)!, text: r.text, score: r.score }));
|
|
return rerankTop(text, hits, limit, reranker);
|
|
},
|
|
close: () => store.close(),
|
|
};
|
|
}
|
|
|
|
/** Baseline: rank every memory of the searched scopes by cosine similarity alone. */
|
|
async function createCosineBackend(set: ReferenceSet, embedder: Embedder): Promise<Backend> {
|
|
const started = performance.now();
|
|
const vectors = await embedder.embedBatch(set.memories.map((m) => m.text));
|
|
const ingestMs = performance.now() - started;
|
|
const cosine = (a: Float32Array, b: Float32Array) => {
|
|
let dot = 0, na = 0, nb = 0;
|
|
for (let i = 0; i < a.length; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
|
|
return dot / (Math.sqrt(na) * Math.sqrt(nb) || 1);
|
|
};
|
|
return {
|
|
ingestMs,
|
|
async search(text, scopes, limit, reranker) {
|
|
const query = await embedder.embed(text);
|
|
const hits = set.memories
|
|
.map((m, i) => ({ memory: m, score: cosine(query, vectors[i]) }))
|
|
.filter(({ memory }) => scopes.includes(memory.scope))
|
|
.sort((a, b) => b.score - a.score)
|
|
.slice(0, reranker ? RERANK_POOL : limit)
|
|
.map(({ memory, score }) => ({ ref: memory.id, text: memory.text, score }));
|
|
return rerankTop(text, hits, limit, reranker);
|
|
},
|
|
close: async () => {},
|
|
};
|
|
}
|
|
|
|
async function rerankTop(text: string, hits: Hit[], limit: number, reranker?: Reranker): Promise<Hit[]> {
|
|
if (!reranker || hits.length === 0) return hits.slice(0, limit);
|
|
const scores = await reranker.scoreBatch(text, hits.map((h) => h.text));
|
|
return hits
|
|
.map((hit, i) => ({ ...hit, score: scores[i] }))
|
|
.sort((a, b) => b.score - a.score)
|
|
.slice(0, limit);
|
|
}
|
|
|
|
// ── Metrics ──────────────────────────────────────────────────────────────────
|
|
|
|
interface QueryResult {
|
|
query: string;
|
|
type: string;
|
|
scopes: string[];
|
|
relevant: string[];
|
|
ranked: Array<{ ref: string; score: number }>;
|
|
/** 1-based rank of the first relevant memory within the results, or null. */
|
|
firstRank: number | null;
|
|
recallAt1: number | null;
|
|
recallAt5: number | null;
|
|
reciprocalRank: number | null;
|
|
latencyMs: number;
|
|
}
|
|
|
|
/** Share of the relevant memories found in the top k, out of the most that fit (min(k, relevant)). */
|
|
function recallAt(ranked: string[], relevant: Set<string>, k: number): number {
|
|
const found = ranked.slice(0, k).filter((ref) => relevant.has(ref)).length;
|
|
return found / Math.min(k, relevant.size);
|
|
}
|
|
|
|
function evaluate(query: QueryFixture, hits: Hit[], latencyMs: number): QueryResult {
|
|
const relevant = new Set(query.relevant);
|
|
const ranked = hits.map((h) => h.ref);
|
|
const index = ranked.findIndex((ref) => relevant.has(ref));
|
|
const firstRank = index >= 0 ? index + 1 : null;
|
|
const answerable = relevant.size > 0;
|
|
return {
|
|
query: query.id,
|
|
type: query.type,
|
|
scopes: query.scopes,
|
|
relevant: query.relevant,
|
|
ranked: hits.map((h) => ({ ref: h.ref, score: Number(h.score.toFixed(4)) })),
|
|
firstRank,
|
|
recallAt1: answerable ? recallAt(ranked, relevant, 1) : null,
|
|
recallAt5: answerable ? recallAt(ranked, relevant, 5) : null,
|
|
reciprocalRank: answerable ? (firstRank ? 1 / firstRank : 0) : null,
|
|
latencyMs,
|
|
};
|
|
}
|
|
|
|
interface Summary { n: number; recallAt1: number; recallAt5: number; mrr: number }
|
|
|
|
function summarize(results: QueryResult[]): Summary {
|
|
const answerable = results.filter((r) => r.relevant.length > 0);
|
|
const mean = (pick: (r: QueryResult) => number | null) =>
|
|
answerable.reduce((sum, r) => sum + (pick(r) ?? 0), 0) / (answerable.length || 1);
|
|
return {
|
|
n: answerable.length,
|
|
recallAt1: mean((r) => r.recallAt1),
|
|
recallAt5: mean((r) => r.recallAt5),
|
|
mrr: mean((r) => r.reciprocalRank),
|
|
};
|
|
}
|
|
|
|
function percentile(values: number[], p: number): number {
|
|
const sorted = [...values].sort((a, b) => a - b);
|
|
return sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))] ?? 0;
|
|
}
|
|
|
|
/**
|
|
* How well the top score separates a correct first hit from a query memory
|
|
* cannot answer: the chance that a correct top-1 outscores a no-answer top-1
|
|
* (ties count half). 1.0 means a score threshold could tell them apart.
|
|
*/
|
|
function abstainSeparation(results: QueryResult[]): number | null {
|
|
const correct = results.filter((r) => r.firstRank === 1).map((r) => r.ranked[0].score);
|
|
const empty = results.filter((r) => r.relevant.length === 0).map((r) => r.ranked[0]?.score ?? -Infinity);
|
|
if (correct.length === 0 || empty.length === 0) return null;
|
|
let wins = 0;
|
|
for (const c of correct) for (const e of empty) wins += c > e ? 1 : c === e ? 0.5 : 0;
|
|
return wins / (correct.length * empty.length);
|
|
}
|
|
|
|
// ── Report ───────────────────────────────────────────────────────────────────
|
|
|
|
const f2 = (value: number) => value.toFixed(2);
|
|
const f1 = (value: number) => value.toFixed(1);
|
|
|
|
function table(header: string[], rows: string[][]): string {
|
|
return [
|
|
`| ${header.join(' | ')} |`,
|
|
`|${header.map(() => '---').join('|')}|`,
|
|
...rows.map((row) => `| ${row.join(' | ')} |`),
|
|
].join('\n');
|
|
}
|
|
|
|
function report(set: ReferenceSet, runs: Map<EngineName, QueryResult[]>, setup: string[]): string {
|
|
const engines = [...runs.keys()];
|
|
const out: string[] = ['# Memory search quality', '', ...setup.map((line) => `- ${line}`), ''];
|
|
const cell = (s: Summary) => `${f2(s.recallAt1)} / ${f2(s.recallAt5)} / ${f2(s.mrr)}`;
|
|
|
|
out.push('## Overall (answerable queries)', '');
|
|
out.push(table(
|
|
['engine', 'n', 'recall@1', 'recall@5', 'MRR', 'median ms', 'p95 ms', 'abstain separation'],
|
|
engines.map((engine) => {
|
|
const results = runs.get(engine)!;
|
|
const s = summarize(results);
|
|
const latencies = results.map((r) => r.latencyMs);
|
|
const separation = abstainSeparation(results);
|
|
return [engine, String(s.n), f2(s.recallAt1), f2(s.recallAt5), f2(s.mrr),
|
|
f1(percentile(latencies, 0.5)), f1(percentile(latencies, 0.95)), separation === null ? '-' : f2(separation)];
|
|
}),
|
|
), '');
|
|
|
|
const types = [...new Set(set.queries.map((q) => q.type))].filter((type) => type !== 'no-answer');
|
|
out.push('## By query type (recall@1 / recall@5 / MRR)', '');
|
|
out.push(table(
|
|
['type', 'n', ...engines],
|
|
types.map((type) => [
|
|
type,
|
|
String(set.queries.filter((q) => q.type === type).length),
|
|
...engines.map((engine) => cell(summarize(runs.get(engine)!.filter((r) => r.type === type)))),
|
|
]),
|
|
), '');
|
|
|
|
out.push('## By number of scopes searched (recall@1 / recall@5 / MRR)', '');
|
|
out.push(table(
|
|
['scopes', 'n', ...engines],
|
|
[['one', (n: number) => n === 1], ['two or more', (n: number) => n > 1]].map(([label, test]) => {
|
|
const pick = (r: QueryResult) => (test as (n: number) => boolean)(r.scopes.length) && r.relevant.length > 0;
|
|
return [
|
|
label as string,
|
|
String(runs.get(engines[0])!.filter(pick).length),
|
|
...engines.map((engine) => cell(summarize(runs.get(engine)!.filter(pick)))),
|
|
];
|
|
}),
|
|
), '');
|
|
|
|
out.push('## Misses', '');
|
|
out.push('Answerable queries where at least one engine did not rank a relevant memory first. '
|
|
+ 'Each engine cell: rank of the first relevant memory ("-" if not in the results), recall@5, and the top result.', '');
|
|
const missRows: string[][] = [];
|
|
for (const query of set.queries.filter((q) => q.relevant.length > 0)) {
|
|
const per = engines.map((engine) => runs.get(engine)!.find((r) => r.query === query.id)!);
|
|
if (per.every((r) => r.firstRank === 1 && r.recallAt5 === 1)) continue;
|
|
missRows.push([
|
|
query.id, query.type, query.text, query.relevant.join(' '),
|
|
...per.map((r) => `${r.firstRank ?? '-'} · ${f2(r.recallAt5 ?? 0)} · ${r.ranked[0]?.ref ?? '-'}`),
|
|
]);
|
|
}
|
|
out.push(table(['query', 'type', 'text', 'relevant', ...engines], missRows), '');
|
|
|
|
out.push('## No-answer queries (top result and its score)', '');
|
|
out.push(table(
|
|
['query', 'text', ...engines],
|
|
set.queries.filter((q) => q.relevant.length === 0).map((query) => [
|
|
query.id, query.text,
|
|
...engines.map((engine) => {
|
|
const top = runs.get(engine)!.find((r) => r.query === query.id)!.ranked[0];
|
|
return top ? `${top.ref} (${top.score.toFixed(3)})` : '-';
|
|
}),
|
|
]),
|
|
), '');
|
|
return out.join('\n');
|
|
}
|
|
|
|
function packageVersion(fromDir: string, name: string): string {
|
|
try {
|
|
let dir = path.dirname(createRequire(path.join(fromDir, 'package.json')).resolve(name));
|
|
while (dir !== path.dirname(dir)) {
|
|
const file = path.join(dir, 'package.json');
|
|
if (fs.existsSync(file)) {
|
|
const pkg = JSON.parse(fs.readFileSync(file, 'utf8')) as { name?: string; version?: string };
|
|
if (pkg.name === name) return pkg.version ?? 'unknown';
|
|
}
|
|
dir = path.dirname(dir);
|
|
}
|
|
} catch { /* reported as unknown */ }
|
|
return 'unknown';
|
|
}
|
|
|
|
// ── Main ─────────────────────────────────────────────────────────────────────
|
|
|
|
async function main(): Promise<void> {
|
|
const options = parseArgs(process.argv.slice(2));
|
|
const set = JSON.parse(fs.readFileSync(path.join(BENCH_DIR, 'reference-set.json'), 'utf8')) as ReferenceSet;
|
|
const modelsDir = process.env.WAGGLE_MODELS_DIR ?? path.join(os.homedir(), '.waggle', 'models');
|
|
const plan = planEmbedder(options.embedder, modelsDir);
|
|
guardNetwork(plan.remoteHost);
|
|
|
|
const core = await import('@waggle/core') as unknown as Record<string, unknown>;
|
|
const wanted = options.engines ?? [...ENGINE_NAMES];
|
|
|
|
const embedder = await plan.create(core);
|
|
let reranker: Reranker | undefined;
|
|
if (wanted.some((name) => name.endsWith('-rerank'))) {
|
|
// The sidecar caches the reranker under <models>/reranker (packages/server/src/local/index.ts).
|
|
reranker = await (core.createInProcessReranker as RerankerFactory)({ cacheDir: path.join(modelsDir, 'reranker') });
|
|
}
|
|
|
|
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-search-bench-'));
|
|
const backends = new Map<BackendName, Backend>();
|
|
try {
|
|
const needed = new Set(wanted.map(backendOf));
|
|
if (needed.has('mem0')) {
|
|
backends.set('mem0', await createMem0Backend(set, embedder, fs.mkdtempSync(path.join(tempRoot, 'mem0-'))));
|
|
}
|
|
if (needed.has('cosine')) backends.set('cosine', await createCosineBackend(set, embedder));
|
|
|
|
const runs = new Map<EngineName, QueryResult[]>();
|
|
for (const engine of wanted) {
|
|
const backend = backends.get(backendOf(engine))!;
|
|
const engineReranker = engine.endsWith('-rerank') ? reranker : undefined;
|
|
// Warm up so the first timed query does not pay for lazy initialisation.
|
|
await backend.search('warm-up query', ['personal'], options.limit, engineReranker);
|
|
const results: QueryResult[] = [];
|
|
for (const query of set.queries) {
|
|
const started = performance.now();
|
|
const hits = await backend.search(query.text, query.scopes, options.limit, engineReranker);
|
|
results.push(evaluate(query, hits, performance.now() - started));
|
|
}
|
|
runs.set(engine, results);
|
|
}
|
|
|
|
const memo = (name: BackendName) => backends.get(name) ? `${f1(backends.get(name)!.ingestMs / 1000)} s` : 'not run';
|
|
const setup = [
|
|
`Reference set: ${set.memories.length} memories, ${set.queries.length} queries (${set.queries.filter((q) => q.relevant.length === 0).length} without an answer)`,
|
|
`Embedder: ${plan.label}`,
|
|
`Results per query: ${options.limit}; reranker pool: ${RERANK_POOL}`,
|
|
`mem0ai ${packageVersion(path.join(BENCH_DIR, '..'), 'mem0ai')}, @huggingface/transformers ${packageVersion(path.join(BENCH_DIR, '..', '..', 'core'), '@huggingface/transformers')}, Node ${process.version}`,
|
|
`Machine: ${os.cpus()[0]?.model ?? 'unknown CPU'}, ${os.cpus().length} cores, ${Math.round(os.totalmem() / 2 ** 30)} GB, ${os.platform()} ${os.release()}`,
|
|
`Load time: mem0 ${memo('mem0')}, cosine ${memo('cosine')} (embedding included)`,
|
|
`Network: ${requests.local} local requests, ${requests.remote} to the embedding API, ${requests.blocked.length} blocked`,
|
|
];
|
|
console.log(report(set, runs, setup));
|
|
|
|
if (options.out) {
|
|
fs.writeFileSync(options.out, `${JSON.stringify({ setup, results: Object.fromEntries(runs) }, null, 2)}\n`);
|
|
console.error(`Wrote ${options.out}`);
|
|
}
|
|
if (requests.blocked.length > 0) {
|
|
throw new Error(`blocked network requests: ${requests.blocked.join(', ')}`);
|
|
}
|
|
} finally {
|
|
for (const backend of backends.values()) await backend.close();
|
|
fs.rmSync(tempRoot, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
main().catch((error: unknown) => {
|
|
console.error(error instanceof Error ? error.stack ?? error.message : error);
|
|
process.exitCode = 1;
|
|
});
|