refactor(memory-mem0): drop the old engine from the search-quality bench (P2.2)

The bench compared mem0 with MindDB, FrameStore and HybridSearch from
the hive-mind substrate, which @waggle/core no longer exports and which
left the repository. It now runs mem0, mem0-rerank and cosine; the
recorded comparison stays in docs/oss/research/memory-search-quality.md.
This commit is contained in:
shumeiko.it
2026-11-01 00:42:47 +01:00
parent 4c1b404339
commit 613b23be17
2 changed files with 21 additions and 88 deletions

View File

@@ -315,13 +315,14 @@ What this bench does not cover:
## How to re-run ## How to re-run
From the repository root, after `npm ci` and `npm run build:packages`: From the repository root, after `npm ci` and `npm run build:packages`. The old engine left
the repository in P2.2, so the bench runs mem0, mem0-rerank and cosine:
```bash ```bash
# All engines this checkout has (old, old-rerank, mem0, mem0-rerank, cosine) # All engines (mem0, mem0-rerank, cosine)
npx tsx packages/memory-mem0/bench/search-quality.ts npx tsx packages/memory-mem0/bench/search-quality.ts
# After the old engine is removed # Without the reranker model
npx tsx packages/memory-mem0/bench/search-quality.ts --engines mem0,cosine npx tsx packages/memory-mem0/bench/search-quality.ts --engines mem0,cosine
# Per-query results as JSON # Per-query results as JSON

View File

@@ -1,22 +1,20 @@
/** /**
* Search-quality benchmark for the memory switch (P1.5). * Search-quality benchmark for memory (P1.5).
* *
* Loads the reference set (`reference-set.json`) into the old engine (MindDB + * Loads the reference set (`reference-set.json`) into the mem0 store and a
* FrameStore + HybridSearch, re-exported by @waggle/core from the hive-mind * cosine baseline, runs every query against each, and prints recall@1,
* substrate) and into the mem0 store, runs every query against each, and prints * recall@5, MRR and latency per engine and per query type. The comparison with
* recall@1, recall@5, MRR and latency per engine and per query type. The * the engine mem0 replaced is in docs/oss/research/memory-search-quality.md.
* findings are in docs/oss/research/memory-search-quality.md.
* *
* Run from the repository root after `npm run build:packages`: * Run from the repository root after `npm run build:packages`:
* *
* npx tsx packages/memory-mem0/bench/search-quality.ts [options] * npx tsx packages/memory-mem0/bench/search-quality.ts [options]
* *
* --engines <list> Comma-separated: old, old-rerank, mem0, mem0-rerank, cosine. * --engines <list> Comma-separated: mem0, mem0-rerank, cosine. Default:
* Default: every engine this checkout still has. Once the * every engine. `mem0-rerank` re-orders mem0's results
* hive-mind substrate is removed, `mem0` and `cosine` are * with the in-process cross-encoder. `cosine` is a
* left (the `-rerank` variants use its cross-encoder). * baseline: plain cosine similarity over the same
* `cosine` is a baseline: plain cosine similarity over the * embeddings, with no keyword signal.
* same embeddings, with no keyword signal.
* --embedder <name> inprocess (default, offline), ollama or openai. * --embedder <name> inprocess (default, offline), ollama or openai.
* --limit <n> Results per query (default 10). * --limit <n> Results per query (default 10).
* --out <file> Also write the per-query results as JSON. * --out <file> Also write the per-query results as JSON.
@@ -49,11 +47,11 @@ import type { MemoryImportance, MemoryKind } from '@waggle/memory-contract';
process.env.MEM0_TELEMETRY = 'false'; process.env.MEM0_TELEMETRY = 'false';
const BENCH_DIR = path.dirname(fileURLToPath(import.meta.url)); const BENCH_DIR = path.dirname(fileURLToPath(import.meta.url));
const ENGINE_NAMES = ['old', 'old-rerank', 'mem0', 'mem0-rerank', 'cosine'] as const; const ENGINE_NAMES = ['mem0', 'mem0-rerank', 'cosine'] as const;
type EngineName = (typeof ENGINE_NAMES)[number]; type EngineName = (typeof ENGINE_NAMES)[number];
type BackendName = 'old' | 'mem0' | 'cosine'; type BackendName = 'mem0' | 'cosine';
const backendOf = (engine: EngineName): BackendName => engine.replace(/-rerank$/, '') as BackendName; const backendOf = (engine: EngineName): BackendName => engine.replace(/-rerank$/, '') as BackendName;
/** Candidates a reranker re-orders; HybridSearch's default pool size. */ /** Candidates a reranker re-orders. */
const RERANK_POOL = 30; const RERANK_POOL = 30;
interface MemoryFixture { id: string; scope: string; kind: string; importance: string; text: string } interface MemoryFixture { id: string; scope: string; kind: string; importance: string; text: string }
@@ -73,20 +71,7 @@ interface Backend {
close(): Promise<void>; close(): Promise<void>;
} }
/** The parts of the hive-mind substrate the old engine needs; absent once it is removed. */ type RerankerFactory = (config?: { cacheDir?: string }) => Promise<Reranker>;
interface LegacyCore {
MindDB?: new (file: string) => { close(): void };
FrameStore?: new (db: unknown) => {
createIFrame(gopId: string, content: string, importance?: string, source?: string): { id: number };
};
SessionStore?: new (db: unknown) => { ensure(gopId: string): unknown };
HybridSearch?: new (db: unknown, embedder: Embedder) => {
indexFrame(id: number, content: string): Promise<void>;
search(query: string, options: { limit?: number; profile?: string; reranker?: Reranker }):
Promise<Array<{ frame: { id: number; content: string }; finalScore: number }>>;
};
createInProcessReranker?: (config?: { cacheDir?: string }) => Promise<Reranker>;
}
interface Options { interface Options {
engines: EngineName[] | null; engines: EngineName[] | null;
@@ -198,47 +183,6 @@ function planEmbedder(kind: Options['embedder'], modelsDir: string): EmbedderPla
// ── Backends ───────────────────────────────────────────────────────────────── // ── Backends ─────────────────────────────────────────────────────────────────
async function createOldBackend(core: LegacyCore, set: ReferenceSet, embedder: Embedder, dir: string): Promise<Backend> {
const { MindDB, FrameStore, SessionStore, HybridSearch } = core;
if (!MindDB || !FrameStore || !SessionStore || !HybridSearch) throw new Error('the hive-mind substrate is gone');
// Like the sidecar: one .mind file for personal memory and one per workspace.
const minds = new Map<string, { db: { close(): void }; search: InstanceType<NonNullable<LegacyCore['HybridSearch']>>; refs: Map<number, string> }>();
const started = performance.now();
for (const memory of set.memories) {
let mind = minds.get(memory.scope);
if (!mind) {
const db = new MindDB(path.join(dir, `${memory.scope.replace(/\W+/g, '-')}.mind`));
new SessionStore(db).ensure('bench');
mind = { db, search: new HybridSearch(db, embedder), refs: new Map() };
minds.set(memory.scope, mind);
}
const frame = new FrameStore(mind.db).createIFrame('bench', memory.text, memory.importance, 'user_stated');
await mind.search.indexFrame(frame.id, memory.text);
mind.refs.set(frame.id, memory.id);
}
const ingestMs = performance.now() - started;
return {
ingestMs,
async search(text, scopes, limit, reranker) {
const hits: Hit[] = [];
for (const scope of scopes) {
const mind = minds.get(scope);
if (!mind) continue;
// Recall uses the 'balanced' profile (packages/agent/src/orchestrator.ts).
for (const result of await mind.search.search(text, { limit, profile: 'balanced', reranker })) {
hits.push({ ref: mind.refs.get(result.frame.id)!, text: result.frame.content, score: result.finalScore });
}
}
// One ranked list across minds, ordered by each mind's own final score.
return hits.sort((a, b) => b.score - a.score).slice(0, limit);
},
async close() {
for (const mind of minds.values()) mind.db.close();
},
};
}
async function createMem0Backend(set: ReferenceSet, embedder: Embedder, dir: string): Promise<Backend> { async function createMem0Backend(set: ReferenceSet, embedder: Embedder, dir: string): Promise<Backend> {
const { createMem0MemoryStore } = await import('../src/index.js'); const { createMem0MemoryStore } = await import('../src/index.js');
const { scopeFromKey } = await import('@waggle/memory-contract'); const { scopeFromKey } = await import('@waggle/memory-contract');
@@ -492,31 +436,19 @@ async function main(): Promise<void> {
guardNetwork(plan.remoteHost); guardNetwork(plan.remoteHost);
const core = await import('@waggle/core') as unknown as Record<string, unknown>; const core = await import('@waggle/core') as unknown as Record<string, unknown>;
const legacy = core as LegacyCore; const wanted = options.engines ?? [...ENGINE_NAMES];
const hasOldEngine = Boolean(legacy.MindDB && legacy.FrameStore && legacy.SessionStore && legacy.HybridSearch);
const hasReranker = Boolean(legacy.createInProcessReranker);
const available = (name: EngineName) =>
(hasOldEngine || backendOf(name) !== 'old') && (hasReranker || !name.endsWith('-rerank'));
const wanted = options.engines ?? ENGINE_NAMES.filter(available);
const missing = wanted.filter((name) => !available(name));
if (missing.length > 0) {
throw new Error(`not in this checkout (the hive-mind substrate is gone): ${missing.join(', ')}; try --engines mem0,cosine`);
}
const embedder = await plan.create(core); const embedder = await plan.create(core);
let reranker: Reranker | undefined; let reranker: Reranker | undefined;
if (wanted.some((name) => name.endsWith('-rerank'))) { if (wanted.some((name) => name.endsWith('-rerank'))) {
// The sidecar caches the reranker under <models>/reranker (packages/server/src/local/index.ts). // The sidecar caches the reranker under <models>/reranker (packages/server/src/local/index.ts).
reranker = await legacy.createInProcessReranker!({ cacheDir: path.join(modelsDir, 'reranker') }); reranker = await (core.createInProcessReranker as RerankerFactory)({ cacheDir: path.join(modelsDir, 'reranker') });
} }
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-search-bench-')); const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-search-bench-'));
const backends = new Map<BackendName, Backend>(); const backends = new Map<BackendName, Backend>();
try { try {
const needed = new Set(wanted.map(backendOf)); const needed = new Set(wanted.map(backendOf));
if (needed.has('old')) {
backends.set('old', await createOldBackend(legacy, set, embedder, fs.mkdtempSync(path.join(tempRoot, 'old-'))));
}
if (needed.has('mem0')) { if (needed.has('mem0')) {
backends.set('mem0', await createMem0Backend(set, embedder, fs.mkdtempSync(path.join(tempRoot, 'mem0-')))); backends.set('mem0', await createMem0Backend(set, embedder, fs.mkdtempSync(path.join(tempRoot, 'mem0-'))));
} }
@@ -544,7 +476,7 @@ async function main(): Promise<void> {
`Results per query: ${options.limit}; reranker pool: ${RERANK_POOL}`, `Results per query: ${options.limit}; reranker pool: ${RERANK_POOL}`,
`mem0ai ${packageVersion(path.join(BENCH_DIR, '..'), 'mem0ai')}, @huggingface/transformers ${packageVersion(path.join(BENCH_DIR, '..', '..', 'core'), '@huggingface/transformers')}, Node ${process.version}`, `mem0ai ${packageVersion(path.join(BENCH_DIR, '..'), 'mem0ai')}, @huggingface/transformers ${packageVersion(path.join(BENCH_DIR, '..', '..', 'core'), '@huggingface/transformers')}, Node ${process.version}`,
`Machine: ${os.cpus()[0]?.model ?? 'unknown CPU'}, ${os.cpus().length} cores, ${Math.round(os.totalmem() / 2 ** 30)} GB, ${os.platform()} ${os.release()}`, `Machine: ${os.cpus()[0]?.model ?? 'unknown CPU'}, ${os.cpus().length} cores, ${Math.round(os.totalmem() / 2 ** 30)} GB, ${os.platform()} ${os.release()}`,
`Load time: old engine ${memo('old')}, mem0 ${memo('mem0')}, cosine ${memo('cosine')} (embedding included)`, `Load time: mem0 ${memo('mem0')}, cosine ${memo('cosine')} (embedding included)`,
`Network: ${requests.local} local requests, ${requests.remote} to the embedding API, ${requests.blocked.length} blocked`, `Network: ${requests.local} local requests, ${requests.remote} to the embedding API, ${requests.blocked.length} blocked`,
]; ];
console.log(report(set, runs, setup)); console.log(report(set, runs, setup));