diff --git a/docs/oss/research/memory-search-quality.md b/docs/oss/research/memory-search-quality.md index b3811c8..f4883da 100644 --- a/docs/oss/research/memory-search-quality.md +++ b/docs/oss/research/memory-search-quality.md @@ -315,13 +315,14 @@ What this bench does not cover: ## How to re-run -From the repository root, after `npm ci` and `npm run build:packages`: +From the repository root, after `npm ci` and `npm run build:packages`. The old engine left +the repository in P2.2, so the bench runs mem0, mem0-rerank and cosine: ```bash -# All engines this checkout has (old, old-rerank, mem0, mem0-rerank, cosine) +# All engines (mem0, mem0-rerank, cosine) npx tsx packages/memory-mem0/bench/search-quality.ts -# After the old engine is removed +# Without the reranker model npx tsx packages/memory-mem0/bench/search-quality.ts --engines mem0,cosine # Per-query results as JSON diff --git a/packages/memory-mem0/bench/search-quality.ts b/packages/memory-mem0/bench/search-quality.ts index 8df5376..1175e5f 100644 --- a/packages/memory-mem0/bench/search-quality.ts +++ b/packages/memory-mem0/bench/search-quality.ts @@ -1,22 +1,20 @@ /** - * Search-quality benchmark for the memory switch (P1.5). + * Search-quality benchmark for memory (P1.5). * - * Loads the reference set (`reference-set.json`) into the old engine (MindDB + - * FrameStore + HybridSearch, re-exported by @waggle/core from the hive-mind - * substrate) and into the mem0 store, runs every query against each, and prints - * recall@1, recall@5, MRR and latency per engine and per query type. The - * findings are in docs/oss/research/memory-search-quality.md. + * Loads the reference set (`reference-set.json`) into the mem0 store and a + * cosine baseline, runs every query against each, and prints recall@1, + * recall@5, MRR and latency per engine and per query type. The comparison with + * the engine mem0 replaced is in docs/oss/research/memory-search-quality.md. * * Run from the repository root after `npm run build:packages`: * * npx tsx packages/memory-mem0/bench/search-quality.ts [options] * - * --engines Comma-separated: old, old-rerank, mem0, mem0-rerank, cosine. - * Default: every engine this checkout still has. Once the - * hive-mind substrate is removed, `mem0` and `cosine` are - * left (the `-rerank` variants use its cross-encoder). - * `cosine` is a baseline: plain cosine similarity over the - * same embeddings, with no keyword signal. + * --engines Comma-separated: mem0, mem0-rerank, cosine. Default: + * every engine. `mem0-rerank` re-orders mem0's results + * with the in-process cross-encoder. `cosine` is a + * baseline: plain cosine similarity over the same + * embeddings, with no keyword signal. * --embedder inprocess (default, offline), ollama or openai. * --limit Results per query (default 10). * --out Also write the per-query results as JSON. @@ -49,11 +47,11 @@ import type { MemoryImportance, MemoryKind } from '@waggle/memory-contract'; process.env.MEM0_TELEMETRY = 'false'; const BENCH_DIR = path.dirname(fileURLToPath(import.meta.url)); -const ENGINE_NAMES = ['old', 'old-rerank', 'mem0', 'mem0-rerank', 'cosine'] as const; +const ENGINE_NAMES = ['mem0', 'mem0-rerank', 'cosine'] as const; type EngineName = (typeof ENGINE_NAMES)[number]; -type BackendName = 'old' | 'mem0' | 'cosine'; +type BackendName = 'mem0' | 'cosine'; const backendOf = (engine: EngineName): BackendName => engine.replace(/-rerank$/, '') as BackendName; -/** Candidates a reranker re-orders; HybridSearch's default pool size. */ +/** Candidates a reranker re-orders. */ const RERANK_POOL = 30; interface MemoryFixture { id: string; scope: string; kind: string; importance: string; text: string } @@ -73,20 +71,7 @@ interface Backend { close(): Promise; } -/** The parts of the hive-mind substrate the old engine needs; absent once it is removed. */ -interface LegacyCore { - MindDB?: new (file: string) => { close(): void }; - FrameStore?: new (db: unknown) => { - createIFrame(gopId: string, content: string, importance?: string, source?: string): { id: number }; - }; - SessionStore?: new (db: unknown) => { ensure(gopId: string): unknown }; - HybridSearch?: new (db: unknown, embedder: Embedder) => { - indexFrame(id: number, content: string): Promise; - search(query: string, options: { limit?: number; profile?: string; reranker?: Reranker }): - Promise>; - }; - createInProcessReranker?: (config?: { cacheDir?: string }) => Promise; -} +type RerankerFactory = (config?: { cacheDir?: string }) => Promise; interface Options { engines: EngineName[] | null; @@ -198,47 +183,6 @@ function planEmbedder(kind: Options['embedder'], modelsDir: string): EmbedderPla // ── Backends ───────────────────────────────────────────────────────────────── -async function createOldBackend(core: LegacyCore, set: ReferenceSet, embedder: Embedder, dir: string): Promise { - const { MindDB, FrameStore, SessionStore, HybridSearch } = core; - if (!MindDB || !FrameStore || !SessionStore || !HybridSearch) throw new Error('the hive-mind substrate is gone'); - // Like the sidecar: one .mind file for personal memory and one per workspace. - const minds = new Map>; refs: Map }>(); - const started = performance.now(); - for (const memory of set.memories) { - let mind = minds.get(memory.scope); - if (!mind) { - const db = new MindDB(path.join(dir, `${memory.scope.replace(/\W+/g, '-')}.mind`)); - new SessionStore(db).ensure('bench'); - mind = { db, search: new HybridSearch(db, embedder), refs: new Map() }; - minds.set(memory.scope, mind); - } - const frame = new FrameStore(mind.db).createIFrame('bench', memory.text, memory.importance, 'user_stated'); - await mind.search.indexFrame(frame.id, memory.text); - mind.refs.set(frame.id, memory.id); - } - const ingestMs = performance.now() - started; - - return { - ingestMs, - async search(text, scopes, limit, reranker) { - const hits: Hit[] = []; - for (const scope of scopes) { - const mind = minds.get(scope); - if (!mind) continue; - // Recall uses the 'balanced' profile (packages/agent/src/orchestrator.ts). - for (const result of await mind.search.search(text, { limit, profile: 'balanced', reranker })) { - hits.push({ ref: mind.refs.get(result.frame.id)!, text: result.frame.content, score: result.finalScore }); - } - } - // One ranked list across minds, ordered by each mind's own final score. - return hits.sort((a, b) => b.score - a.score).slice(0, limit); - }, - async close() { - for (const mind of minds.values()) mind.db.close(); - }, - }; -} - async function createMem0Backend(set: ReferenceSet, embedder: Embedder, dir: string): Promise { const { createMem0MemoryStore } = await import('../src/index.js'); const { scopeFromKey } = await import('@waggle/memory-contract'); @@ -492,31 +436,19 @@ async function main(): Promise { guardNetwork(plan.remoteHost); const core = await import('@waggle/core') as unknown as Record; - const legacy = core as LegacyCore; - const hasOldEngine = Boolean(legacy.MindDB && legacy.FrameStore && legacy.SessionStore && legacy.HybridSearch); - const hasReranker = Boolean(legacy.createInProcessReranker); - const available = (name: EngineName) => - (hasOldEngine || backendOf(name) !== 'old') && (hasReranker || !name.endsWith('-rerank')); - const wanted = options.engines ?? ENGINE_NAMES.filter(available); - const missing = wanted.filter((name) => !available(name)); - if (missing.length > 0) { - throw new Error(`not in this checkout (the hive-mind substrate is gone): ${missing.join(', ')}; try --engines mem0,cosine`); - } + const wanted = options.engines ?? [...ENGINE_NAMES]; const embedder = await plan.create(core); let reranker: Reranker | undefined; if (wanted.some((name) => name.endsWith('-rerank'))) { // The sidecar caches the reranker under /reranker (packages/server/src/local/index.ts). - reranker = await legacy.createInProcessReranker!({ cacheDir: path.join(modelsDir, 'reranker') }); + reranker = await (core.createInProcessReranker as RerankerFactory)({ cacheDir: path.join(modelsDir, 'reranker') }); } const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-search-bench-')); const backends = new Map(); try { const needed = new Set(wanted.map(backendOf)); - if (needed.has('old')) { - backends.set('old', await createOldBackend(legacy, set, embedder, fs.mkdtempSync(path.join(tempRoot, 'old-')))); - } if (needed.has('mem0')) { backends.set('mem0', await createMem0Backend(set, embedder, fs.mkdtempSync(path.join(tempRoot, 'mem0-')))); } @@ -544,7 +476,7 @@ async function main(): Promise { `Results per query: ${options.limit}; reranker pool: ${RERANK_POOL}`, `mem0ai ${packageVersion(path.join(BENCH_DIR, '..'), 'mem0ai')}, @huggingface/transformers ${packageVersion(path.join(BENCH_DIR, '..', '..', 'core'), '@huggingface/transformers')}, Node ${process.version}`, `Machine: ${os.cpus()[0]?.model ?? 'unknown CPU'}, ${os.cpus().length} cores, ${Math.round(os.totalmem() / 2 ** 30)} GB, ${os.platform()} ${os.release()}`, - `Load time: old engine ${memo('old')}, mem0 ${memo('mem0')}, cosine ${memo('cosine')} (embedding included)`, + `Load time: mem0 ${memo('mem0')}, cosine ${memo('cosine')} (embedding included)`, `Network: ${requests.local} local requests, ${requests.remote} to the embedding API, ${requests.blocked.length} blocked`, ]; console.log(report(set, runs, setup));