refactor(memory-mem0): drop the old engine from the search-quality bench (P2.2)
The bench compared mem0 with MindDB, FrameStore and HybridSearch from the hive-mind substrate, which @waggle/core no longer exports and which left the repository. It now runs mem0, mem0-rerank and cosine; the recorded comparison stays in docs/oss/research/memory-search-quality.md.
This commit is contained in:
@@ -315,13 +315,14 @@ What this bench does not cover:
|
|||||||
|
|
||||||
## How to re-run
|
## How to re-run
|
||||||
|
|
||||||
From the repository root, after `npm ci` and `npm run build:packages`:
|
From the repository root, after `npm ci` and `npm run build:packages`. The old engine left
|
||||||
|
the repository in P2.2, so the bench runs mem0, mem0-rerank and cosine:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# All engines this checkout has (old, old-rerank, mem0, mem0-rerank, cosine)
|
# All engines (mem0, mem0-rerank, cosine)
|
||||||
npx tsx packages/memory-mem0/bench/search-quality.ts
|
npx tsx packages/memory-mem0/bench/search-quality.ts
|
||||||
|
|
||||||
# After the old engine is removed
|
# Without the reranker model
|
||||||
npx tsx packages/memory-mem0/bench/search-quality.ts --engines mem0,cosine
|
npx tsx packages/memory-mem0/bench/search-quality.ts --engines mem0,cosine
|
||||||
|
|
||||||
# Per-query results as JSON
|
# Per-query results as JSON
|
||||||
|
|||||||
@@ -1,22 +1,20 @@
|
|||||||
/**
|
/**
|
||||||
* Search-quality benchmark for the memory switch (P1.5).
|
* Search-quality benchmark for memory (P1.5).
|
||||||
*
|
*
|
||||||
* Loads the reference set (`reference-set.json`) into the old engine (MindDB +
|
* Loads the reference set (`reference-set.json`) into the mem0 store and a
|
||||||
* FrameStore + HybridSearch, re-exported by @waggle/core from the hive-mind
|
* cosine baseline, runs every query against each, and prints recall@1,
|
||||||
* substrate) and into the mem0 store, runs every query against each, and prints
|
* recall@5, MRR and latency per engine and per query type. The comparison with
|
||||||
* recall@1, recall@5, MRR and latency per engine and per query type. The
|
* the engine mem0 replaced is in docs/oss/research/memory-search-quality.md.
|
||||||
* findings are in docs/oss/research/memory-search-quality.md.
|
|
||||||
*
|
*
|
||||||
* Run from the repository root after `npm run build:packages`:
|
* Run from the repository root after `npm run build:packages`:
|
||||||
*
|
*
|
||||||
* npx tsx packages/memory-mem0/bench/search-quality.ts [options]
|
* npx tsx packages/memory-mem0/bench/search-quality.ts [options]
|
||||||
*
|
*
|
||||||
* --engines <list> Comma-separated: old, old-rerank, mem0, mem0-rerank, cosine.
|
* --engines <list> Comma-separated: mem0, mem0-rerank, cosine. Default:
|
||||||
* Default: every engine this checkout still has. Once the
|
* every engine. `mem0-rerank` re-orders mem0's results
|
||||||
* hive-mind substrate is removed, `mem0` and `cosine` are
|
* with the in-process cross-encoder. `cosine` is a
|
||||||
* left (the `-rerank` variants use its cross-encoder).
|
* baseline: plain cosine similarity over the same
|
||||||
* `cosine` is a baseline: plain cosine similarity over the
|
* embeddings, with no keyword signal.
|
||||||
* same embeddings, with no keyword signal.
|
|
||||||
* --embedder <name> inprocess (default, offline), ollama or openai.
|
* --embedder <name> inprocess (default, offline), ollama or openai.
|
||||||
* --limit <n> Results per query (default 10).
|
* --limit <n> Results per query (default 10).
|
||||||
* --out <file> Also write the per-query results as JSON.
|
* --out <file> Also write the per-query results as JSON.
|
||||||
@@ -49,11 +47,11 @@ import type { MemoryImportance, MemoryKind } from '@waggle/memory-contract';
|
|||||||
process.env.MEM0_TELEMETRY = 'false';
|
process.env.MEM0_TELEMETRY = 'false';
|
||||||
|
|
||||||
const BENCH_DIR = path.dirname(fileURLToPath(import.meta.url));
|
const BENCH_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||||
const ENGINE_NAMES = ['old', 'old-rerank', 'mem0', 'mem0-rerank', 'cosine'] as const;
|
const ENGINE_NAMES = ['mem0', 'mem0-rerank', 'cosine'] as const;
|
||||||
type EngineName = (typeof ENGINE_NAMES)[number];
|
type EngineName = (typeof ENGINE_NAMES)[number];
|
||||||
type BackendName = 'old' | 'mem0' | 'cosine';
|
type BackendName = 'mem0' | 'cosine';
|
||||||
const backendOf = (engine: EngineName): BackendName => engine.replace(/-rerank$/, '') as BackendName;
|
const backendOf = (engine: EngineName): BackendName => engine.replace(/-rerank$/, '') as BackendName;
|
||||||
/** Candidates a reranker re-orders; HybridSearch's default pool size. */
|
/** Candidates a reranker re-orders. */
|
||||||
const RERANK_POOL = 30;
|
const RERANK_POOL = 30;
|
||||||
|
|
||||||
interface MemoryFixture { id: string; scope: string; kind: string; importance: string; text: string }
|
interface MemoryFixture { id: string; scope: string; kind: string; importance: string; text: string }
|
||||||
@@ -73,20 +71,7 @@ interface Backend {
|
|||||||
close(): Promise<void>;
|
close(): Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The parts of the hive-mind substrate the old engine needs; absent once it is removed. */
|
type RerankerFactory = (config?: { cacheDir?: string }) => Promise<Reranker>;
|
||||||
interface LegacyCore {
|
|
||||||
MindDB?: new (file: string) => { close(): void };
|
|
||||||
FrameStore?: new (db: unknown) => {
|
|
||||||
createIFrame(gopId: string, content: string, importance?: string, source?: string): { id: number };
|
|
||||||
};
|
|
||||||
SessionStore?: new (db: unknown) => { ensure(gopId: string): unknown };
|
|
||||||
HybridSearch?: new (db: unknown, embedder: Embedder) => {
|
|
||||||
indexFrame(id: number, content: string): Promise<void>;
|
|
||||||
search(query: string, options: { limit?: number; profile?: string; reranker?: Reranker }):
|
|
||||||
Promise<Array<{ frame: { id: number; content: string }; finalScore: number }>>;
|
|
||||||
};
|
|
||||||
createInProcessReranker?: (config?: { cacheDir?: string }) => Promise<Reranker>;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface Options {
|
interface Options {
|
||||||
engines: EngineName[] | null;
|
engines: EngineName[] | null;
|
||||||
@@ -198,47 +183,6 @@ function planEmbedder(kind: Options['embedder'], modelsDir: string): EmbedderPla
|
|||||||
|
|
||||||
// ── Backends ─────────────────────────────────────────────────────────────────
|
// ── Backends ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
async function createOldBackend(core: LegacyCore, set: ReferenceSet, embedder: Embedder, dir: string): Promise<Backend> {
|
|
||||||
const { MindDB, FrameStore, SessionStore, HybridSearch } = core;
|
|
||||||
if (!MindDB || !FrameStore || !SessionStore || !HybridSearch) throw new Error('the hive-mind substrate is gone');
|
|
||||||
// Like the sidecar: one .mind file for personal memory and one per workspace.
|
|
||||||
const minds = new Map<string, { db: { close(): void }; search: InstanceType<NonNullable<LegacyCore['HybridSearch']>>; refs: Map<number, string> }>();
|
|
||||||
const started = performance.now();
|
|
||||||
for (const memory of set.memories) {
|
|
||||||
let mind = minds.get(memory.scope);
|
|
||||||
if (!mind) {
|
|
||||||
const db = new MindDB(path.join(dir, `${memory.scope.replace(/\W+/g, '-')}.mind`));
|
|
||||||
new SessionStore(db).ensure('bench');
|
|
||||||
mind = { db, search: new HybridSearch(db, embedder), refs: new Map() };
|
|
||||||
minds.set(memory.scope, mind);
|
|
||||||
}
|
|
||||||
const frame = new FrameStore(mind.db).createIFrame('bench', memory.text, memory.importance, 'user_stated');
|
|
||||||
await mind.search.indexFrame(frame.id, memory.text);
|
|
||||||
mind.refs.set(frame.id, memory.id);
|
|
||||||
}
|
|
||||||
const ingestMs = performance.now() - started;
|
|
||||||
|
|
||||||
return {
|
|
||||||
ingestMs,
|
|
||||||
async search(text, scopes, limit, reranker) {
|
|
||||||
const hits: Hit[] = [];
|
|
||||||
for (const scope of scopes) {
|
|
||||||
const mind = minds.get(scope);
|
|
||||||
if (!mind) continue;
|
|
||||||
// Recall uses the 'balanced' profile (packages/agent/src/orchestrator.ts).
|
|
||||||
for (const result of await mind.search.search(text, { limit, profile: 'balanced', reranker })) {
|
|
||||||
hits.push({ ref: mind.refs.get(result.frame.id)!, text: result.frame.content, score: result.finalScore });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// One ranked list across minds, ordered by each mind's own final score.
|
|
||||||
return hits.sort((a, b) => b.score - a.score).slice(0, limit);
|
|
||||||
},
|
|
||||||
async close() {
|
|
||||||
for (const mind of minds.values()) mind.db.close();
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function createMem0Backend(set: ReferenceSet, embedder: Embedder, dir: string): Promise<Backend> {
|
async function createMem0Backend(set: ReferenceSet, embedder: Embedder, dir: string): Promise<Backend> {
|
||||||
const { createMem0MemoryStore } = await import('../src/index.js');
|
const { createMem0MemoryStore } = await import('../src/index.js');
|
||||||
const { scopeFromKey } = await import('@waggle/memory-contract');
|
const { scopeFromKey } = await import('@waggle/memory-contract');
|
||||||
@@ -492,31 +436,19 @@ async function main(): Promise<void> {
|
|||||||
guardNetwork(plan.remoteHost);
|
guardNetwork(plan.remoteHost);
|
||||||
|
|
||||||
const core = await import('@waggle/core') as unknown as Record<string, unknown>;
|
const core = await import('@waggle/core') as unknown as Record<string, unknown>;
|
||||||
const legacy = core as LegacyCore;
|
const wanted = options.engines ?? [...ENGINE_NAMES];
|
||||||
const hasOldEngine = Boolean(legacy.MindDB && legacy.FrameStore && legacy.SessionStore && legacy.HybridSearch);
|
|
||||||
const hasReranker = Boolean(legacy.createInProcessReranker);
|
|
||||||
const available = (name: EngineName) =>
|
|
||||||
(hasOldEngine || backendOf(name) !== 'old') && (hasReranker || !name.endsWith('-rerank'));
|
|
||||||
const wanted = options.engines ?? ENGINE_NAMES.filter(available);
|
|
||||||
const missing = wanted.filter((name) => !available(name));
|
|
||||||
if (missing.length > 0) {
|
|
||||||
throw new Error(`not in this checkout (the hive-mind substrate is gone): ${missing.join(', ')}; try --engines mem0,cosine`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const embedder = await plan.create(core);
|
const embedder = await plan.create(core);
|
||||||
let reranker: Reranker | undefined;
|
let reranker: Reranker | undefined;
|
||||||
if (wanted.some((name) => name.endsWith('-rerank'))) {
|
if (wanted.some((name) => name.endsWith('-rerank'))) {
|
||||||
// The sidecar caches the reranker under <models>/reranker (packages/server/src/local/index.ts).
|
// The sidecar caches the reranker under <models>/reranker (packages/server/src/local/index.ts).
|
||||||
reranker = await legacy.createInProcessReranker!({ cacheDir: path.join(modelsDir, 'reranker') });
|
reranker = await (core.createInProcessReranker as RerankerFactory)({ cacheDir: path.join(modelsDir, 'reranker') });
|
||||||
}
|
}
|
||||||
|
|
||||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-search-bench-'));
|
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-search-bench-'));
|
||||||
const backends = new Map<BackendName, Backend>();
|
const backends = new Map<BackendName, Backend>();
|
||||||
try {
|
try {
|
||||||
const needed = new Set(wanted.map(backendOf));
|
const needed = new Set(wanted.map(backendOf));
|
||||||
if (needed.has('old')) {
|
|
||||||
backends.set('old', await createOldBackend(legacy, set, embedder, fs.mkdtempSync(path.join(tempRoot, 'old-'))));
|
|
||||||
}
|
|
||||||
if (needed.has('mem0')) {
|
if (needed.has('mem0')) {
|
||||||
backends.set('mem0', await createMem0Backend(set, embedder, fs.mkdtempSync(path.join(tempRoot, 'mem0-'))));
|
backends.set('mem0', await createMem0Backend(set, embedder, fs.mkdtempSync(path.join(tempRoot, 'mem0-'))));
|
||||||
}
|
}
|
||||||
@@ -544,7 +476,7 @@ async function main(): Promise<void> {
|
|||||||
`Results per query: ${options.limit}; reranker pool: ${RERANK_POOL}`,
|
`Results per query: ${options.limit}; reranker pool: ${RERANK_POOL}`,
|
||||||
`mem0ai ${packageVersion(path.join(BENCH_DIR, '..'), 'mem0ai')}, @huggingface/transformers ${packageVersion(path.join(BENCH_DIR, '..', '..', 'core'), '@huggingface/transformers')}, Node ${process.version}`,
|
`mem0ai ${packageVersion(path.join(BENCH_DIR, '..'), 'mem0ai')}, @huggingface/transformers ${packageVersion(path.join(BENCH_DIR, '..', '..', 'core'), '@huggingface/transformers')}, Node ${process.version}`,
|
||||||
`Machine: ${os.cpus()[0]?.model ?? 'unknown CPU'}, ${os.cpus().length} cores, ${Math.round(os.totalmem() / 2 ** 30)} GB, ${os.platform()} ${os.release()}`,
|
`Machine: ${os.cpus()[0]?.model ?? 'unknown CPU'}, ${os.cpus().length} cores, ${Math.round(os.totalmem() / 2 ** 30)} GB, ${os.platform()} ${os.release()}`,
|
||||||
`Load time: old engine ${memo('old')}, mem0 ${memo('mem0')}, cosine ${memo('cosine')} (embedding included)`,
|
`Load time: mem0 ${memo('mem0')}, cosine ${memo('cosine')} (embedding included)`,
|
||||||
`Network: ${requests.local} local requests, ${requests.remote} to the embedding API, ${requests.blocked.length} blocked`,
|
`Network: ${requests.local} local requests, ${requests.remote} to the embedding API, ${requests.blocked.length} blocked`,
|
||||||
];
|
];
|
||||||
console.log(report(set, runs, setup));
|
console.log(report(set, runs, setup));
|
||||||
|
|||||||
Reference in New Issue
Block a user