refactor(memory-mem0): drop the old engine from the search-quality bench (P2.2)
The bench compared mem0 with MindDB, FrameStore and HybridSearch from the hive-mind substrate, which @waggle/core no longer exports and which left the repository. It now runs mem0, mem0-rerank and cosine; the recorded comparison stays in docs/oss/research/memory-search-quality.md.
This commit is contained in:
@@ -1,22 +1,20 @@
|
||||
/**
|
||||
* Search-quality benchmark for the memory switch (P1.5).
|
||||
* Search-quality benchmark for memory (P1.5).
|
||||
*
|
||||
* Loads the reference set (`reference-set.json`) into the old engine (MindDB +
|
||||
* FrameStore + HybridSearch, re-exported by @waggle/core from the hive-mind
|
||||
* substrate) and into the mem0 store, runs every query against each, and prints
|
||||
* recall@1, recall@5, MRR and latency per engine and per query type. The
|
||||
* findings are in docs/oss/research/memory-search-quality.md.
|
||||
* Loads the reference set (`reference-set.json`) into the mem0 store and a
|
||||
* cosine baseline, runs every query against each, and prints recall@1,
|
||||
* recall@5, MRR and latency per engine and per query type. The comparison with
|
||||
* the engine mem0 replaced is in docs/oss/research/memory-search-quality.md.
|
||||
*
|
||||
* Run from the repository root after `npm run build:packages`:
|
||||
*
|
||||
* npx tsx packages/memory-mem0/bench/search-quality.ts [options]
|
||||
*
|
||||
* --engines <list> Comma-separated: old, old-rerank, mem0, mem0-rerank, cosine.
|
||||
* Default: every engine this checkout still has. Once the
|
||||
* hive-mind substrate is removed, `mem0` and `cosine` are
|
||||
* left (the `-rerank` variants use its cross-encoder).
|
||||
* `cosine` is a baseline: plain cosine similarity over the
|
||||
* same embeddings, with no keyword signal.
|
||||
* --engines <list> Comma-separated: mem0, mem0-rerank, cosine. Default:
|
||||
* every engine. `mem0-rerank` re-orders mem0's results
|
||||
* with the in-process cross-encoder. `cosine` is a
|
||||
* baseline: plain cosine similarity over the same
|
||||
* embeddings, with no keyword signal.
|
||||
* --embedder <name> inprocess (default, offline), ollama or openai.
|
||||
* --limit <n> Results per query (default 10).
|
||||
* --out <file> Also write the per-query results as JSON.
|
||||
@@ -49,11 +47,11 @@ import type { MemoryImportance, MemoryKind } from '@waggle/memory-contract';
|
||||
process.env.MEM0_TELEMETRY = 'false';
|
||||
|
||||
const BENCH_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ENGINE_NAMES = ['old', 'old-rerank', 'mem0', 'mem0-rerank', 'cosine'] as const;
|
||||
const ENGINE_NAMES = ['mem0', 'mem0-rerank', 'cosine'] as const;
|
||||
type EngineName = (typeof ENGINE_NAMES)[number];
|
||||
type BackendName = 'old' | 'mem0' | 'cosine';
|
||||
type BackendName = 'mem0' | 'cosine';
|
||||
const backendOf = (engine: EngineName): BackendName => engine.replace(/-rerank$/, '') as BackendName;
|
||||
/** Candidates a reranker re-orders; HybridSearch's default pool size. */
|
||||
/** Candidates a reranker re-orders. */
|
||||
const RERANK_POOL = 30;
|
||||
|
||||
interface MemoryFixture { id: string; scope: string; kind: string; importance: string; text: string }
|
||||
@@ -73,20 +71,7 @@ interface Backend {
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
/** The parts of the hive-mind substrate the old engine needs; absent once it is removed. */
|
||||
interface LegacyCore {
|
||||
MindDB?: new (file: string) => { close(): void };
|
||||
FrameStore?: new (db: unknown) => {
|
||||
createIFrame(gopId: string, content: string, importance?: string, source?: string): { id: number };
|
||||
};
|
||||
SessionStore?: new (db: unknown) => { ensure(gopId: string): unknown };
|
||||
HybridSearch?: new (db: unknown, embedder: Embedder) => {
|
||||
indexFrame(id: number, content: string): Promise<void>;
|
||||
search(query: string, options: { limit?: number; profile?: string; reranker?: Reranker }):
|
||||
Promise<Array<{ frame: { id: number; content: string }; finalScore: number }>>;
|
||||
};
|
||||
createInProcessReranker?: (config?: { cacheDir?: string }) => Promise<Reranker>;
|
||||
}
|
||||
type RerankerFactory = (config?: { cacheDir?: string }) => Promise<Reranker>;
|
||||
|
||||
interface Options {
|
||||
engines: EngineName[] | null;
|
||||
@@ -198,47 +183,6 @@ function planEmbedder(kind: Options['embedder'], modelsDir: string): EmbedderPla
|
||||
|
||||
// ── Backends ─────────────────────────────────────────────────────────────────
|
||||
|
||||
async function createOldBackend(core: LegacyCore, set: ReferenceSet, embedder: Embedder, dir: string): Promise<Backend> {
|
||||
const { MindDB, FrameStore, SessionStore, HybridSearch } = core;
|
||||
if (!MindDB || !FrameStore || !SessionStore || !HybridSearch) throw new Error('the hive-mind substrate is gone');
|
||||
// Like the sidecar: one .mind file for personal memory and one per workspace.
|
||||
const minds = new Map<string, { db: { close(): void }; search: InstanceType<NonNullable<LegacyCore['HybridSearch']>>; refs: Map<number, string> }>();
|
||||
const started = performance.now();
|
||||
for (const memory of set.memories) {
|
||||
let mind = minds.get(memory.scope);
|
||||
if (!mind) {
|
||||
const db = new MindDB(path.join(dir, `${memory.scope.replace(/\W+/g, '-')}.mind`));
|
||||
new SessionStore(db).ensure('bench');
|
||||
mind = { db, search: new HybridSearch(db, embedder), refs: new Map() };
|
||||
minds.set(memory.scope, mind);
|
||||
}
|
||||
const frame = new FrameStore(mind.db).createIFrame('bench', memory.text, memory.importance, 'user_stated');
|
||||
await mind.search.indexFrame(frame.id, memory.text);
|
||||
mind.refs.set(frame.id, memory.id);
|
||||
}
|
||||
const ingestMs = performance.now() - started;
|
||||
|
||||
return {
|
||||
ingestMs,
|
||||
async search(text, scopes, limit, reranker) {
|
||||
const hits: Hit[] = [];
|
||||
for (const scope of scopes) {
|
||||
const mind = minds.get(scope);
|
||||
if (!mind) continue;
|
||||
// Recall uses the 'balanced' profile (packages/agent/src/orchestrator.ts).
|
||||
for (const result of await mind.search.search(text, { limit, profile: 'balanced', reranker })) {
|
||||
hits.push({ ref: mind.refs.get(result.frame.id)!, text: result.frame.content, score: result.finalScore });
|
||||
}
|
||||
}
|
||||
// One ranked list across minds, ordered by each mind's own final score.
|
||||
return hits.sort((a, b) => b.score - a.score).slice(0, limit);
|
||||
},
|
||||
async close() {
|
||||
for (const mind of minds.values()) mind.db.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function createMem0Backend(set: ReferenceSet, embedder: Embedder, dir: string): Promise<Backend> {
|
||||
const { createMem0MemoryStore } = await import('../src/index.js');
|
||||
const { scopeFromKey } = await import('@waggle/memory-contract');
|
||||
@@ -492,31 +436,19 @@ async function main(): Promise<void> {
|
||||
guardNetwork(plan.remoteHost);
|
||||
|
||||
const core = await import('@waggle/core') as unknown as Record<string, unknown>;
|
||||
const legacy = core as LegacyCore;
|
||||
const hasOldEngine = Boolean(legacy.MindDB && legacy.FrameStore && legacy.SessionStore && legacy.HybridSearch);
|
||||
const hasReranker = Boolean(legacy.createInProcessReranker);
|
||||
const available = (name: EngineName) =>
|
||||
(hasOldEngine || backendOf(name) !== 'old') && (hasReranker || !name.endsWith('-rerank'));
|
||||
const wanted = options.engines ?? ENGINE_NAMES.filter(available);
|
||||
const missing = wanted.filter((name) => !available(name));
|
||||
if (missing.length > 0) {
|
||||
throw new Error(`not in this checkout (the hive-mind substrate is gone): ${missing.join(', ')}; try --engines mem0,cosine`);
|
||||
}
|
||||
const wanted = options.engines ?? [...ENGINE_NAMES];
|
||||
|
||||
const embedder = await plan.create(core);
|
||||
let reranker: Reranker | undefined;
|
||||
if (wanted.some((name) => name.endsWith('-rerank'))) {
|
||||
// The sidecar caches the reranker under <models>/reranker (packages/server/src/local/index.ts).
|
||||
reranker = await legacy.createInProcessReranker!({ cacheDir: path.join(modelsDir, 'reranker') });
|
||||
reranker = await (core.createInProcessReranker as RerankerFactory)({ cacheDir: path.join(modelsDir, 'reranker') });
|
||||
}
|
||||
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-search-bench-'));
|
||||
const backends = new Map<BackendName, Backend>();
|
||||
try {
|
||||
const needed = new Set(wanted.map(backendOf));
|
||||
if (needed.has('old')) {
|
||||
backends.set('old', await createOldBackend(legacy, set, embedder, fs.mkdtempSync(path.join(tempRoot, 'old-'))));
|
||||
}
|
||||
if (needed.has('mem0')) {
|
||||
backends.set('mem0', await createMem0Backend(set, embedder, fs.mkdtempSync(path.join(tempRoot, 'mem0-'))));
|
||||
}
|
||||
@@ -544,7 +476,7 @@ async function main(): Promise<void> {
|
||||
`Results per query: ${options.limit}; reranker pool: ${RERANK_POOL}`,
|
||||
`mem0ai ${packageVersion(path.join(BENCH_DIR, '..'), 'mem0ai')}, @huggingface/transformers ${packageVersion(path.join(BENCH_DIR, '..', '..', 'core'), '@huggingface/transformers')}, Node ${process.version}`,
|
||||
`Machine: ${os.cpus()[0]?.model ?? 'unknown CPU'}, ${os.cpus().length} cores, ${Math.round(os.totalmem() / 2 ** 30)} GB, ${os.platform()} ${os.release()}`,
|
||||
`Load time: old engine ${memo('old')}, mem0 ${memo('mem0')}, cosine ${memo('cosine')} (embedding included)`,
|
||||
`Load time: mem0 ${memo('mem0')}, cosine ${memo('cosine')} (embedding included)`,
|
||||
`Network: ${requests.local} local requests, ${requests.remote} to the embedding API, ${requests.blocked.length} blocked`,
|
||||
];
|
||||
console.log(report(set, runs, setup));
|
||||
|
||||
Reference in New Issue
Block a user