feat(memory): refuse to open a memory store with a different embedder
Vectors of different models are not comparable even at the same size; the store records its embedder in waggle-memory.json and names the mismatch. Also drop the weaver references from the cli and server tsconfigs.
This commit is contained in:
@@ -7,7 +7,7 @@
|
||||
* facts across scopes. The contract's own fields are kept in mem0 metadata.
|
||||
*/
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdirSync } from 'node:fs';
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import type { Memory, MemoryItem } from 'mem0ai/oss';
|
||||
import {
|
||||
@@ -42,6 +42,12 @@ import {
|
||||
/** Turns text into vectors. Waggle passes its in-process embedder. */
|
||||
export interface TextEmbedder {
|
||||
readonly dimensions: number;
|
||||
/**
|
||||
* Names the model behind the vectors (e.g. 'inprocess:Xenova/all-MiniLM-L6-v2').
|
||||
* The store remembers it and refuses to open with a different one, whose
|
||||
* vectors would not be comparable. Defaults to the dimensions only.
|
||||
*/
|
||||
readonly id?: string;
|
||||
embed(text: string): Promise<ArrayLike<number>>;
|
||||
embedBatch(texts: string[]): Promise<ArrayLike<number>[]>;
|
||||
}
|
||||
@@ -62,6 +68,29 @@ export interface Mem0MemoryStoreOptions {
|
||||
readonly extractionModel?: ExtractionModel;
|
||||
}
|
||||
|
||||
/** The memory directory's own record of how it was created. */
|
||||
const STORE_INFO_FILE = 'waggle-memory.json';
|
||||
|
||||
/** The store was written with another embedder; its vectors cannot be compared with new ones. */
|
||||
export class EmbedderMismatchError extends Error {
|
||||
constructor(readonly expected: string, readonly actual: string) {
|
||||
super(`This memory store was built with embedder "${expected}", not "${actual}". `
|
||||
+ 'Switch back to that embedder, or move the memory directory aside to start a new store.');
|
||||
this.name = 'EmbedderMismatchError';
|
||||
}
|
||||
}
|
||||
|
||||
function checkEmbedder(directory: string, embedder: TextEmbedder): void {
|
||||
const actual = embedder.id ?? `dimensions:${embedder.dimensions}`;
|
||||
const infoPath = path.join(directory, STORE_INFO_FILE);
|
||||
if (existsSync(infoPath)) {
|
||||
const info = JSON.parse(readFileSync(infoPath, 'utf8')) as { embedder?: string };
|
||||
if (info.embedder && info.embedder !== actual) throw new EmbedderMismatchError(info.embedder, actual);
|
||||
return;
|
||||
}
|
||||
writeFileSync(infoPath, JSON.stringify({ embedder: actual }, null, 2));
|
||||
}
|
||||
|
||||
const USER_ID = 'local';
|
||||
/** mem0 has no unbounded listing; this is far above a personal memory's size. */
|
||||
const ALL = 1_000_000;
|
||||
@@ -70,6 +99,7 @@ const SEARCH_POOL = 100;
|
||||
|
||||
export async function createMem0MemoryStore(options: Mem0MemoryStoreOptions): Promise<MemoryStore> {
|
||||
mkdirSync(options.directory, { recursive: true });
|
||||
checkEmbedder(options.directory, options.embedder);
|
||||
// mem0 reads both variables once, when its module loads. Its telemetry would
|
||||
// send every call to mem0's PostHog project (ADR-0006), and MEM0_DIR keeps it
|
||||
// out of ~/.mem0. That is why mem0 is imported here and not at the top.
|
||||
@@ -372,6 +402,7 @@ export function createWordEmbedder(dimensions = 256): TextEmbedder {
|
||||
};
|
||||
return {
|
||||
dimensions,
|
||||
id: `words:${dimensions}`,
|
||||
embed,
|
||||
embedBatch: (texts) => Promise.all(texts.map(embed)),
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user