feat(memory): refuse to open a memory store with a different embedder
Vectors of different models are not comparable even at the same size; the store records its embedder in waggle-memory.json and names the mismatch. Also drop the weaver references from the cli and server tsconfigs.
This commit is contained in:
@@ -21,6 +21,7 @@
|
||||
"references": [
|
||||
{ "path": "../core" },
|
||||
{ "path": "../agent" },
|
||||
{ "path": "../weaver" }
|
||||
{ "path": "../memory-contract" },
|
||||
{ "path": "../memory-mem0" }
|
||||
]
|
||||
}
|
||||
|
||||
@@ -13,6 +13,8 @@ import { withTransformersModelLoad } from './transformers-model-load.js';
|
||||
|
||||
const log = createCoreLogger('inprocess-embedder');
|
||||
|
||||
const DEFAULT_MODEL = 'Xenova/all-MiniLM-L6-v2';
|
||||
|
||||
export interface InProcessEmbedderConfig {
|
||||
model?: string;
|
||||
cacheDir?: string;
|
||||
@@ -29,7 +31,7 @@ export function normalizeDimensions(embedding: Float32Array, targetDims: number)
|
||||
}
|
||||
|
||||
export async function createInProcessEmbedder(config?: Partial<InProcessEmbedderConfig>): Promise<Embedder> {
|
||||
const model = config?.model ?? 'Xenova/all-MiniLM-L6-v2';
|
||||
const model = config?.model ?? DEFAULT_MODEL;
|
||||
const cacheDir = config?.cacheDir ?? path.join(os.homedir(), '.waggle', 'models');
|
||||
const targetDims = config?.targetDimensions ?? 1024;
|
||||
|
||||
@@ -79,13 +81,15 @@ export const MEMORY_EMBEDDING_DIMENSIONS = 384;
|
||||
* The embedder memory uses: the in-process model, loaded on first use. Every
|
||||
* process that opens a data directory's memory must use the same one.
|
||||
*/
|
||||
export function createMemoryEmbedder(options: { cacheDir: string; model?: string }): Embedder {
|
||||
return createLazyEmbedder(
|
||||
export function createMemoryEmbedder(options: { cacheDir: string; model?: string }): Embedder & { id: string } {
|
||||
const model = options.model ?? DEFAULT_MODEL;
|
||||
const embedder = createLazyEmbedder(
|
||||
() => createInProcessEmbedder({
|
||||
model: options.model,
|
||||
model,
|
||||
cacheDir: options.cacheDir,
|
||||
targetDimensions: MEMORY_EMBEDDING_DIMENSIONS,
|
||||
}),
|
||||
MEMORY_EMBEDDING_DIMENSIONS,
|
||||
);
|
||||
return { ...embedder, id: `inprocess:${model}:${MEMORY_EMBEDDING_DIMENSIONS}` };
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
* facts across scopes. The contract's own fields are kept in mem0 metadata.
|
||||
*/
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdirSync } from 'node:fs';
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import type { Memory, MemoryItem } from 'mem0ai/oss';
|
||||
import {
|
||||
@@ -42,6 +42,12 @@ import {
|
||||
/** Turns text into vectors. Waggle passes its in-process embedder. */
|
||||
export interface TextEmbedder {
|
||||
readonly dimensions: number;
|
||||
/**
|
||||
* Names the model behind the vectors (e.g. 'inprocess:Xenova/all-MiniLM-L6-v2').
|
||||
* The store remembers it and refuses to open with a different one, whose
|
||||
* vectors would not be comparable. Defaults to the dimensions only.
|
||||
*/
|
||||
readonly id?: string;
|
||||
embed(text: string): Promise<ArrayLike<number>>;
|
||||
embedBatch(texts: string[]): Promise<ArrayLike<number>[]>;
|
||||
}
|
||||
@@ -62,6 +68,29 @@ export interface Mem0MemoryStoreOptions {
|
||||
readonly extractionModel?: ExtractionModel;
|
||||
}
|
||||
|
||||
/** The memory directory's own record of how it was created. */
|
||||
const STORE_INFO_FILE = 'waggle-memory.json';
|
||||
|
||||
/** The store was written with another embedder; its vectors cannot be compared with new ones. */
|
||||
export class EmbedderMismatchError extends Error {
|
||||
constructor(readonly expected: string, readonly actual: string) {
|
||||
super(`This memory store was built with embedder "${expected}", not "${actual}". `
|
||||
+ 'Switch back to that embedder, or move the memory directory aside to start a new store.');
|
||||
this.name = 'EmbedderMismatchError';
|
||||
}
|
||||
}
|
||||
|
||||
function checkEmbedder(directory: string, embedder: TextEmbedder): void {
|
||||
const actual = embedder.id ?? `dimensions:${embedder.dimensions}`;
|
||||
const infoPath = path.join(directory, STORE_INFO_FILE);
|
||||
if (existsSync(infoPath)) {
|
||||
const info = JSON.parse(readFileSync(infoPath, 'utf8')) as { embedder?: string };
|
||||
if (info.embedder && info.embedder !== actual) throw new EmbedderMismatchError(info.embedder, actual);
|
||||
return;
|
||||
}
|
||||
writeFileSync(infoPath, JSON.stringify({ embedder: actual }, null, 2));
|
||||
}
|
||||
|
||||
const USER_ID = 'local';
|
||||
/** mem0 has no unbounded listing; this is far above a personal memory's size. */
|
||||
const ALL = 1_000_000;
|
||||
@@ -70,6 +99,7 @@ const SEARCH_POOL = 100;
|
||||
|
||||
export async function createMem0MemoryStore(options: Mem0MemoryStoreOptions): Promise<MemoryStore> {
|
||||
mkdirSync(options.directory, { recursive: true });
|
||||
checkEmbedder(options.directory, options.embedder);
|
||||
// mem0 reads both variables once, when its module loads. Its telemetry would
|
||||
// send every call to mem0's PostHog project (ADR-0006), and MEM0_DIR keeps it
|
||||
// out of ~/.mem0. That is why mem0 is imported here and not at the top.
|
||||
@@ -372,6 +402,7 @@ export function createWordEmbedder(dimensions = 256): TextEmbedder {
|
||||
};
|
||||
return {
|
||||
dimensions,
|
||||
id: `words:${dimensions}`,
|
||||
embed,
|
||||
embedBatch: (texts) => Promise.all(texts.map(embed)),
|
||||
};
|
||||
|
||||
@@ -5,7 +5,7 @@ import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { PERSONAL_SCOPE } from '@waggle/memory-contract';
|
||||
import { createMem0MemoryStore, createWordEmbedder, type TextEmbedder } from '../src/index.js';
|
||||
import { createMem0MemoryStore, createWordEmbedder, EmbedderMismatchError, type TextEmbedder } from '../src/index.js';
|
||||
import { describeMemoryStoreContract } from '../../memory-contract/tests/contract-suite.js';
|
||||
|
||||
/** Deterministic bag-of-words embedder: texts that share words get similar vectors. */
|
||||
@@ -164,6 +164,16 @@ describe('mem0 store — local behaviour', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('refuses to open a store with a different embedder', async () => {
|
||||
const directory = tempDir();
|
||||
const first = await createMem0MemoryStore({ directory, embedder: { ...wordEmbedder, id: 'model-a' } });
|
||||
await first.close();
|
||||
await expect(createMem0MemoryStore({ directory, embedder: { ...wordEmbedder, id: 'model-b' } }))
|
||||
.rejects.toThrow(EmbedderMismatchError);
|
||||
const again = await createMem0MemoryStore({ directory, embedder: { ...wordEmbedder, id: 'model-a' } });
|
||||
await again.close();
|
||||
});
|
||||
|
||||
it('ships a word embedder that ranks shared words higher', async () => {
|
||||
const embedder = createWordEmbedder();
|
||||
const [tea, coffee, teaAgain] = await embedder.embedBatch(['green tea', 'black coffee', 'tea']);
|
||||
|
||||
@@ -32,11 +32,12 @@ export async function openMemoryStore(options: OpenMemoryOptions): Promise<Memor
|
||||
* model download; tests use it. Otherwise memory uses the in-process model,
|
||||
* loaded on first use so startup never waits for a download.
|
||||
*/
|
||||
function memoryEmbedder(options: OpenMemoryOptions): Embedder {
|
||||
function memoryEmbedder(options: OpenMemoryOptions): Embedder & { id?: string } {
|
||||
if (process.env.WAGGLE_MEMORY_EMBEDDER === 'words') {
|
||||
const words = createWordEmbedder();
|
||||
return {
|
||||
dimensions: words.dimensions,
|
||||
id: words.id,
|
||||
embed: async (text) => Float32Array.from(await words.embed(text)),
|
||||
embedBatch: async (texts) => (await words.embedBatch(texts)).map((v) => Float32Array.from(v)),
|
||||
};
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
"@waggle/hive-mind-core": ["../hive-mind-core/src/index.ts"],
|
||||
"@waggle/marketplace": ["../marketplace/src/index.ts"],
|
||||
"@waggle/waggle-dance": ["../waggle-dance/src/index.ts"],
|
||||
"@waggle/weaver": ["../weaver/src/index.ts"],
|
||||
"@waggle/sdk": ["../sdk/src/index.ts"]
|
||||
}
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user