feat(memory): refuse to open a memory store with a different embedder

Vectors of different models are not comparable even at the same size; the
store records its embedder in waggle-memory.json and names the mismatch.
Also drop the weaver references from the cli and server tsconfigs.
This commit is contained in:
shumeiko.it
2026-10-24 12:06:10 +02:00
parent 8b5624520c
commit 8555d84b75
6 changed files with 55 additions and 9 deletions

View File

@@ -7,7 +7,7 @@
* facts across scopes. The contract's own fields are kept in mem0 metadata.
*/
import { createHash } from 'node:crypto';
import { mkdirSync } from 'node:fs';
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import path from 'node:path';
import type { Memory, MemoryItem } from 'mem0ai/oss';
import {
@@ -42,6 +42,12 @@ import {
/** Turns text into vectors. Waggle passes its in-process embedder. */
export interface TextEmbedder {
readonly dimensions: number;
/**
* Names the model behind the vectors (e.g. 'inprocess:Xenova/all-MiniLM-L6-v2').
* The store remembers it and refuses to open with a different one, whose
* vectors would not be comparable. Defaults to the dimensions only.
*/
readonly id?: string;
embed(text: string): Promise<ArrayLike<number>>;
embedBatch(texts: string[]): Promise<ArrayLike<number>[]>;
}
@@ -62,6 +68,29 @@ export interface Mem0MemoryStoreOptions {
readonly extractionModel?: ExtractionModel;
}
/** The memory directory's own record of how it was created. */
const STORE_INFO_FILE = 'waggle-memory.json';
/** The store was written with another embedder; its vectors cannot be compared with new ones. */
export class EmbedderMismatchError extends Error {
constructor(readonly expected: string, readonly actual: string) {
super(`This memory store was built with embedder "${expected}", not "${actual}". `
+ 'Switch back to that embedder, or move the memory directory aside to start a new store.');
this.name = 'EmbedderMismatchError';
}
}
function checkEmbedder(directory: string, embedder: TextEmbedder): void {
const actual = embedder.id ?? `dimensions:${embedder.dimensions}`;
const infoPath = path.join(directory, STORE_INFO_FILE);
if (existsSync(infoPath)) {
const info = JSON.parse(readFileSync(infoPath, 'utf8')) as { embedder?: string };
if (info.embedder && info.embedder !== actual) throw new EmbedderMismatchError(info.embedder, actual);
return;
}
writeFileSync(infoPath, JSON.stringify({ embedder: actual }, null, 2));
}
const USER_ID = 'local';
/** mem0 has no unbounded listing; this is far above a personal memory's size. */
const ALL = 1_000_000;
@@ -70,6 +99,7 @@ const SEARCH_POOL = 100;
export async function createMem0MemoryStore(options: Mem0MemoryStoreOptions): Promise<MemoryStore> {
mkdirSync(options.directory, { recursive: true });
checkEmbedder(options.directory, options.embedder);
// mem0 reads both variables once, when its module loads. Its telemetry would
// send every call to mem0's PostHog project (ADR-0006), and MEM0_DIR keeps it
// out of ~/.mem0. That is why mem0 is imported here and not at the top.
@@ -372,6 +402,7 @@ export function createWordEmbedder(dimensions = 256): TextEmbedder {
};
return {
dimensions,
id: `words:${dimensions}`,
embed,
embedBatch: (texts) => Promise.all(texts.map(embed)),
};

View File

@@ -5,7 +5,7 @@ import os from 'node:os';
import path from 'node:path';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { PERSONAL_SCOPE } from '@waggle/memory-contract';
import { createMem0MemoryStore, createWordEmbedder, type TextEmbedder } from '../src/index.js';
import { createMem0MemoryStore, createWordEmbedder, EmbedderMismatchError, type TextEmbedder } from '../src/index.js';
import { describeMemoryStoreContract } from '../../memory-contract/tests/contract-suite.js';
/** Deterministic bag-of-words embedder: texts that share words get similar vectors. */
@@ -164,6 +164,16 @@ describe('mem0 store — local behaviour', () => {
}
});
it('refuses to open a store with a different embedder', async () => {
const directory = tempDir();
const first = await createMem0MemoryStore({ directory, embedder: { ...wordEmbedder, id: 'model-a' } });
await first.close();
await expect(createMem0MemoryStore({ directory, embedder: { ...wordEmbedder, id: 'model-b' } }))
.rejects.toThrow(EmbedderMismatchError);
const again = await createMem0MemoryStore({ directory, embedder: { ...wordEmbedder, id: 'model-a' } });
await again.close();
});
it('ships a word embedder that ranks shared words higher', async () => {
const embedder = createWordEmbedder();
const [tea, coffee, teaAgain] = await embedder.embedBatch(['green tea', 'black coffee', 'tea']);