diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json index fe64ce0..2dfcb87 100644 --- a/packages/cli/tsconfig.json +++ b/packages/cli/tsconfig.json @@ -21,6 +21,7 @@ "references": [ { "path": "../core" }, { "path": "../agent" }, - { "path": "../weaver" } + { "path": "../memory-contract" }, + { "path": "../memory-mem0" } ] } diff --git a/packages/core/src/embeddings/inprocess-embedder.ts b/packages/core/src/embeddings/inprocess-embedder.ts index a0e13b4..128a2d3 100644 --- a/packages/core/src/embeddings/inprocess-embedder.ts +++ b/packages/core/src/embeddings/inprocess-embedder.ts @@ -13,6 +13,8 @@ import { withTransformersModelLoad } from './transformers-model-load.js'; const log = createCoreLogger('inprocess-embedder'); +const DEFAULT_MODEL = 'Xenova/all-MiniLM-L6-v2'; + export interface InProcessEmbedderConfig { model?: string; cacheDir?: string; @@ -29,7 +31,7 @@ export function normalizeDimensions(embedding: Float32Array, targetDims: number) } export async function createInProcessEmbedder(config?: Partial): Promise { - const model = config?.model ?? 'Xenova/all-MiniLM-L6-v2'; + const model = config?.model ?? DEFAULT_MODEL; const cacheDir = config?.cacheDir ?? path.join(os.homedir(), '.waggle', 'models'); const targetDims = config?.targetDimensions ?? 1024; @@ -79,13 +81,15 @@ export const MEMORY_EMBEDDING_DIMENSIONS = 384; * The embedder memory uses: the in-process model, loaded on first use. Every * process that opens a data directory's memory must use the same one. */ -export function createMemoryEmbedder(options: { cacheDir: string; model?: string }): Embedder { - return createLazyEmbedder( +export function createMemoryEmbedder(options: { cacheDir: string; model?: string }): Embedder & { id: string } { + const model = options.model ?? DEFAULT_MODEL; + const embedder = createLazyEmbedder( () => createInProcessEmbedder({ - model: options.model, + model, cacheDir: options.cacheDir, targetDimensions: MEMORY_EMBEDDING_DIMENSIONS, }), MEMORY_EMBEDDING_DIMENSIONS, ); + return { ...embedder, id: `inprocess:${model}:${MEMORY_EMBEDDING_DIMENSIONS}` }; } diff --git a/packages/memory-mem0/src/index.ts b/packages/memory-mem0/src/index.ts index 0c55118..99c8f71 100644 --- a/packages/memory-mem0/src/index.ts +++ b/packages/memory-mem0/src/index.ts @@ -7,7 +7,7 @@ * facts across scopes. The contract's own fields are kept in mem0 metadata. */ import { createHash } from 'node:crypto'; -import { mkdirSync } from 'node:fs'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import path from 'node:path'; import type { Memory, MemoryItem } from 'mem0ai/oss'; import { @@ -42,6 +42,12 @@ import { /** Turns text into vectors. Waggle passes its in-process embedder. */ export interface TextEmbedder { readonly dimensions: number; + /** + * Names the model behind the vectors (e.g. 'inprocess:Xenova/all-MiniLM-L6-v2'). + * The store remembers it and refuses to open with a different one, whose + * vectors would not be comparable. Defaults to the dimensions only. + */ + readonly id?: string; embed(text: string): Promise>; embedBatch(texts: string[]): Promise[]>; } @@ -62,6 +68,29 @@ export interface Mem0MemoryStoreOptions { readonly extractionModel?: ExtractionModel; } +/** The memory directory's own record of how it was created. */ +const STORE_INFO_FILE = 'waggle-memory.json'; + +/** The store was written with another embedder; its vectors cannot be compared with new ones. */ +export class EmbedderMismatchError extends Error { + constructor(readonly expected: string, readonly actual: string) { + super(`This memory store was built with embedder "${expected}", not "${actual}". ` + + 'Switch back to that embedder, or move the memory directory aside to start a new store.'); + this.name = 'EmbedderMismatchError'; + } +} + +function checkEmbedder(directory: string, embedder: TextEmbedder): void { + const actual = embedder.id ?? `dimensions:${embedder.dimensions}`; + const infoPath = path.join(directory, STORE_INFO_FILE); + if (existsSync(infoPath)) { + const info = JSON.parse(readFileSync(infoPath, 'utf8')) as { embedder?: string }; + if (info.embedder && info.embedder !== actual) throw new EmbedderMismatchError(info.embedder, actual); + return; + } + writeFileSync(infoPath, JSON.stringify({ embedder: actual }, null, 2)); +} + const USER_ID = 'local'; /** mem0 has no unbounded listing; this is far above a personal memory's size. */ const ALL = 1_000_000; @@ -70,6 +99,7 @@ const SEARCH_POOL = 100; export async function createMem0MemoryStore(options: Mem0MemoryStoreOptions): Promise { mkdirSync(options.directory, { recursive: true }); + checkEmbedder(options.directory, options.embedder); // mem0 reads both variables once, when its module loads. Its telemetry would // send every call to mem0's PostHog project (ADR-0006), and MEM0_DIR keeps it // out of ~/.mem0. That is why mem0 is imported here and not at the top. @@ -372,6 +402,7 @@ export function createWordEmbedder(dimensions = 256): TextEmbedder { }; return { dimensions, + id: `words:${dimensions}`, embed, embedBatch: (texts) => Promise.all(texts.map(embed)), }; diff --git a/packages/memory-mem0/tests/mem0-store.test.ts b/packages/memory-mem0/tests/mem0-store.test.ts index 218068c..1c17d14 100644 --- a/packages/memory-mem0/tests/mem0-store.test.ts +++ b/packages/memory-mem0/tests/mem0-store.test.ts @@ -5,7 +5,7 @@ import os from 'node:os'; import path from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { PERSONAL_SCOPE } from '@waggle/memory-contract'; -import { createMem0MemoryStore, createWordEmbedder, type TextEmbedder } from '../src/index.js'; +import { createMem0MemoryStore, createWordEmbedder, EmbedderMismatchError, type TextEmbedder } from '../src/index.js'; import { describeMemoryStoreContract } from '../../memory-contract/tests/contract-suite.js'; /** Deterministic bag-of-words embedder: texts that share words get similar vectors. */ @@ -164,6 +164,16 @@ describe('mem0 store — local behaviour', () => { } }); + it('refuses to open a store with a different embedder', async () => { + const directory = tempDir(); + const first = await createMem0MemoryStore({ directory, embedder: { ...wordEmbedder, id: 'model-a' } }); + await first.close(); + await expect(createMem0MemoryStore({ directory, embedder: { ...wordEmbedder, id: 'model-b' } })) + .rejects.toThrow(EmbedderMismatchError); + const again = await createMem0MemoryStore({ directory, embedder: { ...wordEmbedder, id: 'model-a' } }); + await again.close(); + }); + it('ships a word embedder that ranks shared words higher', async () => { const embedder = createWordEmbedder(); const [tea, coffee, teaAgain] = await embedder.embedBatch(['green tea', 'black coffee', 'tea']); diff --git a/packages/server/src/local/memory-runtime.ts b/packages/server/src/local/memory-runtime.ts index ce3634f..8a8665f 100644 --- a/packages/server/src/local/memory-runtime.ts +++ b/packages/server/src/local/memory-runtime.ts @@ -32,11 +32,12 @@ export async function openMemoryStore(options: OpenMemoryOptions): Promise Float32Array.from(await words.embed(text)), embedBatch: async (texts) => (await words.embedBatch(texts)).map((v) => Float32Array.from(v)), }; diff --git a/packages/server/tsconfig.paths-check.json b/packages/server/tsconfig.paths-check.json index 4d581dc..f7e80fe 100644 --- a/packages/server/tsconfig.paths-check.json +++ b/packages/server/tsconfig.paths-check.json @@ -12,7 +12,6 @@ "@waggle/hive-mind-core": ["../hive-mind-core/src/index.ts"], "@waggle/marketplace": ["../marketplace/src/index.ts"], "@waggle/waggle-dance": ["../waggle-dance/src/index.ts"], - "@waggle/weaver": ["../weaver/src/index.ts"], "@waggle/sdk": ["../sdk/src/index.ts"] } },