/** * A `MemoryStore` backed by the open-source mem0 library, run in-process * (ADR-0002, ADR-0007). Everything lives in SQLite files in one directory. * * Mapping onto mem0: every memory belongs to user 'local'; the memory scope is * mem0's `agent_id` ('personal' or 'workspace:'), so mem0 never compares * facts across scopes. The contract's own fields are kept in mem0 metadata. */ import { createHash } from 'node:crypto'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import path from 'node:path'; import type { Memory, MemoryItem } from 'mem0ai/oss'; import { MEMORY_IMPORTANCE, MEMORY_KINDS, MEMORY_SOURCES, MemoryCapabilityError, MemoryNotFoundError, RESERVED_METADATA_KEYS, assertFilter, assertNoReservedKeys, importanceAtLeast, normalizeRememberInput, scopeFromKey, scopeKey, type MemoryCapabilities, type MemoryChange, type MemoryFilter, type MemoryHistoryEntry, type MemoryHit, type MemoryImportance, type MemoryKind, type MemoryQuery, type MemoryRecord, type MemoryScope, type MemorySource, type MemoryStore, type MemoryUpdate, type RememberInput, } from '@waggle/memory-contract'; /** Turns text into vectors. Waggle passes its in-process embedder. */ export interface TextEmbedder { readonly dimensions: number; /** * Names the model behind the vectors (e.g. 'inprocess:Xenova/all-MiniLM-L6-v2'). * The store remembers it and refuses to open with a different one, whose * vectors would not be comparable. Defaults to the dimensions only. */ readonly id?: string; embed(text: string): Promise>; embedBatch(texts: string[]): Promise[]>; } /** An OpenAI-compatible chat endpoint, used to extract facts from conversations. */ export interface ExtractionModel { readonly baseURL: string; readonly apiKey: string; /** The model name, or a function read on every extraction so a model switch applies at once. */ readonly model: string | (() => string); } export interface Mem0MemoryStoreOptions { /** Directory for the SQLite files; created when missing. */ readonly directory: string; readonly embedder: TextEmbedder; /** Without it the store cannot extract facts (`capabilities.extraction` is false). */ readonly extractionModel?: ExtractionModel; } /** The memory directory's own record of how it was created. */ const STORE_INFO_FILE = 'waggle-memory.json'; /** The store was written with another embedder; its vectors cannot be compared with new ones. */ export class EmbedderMismatchError extends Error { constructor(readonly expected: string, readonly actual: string) { super(`This memory store was built with embedder "${expected}", not "${actual}". ` + 'Switch back to that embedder, or move the memory directory aside to start a new store.'); this.name = 'EmbedderMismatchError'; } } function checkEmbedder(directory: string, embedder: TextEmbedder): void { const actual = embedder.id ?? `dimensions:${embedder.dimensions}`; const infoPath = path.join(directory, STORE_INFO_FILE); if (existsSync(infoPath)) { const info = JSON.parse(readFileSync(infoPath, 'utf8')) as { embedder?: string }; if (info.embedder && info.embedder !== actual) throw new EmbedderMismatchError(info.embedder, actual); return; } writeFileSync(infoPath, JSON.stringify({ embedder: actual }, null, 2)); } const USER_ID = 'local'; /** mem0 has no unbounded listing; this is far above a personal memory's size. */ const ALL = 1_000_000; /** Candidates fetched per scope before the contract's filters are applied. */ const SEARCH_POOL = 100; export async function createMem0MemoryStore(options: Mem0MemoryStoreOptions): Promise { mkdirSync(options.directory, { recursive: true }); checkEmbedder(options.directory, options.embedder); // mem0 reads both variables once, when its module loads. Its telemetry would // send every call to mem0's PostHog project (ADR-0006), and MEM0_DIR keeps it // out of ~/.mem0. That is why mem0 is imported here and not at the top. process.env.MEM0_TELEMETRY = 'false'; process.env.MEM0_DIR = options.directory; const { Memory: Mem0 } = await import('mem0ai/oss'); const model = options.extractionModel; const modelName = (): string => (typeof model?.model === 'function' ? model.model() : model?.model ?? ''); const mem0: Memory = new Mem0({ version: 'v1.1', embedder: { provider: 'langchain', config: { model: asLangchainEmbeddings(options.embedder) } }, vectorStore: { provider: 'memory', config: { collectionName: 'waggle', dimension: options.embedder.dimensions, dbPath: path.join(options.directory, 'vectors.db'), }, }, // mem0 requires an LLM even when nothing calls it; without a model it is never used. llm: { provider: 'openai', config: model ? { baseURL: model.baseURL, apiKey: model.apiKey, model: modelName() } : { baseURL: 'http://127.0.0.1:9/v1', apiKey: 'unused', model: 'unused' }, }, historyStore: { provider: 'sqlite', config: { historyDbPath: path.join(options.directory, 'history.db') }, }, }); if (typeof model?.model === 'function') { // mem0 3.3.1 reads its LLM's `model` field on every call; a getter makes it // follow the caller's current model. Re-check on any mem0 upgrade (ADR-0007). const llm = (mem0 as unknown as { llm?: object }).llm; if (!llm) throw new Error('mem0 changed its internals: no llm to configure'); Object.defineProperty(llm, 'model', { get: modelName, configurable: true }); } const capabilities: MemoryCapabilities = { extraction: model !== undefined, semanticSearch: true, history: true, knowledgeGraph: false, }; // Erasing a memory must erase its text. mem0's delete keeps the old text in // its history table, and extraction keeps the last messages of each scope; // both live in history.db, which mem0 3.3.1 holds open as `db.db`. const historyDb = (mem0 as unknown as { db?: { db?: HistoryDatabase } }).db?.db; if (!historyDb) throw new Error('mem0 changed its internals: no history database to purge'); const purgeHistory = historyDb.prepare('DELETE FROM memory_history WHERE memory_id = ?'); const purgeMessages = historyDb.prepare('DELETE FROM messages WHERE session_scope = ?'); let closed = false; const ensureOpen = (): void => { if (closed) throw new Error('memory store is closed'); }; const scopeItems = async (scope: MemoryScope): Promise => { const { results } = await mem0.getAll({ topK: ALL, filters: scopeFilter(scope) }); return results; }; const matching = async (filter: MemoryFilter): Promise => { assertFilter(filter); const records: MemoryRecord[] = []; for (const scope of filter.scopes) { for (const item of await scopeItems(scope)) { const record = toRecord(item); if (record && passes(record, filter)) records.push(record); } } return records; }; const getRecord = async (id: string): Promise => { const item = await mem0.get(id); return item ? toRecord(item) : null; }; return { capabilities, async remember(input: RememberInput): Promise { ensureOpen(); const normalized = normalizeRememberInput(input); const metadata = { ...normalized.metadata, kind: normalized.kind, importance: normalized.importance, source: normalized.source, ...(normalized.origin !== null && { origin: normalized.origin }), ...(normalized.sessionId !== null && { session_id: normalized.sessionId }), }; const target = { userId: USER_ID, agentId: scopeKey(normalized.scope), metadata }; if (normalized.conversation) { if (!model) throw new MemoryCapabilityError('extraction'); const messages = normalized.conversation .filter((m) => m.content.trim()) .map((m) => ({ role: m.role, content: m.content })); const { results } = await mem0.add(messages, { ...target, infer: true }); return toChanges(results, getRecord, 'added'); } const text = normalized.text!; const { results: existing } = await mem0.getAll({ topK: 1, filters: { ...scopeFilter(normalized.scope), hash: md5(text) }, }); const duplicate = existing[0] ? toRecord(existing[0]) : null; if (duplicate) return [{ event: 'unchanged', record: duplicate }]; const { results } = await mem0.add(text, { ...target, infer: false }); return toChanges(results, getRecord, 'added'); }, async search(text: string, query: MemoryQuery): Promise { ensureOpen(); assertFilter(query); const limit = query.limit ?? 10; const hits: MemoryHit[] = []; for (const scope of query.scopes) { const { results } = await mem0.search(text, { topK: Math.max(limit * 5, SEARCH_POOL), threshold: 0, filters: scopeFilter(scope), }); for (const item of results) { const record = toRecord(item); if (record && passes(record, query)) hits.push({ ...record, score: item.score ?? 0 }); } } return hits.sort((a, b) => b.score - a.score).slice(0, limit); }, async list(query: MemoryQuery): Promise { ensureOpen(); const records = await matching(query); return records .sort((a, b) => b.createdAt.localeCompare(a.createdAt)) .slice(0, query.limit ?? 50); }, async count(filter: MemoryFilter): Promise { ensureOpen(); return (await matching(filter)).length; }, async get(id: string): Promise { ensureOpen(); return getRecord(id); }, async update(id: string, change: MemoryUpdate): Promise { ensureOpen(); if (!(await getRecord(id))) throw new MemoryNotFoundError(id); if (change.metadata) assertNoReservedKeys(change.metadata); await mem0.update(id, { ...(change.text?.trim() && { text: change.text.trim() }), metadata: { ...change.metadata, ...(change.kind && { kind: change.kind }), ...(change.importance && { importance: change.importance }), }, }); const updated = await getRecord(id); if (!updated) throw new MemoryNotFoundError(id); return updated; }, async forget(id: string): Promise { ensureOpen(); if (!(await getRecord(id))) return false; await mem0.delete(id); purgeHistory.run(id); return true; }, async forgetMatching(filter: MemoryFilter): Promise { ensureOpen(); const doomed = await matching(filter); for (const record of doomed) { await mem0.delete(record.id); purgeHistory.run(record.id); } if (isWholeScope(filter)) { for (const scope of filter.scopes) purgeMessages.run(sessionScope(scope)); } return doomed.length; }, async history(id: string): Promise { ensureOpen(); const rows = (await mem0.history(id)) as HistoryRow[]; return rows .map(toHistoryEntry) .filter((entry): entry is MemoryHistoryEntry => entry !== null) .sort((a, b) => a.at.localeCompare(b.at)); }, async close(): Promise { // mem0 keeps its SQLite handles open for the life of the process. closed = true; }, }; } interface HistoryRow { action?: string; previous_value?: string | null; new_value?: string | null; created_at?: string | null; updated_at?: string | null; } const HISTORY_EVENTS: Record = { ADD: 'added', UPDATE: 'updated', DELETE: 'deleted', }; function toHistoryEntry(row: HistoryRow): MemoryHistoryEntry | null { const event = HISTORY_EVENTS[row.action ?? '']; const at = row.updated_at ?? row.created_at; if (!event || !at) return null; return { event, previousText: row.previous_value ?? null, text: row.new_value ?? null, at }; } /** The part of better-sqlite3 the store uses on mem0's history database. */ interface HistoryDatabase { prepare(sql: string): { run(...params: unknown[]): unknown }; } /** A filter that selects everything in its scopes. */ function isWholeScope(filter: MemoryFilter): boolean { return !filter.kinds && !filter.minImportance && filter.origin === undefined && filter.sessionId === undefined && filter.since === undefined && filter.until === undefined; } /** mem0's key for the conversation messages it keeps per scope (agent_id, user_id, sorted). */ function sessionScope(scope: MemoryScope): string { const escape = (value: string): string => value.replace(/%/g, '%25').replace(/&/g, '%26').replace(/=/g, '%3D'); return `agent_id=${escape(scopeKey(scope))}&user_id=${escape(USER_ID)}`; } function scopeFilter(scope: MemoryScope): { user_id: string; agent_id: string } { return { user_id: USER_ID, agent_id: scopeKey(scope) }; } /** mem0 hashes memory text with md5; the same hash finds an identical memory. */ function md5(text: string): string { return createHash('md5').update(text).digest('hex'); } function passes(record: MemoryRecord, filter: MemoryFilter): boolean { const importance = filter.minImportance ? importanceAtLeast(filter.minImportance) : null; return (!filter.kinds || filter.kinds.includes(record.kind)) && (!importance || importance.includes(record.importance)) && (filter.origin === undefined || record.origin === filter.origin) && (filter.sessionId === undefined || record.sessionId === filter.sessionId) && (filter.since === undefined || record.createdAt >= filter.since) && (filter.until === undefined || record.createdAt <= filter.until); } /** Converts a mem0 item, or returns null for an item this store did not write. */ function toRecord(item: MemoryItem & { user_id?: string; agent_id?: string }): MemoryRecord | null { const scope = item.agent_id ? scopeFromKey(item.agent_id) : null; if (!scope || item.user_id !== USER_ID || !item.createdAt) return null; const metadata: Record = { ...item.metadata }; const own = Object.fromEntries(RESERVED_METADATA_KEYS.map((key) => [key, metadata[key]])); for (const key of RESERVED_METADATA_KEYS) delete metadata[key]; const updatedAt = item.updatedAt && item.updatedAt !== item.createdAt ? item.updatedAt : null; return { id: item.id, text: item.memory, scope, kind: oneOf(own.kind, MEMORY_KINDS, 'fact') as MemoryKind, importance: oneOf(own.importance, MEMORY_IMPORTANCE, 'normal') as MemoryImportance, source: oneOf(own.source, MEMORY_SOURCES, 'agent_inferred') as MemorySource, origin: typeof own.origin === 'string' ? own.origin : null, sessionId: typeof own.session_id === 'string' ? own.session_id : null, createdAt: item.createdAt, updatedAt, metadata, }; } function oneOf(value: unknown, allowed: readonly string[], fallback: string): string { return typeof value === 'string' && allowed.includes(value) ? value : fallback; } async function toChanges( results: ReadonlyArray<{ id: string }>, getRecord: (id: string) => Promise, event: MemoryChange['event'], ): Promise { const changes: MemoryChange[] = []; for (const { id } of results) { const record = await getRecord(id); if (record) changes.push({ event, record }); } return changes; } /** mem0's `langchain` embedder provider accepts any object with these two methods. */ function asLangchainEmbeddings(embedder: TextEmbedder) { return { embedQuery: async (text: string): Promise => Array.from(await embedder.embed(text)), embedDocuments: async (texts: string[]): Promise => (await embedder.embedBatch(texts)).map((vector) => Array.from(vector)), }; } /** * A deterministic bag-of-words embedder: texts that share words get similar * vectors. It needs no model and no network, so tests and offline development * use it; it does not understand meaning beyond shared words. */ export function createWordEmbedder(dimensions = 256): TextEmbedder { const embed = async (text: string): Promise => { const vector = new Array(dimensions).fill(0); for (const word of text.toLowerCase().match(/[\p{L}\p{N}]{2,}/gu) ?? []) { let hash = 2166136261; for (const char of word) hash = Math.imul(hash ^ char.charCodeAt(0), 16777619); vector[Math.abs(hash) % dimensions] += 1; } const norm = Math.hypot(...vector) || 1; return vector.map((value) => value / norm); }; return { dimensions, id: `words:${dimensions}`, embed, embedBatch: (texts) => Promise.all(texts.map(embed)), }; }