Files
waggle-os/packages/memory-mem0/src/index.ts
shumeiko.it 4a1c4c4c8e fix(memory): erase forgotten memories from mem0's history database
mem0's delete kept the deleted text in memory_history, and fact extraction
keeps the last messages of each scope in the same file. forget now purges
the memory's history rows, and forgetting a whole scope also drops the
conversation messages mem0 kept for it.
2026-10-24 12:47:46 +02:00

443 lines
17 KiB
TypeScript

/**
* A `MemoryStore` backed by the open-source mem0 library, run in-process
* (ADR-0002, ADR-0007). Everything lives in SQLite files in one directory.
*
* Mapping onto mem0: every memory belongs to user 'local'; the memory scope is
* mem0's `agent_id` ('personal' or 'workspace:<id>'), so mem0 never compares
* facts across scopes. The contract's own fields are kept in mem0 metadata.
*/
import { createHash } from 'node:crypto';
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import path from 'node:path';
import type { Memory, MemoryItem } from 'mem0ai/oss';
import {
MEMORY_IMPORTANCE,
MEMORY_KINDS,
MEMORY_SOURCES,
MemoryCapabilityError,
MemoryNotFoundError,
RESERVED_METADATA_KEYS,
assertFilter,
assertNoReservedKeys,
importanceAtLeast,
normalizeRememberInput,
scopeFromKey,
scopeKey,
type MemoryCapabilities,
type MemoryChange,
type MemoryFilter,
type MemoryHistoryEntry,
type MemoryHit,
type MemoryImportance,
type MemoryKind,
type MemoryQuery,
type MemoryRecord,
type MemoryScope,
type MemorySource,
type MemoryStore,
type MemoryUpdate,
type RememberInput,
} from '@waggle/memory-contract';
/** Turns text into vectors. Waggle passes its in-process embedder. */
export interface TextEmbedder {
readonly dimensions: number;
/**
* Names the model behind the vectors (e.g. 'inprocess:Xenova/all-MiniLM-L6-v2').
* The store remembers it and refuses to open with a different one, whose
* vectors would not be comparable. Defaults to the dimensions only.
*/
readonly id?: string;
embed(text: string): Promise<ArrayLike<number>>;
embedBatch(texts: string[]): Promise<ArrayLike<number>[]>;
}
/** An OpenAI-compatible chat endpoint, used to extract facts from conversations. */
export interface ExtractionModel {
readonly baseURL: string;
readonly apiKey: string;
/** The model name, or a function read on every extraction so a model switch applies at once. */
readonly model: string | (() => string);
}
export interface Mem0MemoryStoreOptions {
/** Directory for the SQLite files; created when missing. */
readonly directory: string;
readonly embedder: TextEmbedder;
/** Without it the store cannot extract facts (`capabilities.extraction` is false). */
readonly extractionModel?: ExtractionModel;
}
/** The memory directory's own record of how it was created. */
const STORE_INFO_FILE = 'waggle-memory.json';
/** The store was written with another embedder; its vectors cannot be compared with new ones. */
export class EmbedderMismatchError extends Error {
constructor(readonly expected: string, readonly actual: string) {
super(`This memory store was built with embedder "${expected}", not "${actual}". `
+ 'Switch back to that embedder, or move the memory directory aside to start a new store.');
this.name = 'EmbedderMismatchError';
}
}
function checkEmbedder(directory: string, embedder: TextEmbedder): void {
const actual = embedder.id ?? `dimensions:${embedder.dimensions}`;
const infoPath = path.join(directory, STORE_INFO_FILE);
if (existsSync(infoPath)) {
const info = JSON.parse(readFileSync(infoPath, 'utf8')) as { embedder?: string };
if (info.embedder && info.embedder !== actual) throw new EmbedderMismatchError(info.embedder, actual);
return;
}
writeFileSync(infoPath, JSON.stringify({ embedder: actual }, null, 2));
}
const USER_ID = 'local';
/** mem0 has no unbounded listing; this is far above a personal memory's size. */
const ALL = 1_000_000;
/** Candidates fetched per scope before the contract's filters are applied. */
const SEARCH_POOL = 100;
export async function createMem0MemoryStore(options: Mem0MemoryStoreOptions): Promise<MemoryStore> {
mkdirSync(options.directory, { recursive: true });
checkEmbedder(options.directory, options.embedder);
// mem0 reads both variables once, when its module loads. Its telemetry would
// send every call to mem0's PostHog project (ADR-0006), and MEM0_DIR keeps it
// out of ~/.mem0. That is why mem0 is imported here and not at the top.
process.env.MEM0_TELEMETRY = 'false';
process.env.MEM0_DIR = options.directory;
const { Memory: Mem0 } = await import('mem0ai/oss');
const model = options.extractionModel;
const modelName = (): string => (typeof model?.model === 'function' ? model.model() : model?.model ?? '');
const mem0: Memory = new Mem0({
version: 'v1.1',
embedder: { provider: 'langchain', config: { model: asLangchainEmbeddings(options.embedder) } },
vectorStore: {
provider: 'memory',
config: {
collectionName: 'waggle',
dimension: options.embedder.dimensions,
dbPath: path.join(options.directory, 'vectors.db'),
},
},
// mem0 requires an LLM even when nothing calls it; without a model it is never used.
llm: {
provider: 'openai',
config: model
? { baseURL: model.baseURL, apiKey: model.apiKey, model: modelName() }
: { baseURL: 'http://127.0.0.1:9/v1', apiKey: 'unused', model: 'unused' },
},
historyStore: {
provider: 'sqlite',
config: { historyDbPath: path.join(options.directory, 'history.db') },
},
});
if (typeof model?.model === 'function') {
// mem0 3.3.1 reads its LLM's `model` field on every call; a getter makes it
// follow the caller's current model. Re-check on any mem0 upgrade (ADR-0007).
const llm = (mem0 as unknown as { llm?: object }).llm;
if (!llm) throw new Error('mem0 changed its internals: no llm to configure');
Object.defineProperty(llm, 'model', { get: modelName, configurable: true });
}
const capabilities: MemoryCapabilities = {
extraction: model !== undefined,
semanticSearch: true,
history: true,
knowledgeGraph: false,
};
// Erasing a memory must erase its text. mem0's delete keeps the old text in
// its history table, and extraction keeps the last messages of each scope;
// both live in history.db, which mem0 3.3.1 holds open as `db.db`.
const historyDb = (mem0 as unknown as { db?: { db?: HistoryDatabase } }).db?.db;
if (!historyDb) throw new Error('mem0 changed its internals: no history database to purge');
const purgeHistory = historyDb.prepare('DELETE FROM memory_history WHERE memory_id = ?');
const purgeMessages = historyDb.prepare('DELETE FROM messages WHERE session_scope = ?');
let closed = false;
const ensureOpen = (): void => {
if (closed) throw new Error('memory store is closed');
};
const scopeItems = async (scope: MemoryScope): Promise<MemoryItem[]> => {
const { results } = await mem0.getAll({ topK: ALL, filters: scopeFilter(scope) });
return results;
};
const matching = async (filter: MemoryFilter): Promise<MemoryRecord[]> => {
assertFilter(filter);
const records: MemoryRecord[] = [];
for (const scope of filter.scopes) {
for (const item of await scopeItems(scope)) {
const record = toRecord(item);
if (record && passes(record, filter)) records.push(record);
}
}
return records;
};
const getRecord = async (id: string): Promise<MemoryRecord | null> => {
const item = await mem0.get(id);
return item ? toRecord(item) : null;
};
return {
capabilities,
async remember(input: RememberInput): Promise<MemoryChange[]> {
ensureOpen();
const normalized = normalizeRememberInput(input);
const metadata = {
...normalized.metadata,
kind: normalized.kind,
importance: normalized.importance,
source: normalized.source,
...(normalized.origin !== null && { origin: normalized.origin }),
...(normalized.sessionId !== null && { session_id: normalized.sessionId }),
};
const target = { userId: USER_ID, agentId: scopeKey(normalized.scope), metadata };
if (normalized.conversation) {
if (!model) throw new MemoryCapabilityError('extraction');
const messages = normalized.conversation
.filter((m) => m.content.trim())
.map((m) => ({ role: m.role, content: m.content }));
const { results } = await mem0.add(messages, { ...target, infer: true });
return toChanges(results, getRecord, 'added');
}
const text = normalized.text!;
const { results: existing } = await mem0.getAll({
topK: 1,
filters: { ...scopeFilter(normalized.scope), hash: md5(text) },
});
const duplicate = existing[0] ? toRecord(existing[0]) : null;
if (duplicate) return [{ event: 'unchanged', record: duplicate }];
const { results } = await mem0.add(text, { ...target, infer: false });
return toChanges(results, getRecord, 'added');
},
async search(text: string, query: MemoryQuery): Promise<MemoryHit[]> {
ensureOpen();
assertFilter(query);
const limit = query.limit ?? 10;
const hits: MemoryHit[] = [];
for (const scope of query.scopes) {
const { results } = await mem0.search(text, {
topK: Math.max(limit * 5, SEARCH_POOL),
threshold: 0,
filters: scopeFilter(scope),
});
for (const item of results) {
const record = toRecord(item);
if (record && passes(record, query)) hits.push({ ...record, score: item.score ?? 0 });
}
}
return hits.sort((a, b) => b.score - a.score).slice(0, limit);
},
async list(query: MemoryQuery): Promise<MemoryRecord[]> {
ensureOpen();
const records = await matching(query);
return records
.sort((a, b) => b.createdAt.localeCompare(a.createdAt))
.slice(0, query.limit ?? 50);
},
async count(filter: MemoryFilter): Promise<number> {
ensureOpen();
return (await matching(filter)).length;
},
async get(id: string): Promise<MemoryRecord | null> {
ensureOpen();
return getRecord(id);
},
async update(id: string, change: MemoryUpdate): Promise<MemoryRecord> {
ensureOpen();
if (!(await getRecord(id))) throw new MemoryNotFoundError(id);
if (change.metadata) assertNoReservedKeys(change.metadata);
await mem0.update(id, {
...(change.text?.trim() && { text: change.text.trim() }),
metadata: {
...change.metadata,
...(change.kind && { kind: change.kind }),
...(change.importance && { importance: change.importance }),
},
});
const updated = await getRecord(id);
if (!updated) throw new MemoryNotFoundError(id);
return updated;
},
async forget(id: string): Promise<boolean> {
ensureOpen();
if (!(await getRecord(id))) return false;
await mem0.delete(id);
purgeHistory.run(id);
return true;
},
async forgetMatching(filter: MemoryFilter): Promise<number> {
ensureOpen();
const doomed = await matching(filter);
for (const record of doomed) {
await mem0.delete(record.id);
purgeHistory.run(record.id);
}
if (isWholeScope(filter)) {
for (const scope of filter.scopes) purgeMessages.run(sessionScope(scope));
}
return doomed.length;
},
async history(id: string): Promise<MemoryHistoryEntry[]> {
ensureOpen();
const rows = (await mem0.history(id)) as HistoryRow[];
return rows
.map(toHistoryEntry)
.filter((entry): entry is MemoryHistoryEntry => entry !== null)
.sort((a, b) => a.at.localeCompare(b.at));
},
async close(): Promise<void> {
// mem0 keeps its SQLite handles open for the life of the process.
closed = true;
},
};
}
interface HistoryRow {
action?: string;
previous_value?: string | null;
new_value?: string | null;
created_at?: string | null;
updated_at?: string | null;
}
const HISTORY_EVENTS: Record<string, MemoryHistoryEntry['event']> = {
ADD: 'added',
UPDATE: 'updated',
DELETE: 'deleted',
};
function toHistoryEntry(row: HistoryRow): MemoryHistoryEntry | null {
const event = HISTORY_EVENTS[row.action ?? ''];
const at = row.updated_at ?? row.created_at;
if (!event || !at) return null;
return { event, previousText: row.previous_value ?? null, text: row.new_value ?? null, at };
}
/** The part of better-sqlite3 the store uses on mem0's history database. */
interface HistoryDatabase {
prepare(sql: string): { run(...params: unknown[]): unknown };
}
/** A filter that selects everything in its scopes. */
function isWholeScope(filter: MemoryFilter): boolean {
return !filter.kinds && !filter.minImportance && filter.origin === undefined
&& filter.sessionId === undefined && filter.since === undefined && filter.until === undefined;
}
/** mem0's key for the conversation messages it keeps per scope (agent_id, user_id, sorted). */
function sessionScope(scope: MemoryScope): string {
const escape = (value: string): string =>
value.replace(/%/g, '%25').replace(/&/g, '%26').replace(/=/g, '%3D');
return `agent_id=${escape(scopeKey(scope))}&user_id=${escape(USER_ID)}`;
}
function scopeFilter(scope: MemoryScope): { user_id: string; agent_id: string } {
return { user_id: USER_ID, agent_id: scopeKey(scope) };
}
/** mem0 hashes memory text with md5; the same hash finds an identical memory. */
function md5(text: string): string {
return createHash('md5').update(text).digest('hex');
}
function passes(record: MemoryRecord, filter: MemoryFilter): boolean {
const importance = filter.minImportance ? importanceAtLeast(filter.minImportance) : null;
return (!filter.kinds || filter.kinds.includes(record.kind))
&& (!importance || importance.includes(record.importance))
&& (filter.origin === undefined || record.origin === filter.origin)
&& (filter.sessionId === undefined || record.sessionId === filter.sessionId)
&& (filter.since === undefined || record.createdAt >= filter.since)
&& (filter.until === undefined || record.createdAt <= filter.until);
}
/** Converts a mem0 item, or returns null for an item this store did not write. */
function toRecord(item: MemoryItem & { user_id?: string; agent_id?: string }): MemoryRecord | null {
const scope = item.agent_id ? scopeFromKey(item.agent_id) : null;
if (!scope || item.user_id !== USER_ID || !item.createdAt) return null;
const metadata: Record<string, unknown> = { ...item.metadata };
const own = Object.fromEntries(RESERVED_METADATA_KEYS.map((key) => [key, metadata[key]]));
for (const key of RESERVED_METADATA_KEYS) delete metadata[key];
const updatedAt = item.updatedAt && item.updatedAt !== item.createdAt ? item.updatedAt : null;
return {
id: item.id,
text: item.memory,
scope,
kind: oneOf(own.kind, MEMORY_KINDS, 'fact') as MemoryKind,
importance: oneOf(own.importance, MEMORY_IMPORTANCE, 'normal') as MemoryImportance,
source: oneOf(own.source, MEMORY_SOURCES, 'agent_inferred') as MemorySource,
origin: typeof own.origin === 'string' ? own.origin : null,
sessionId: typeof own.session_id === 'string' ? own.session_id : null,
createdAt: item.createdAt,
updatedAt,
metadata,
};
}
function oneOf(value: unknown, allowed: readonly string[], fallback: string): string {
return typeof value === 'string' && allowed.includes(value) ? value : fallback;
}
async function toChanges(
results: ReadonlyArray<{ id: string }>,
getRecord: (id: string) => Promise<MemoryRecord | null>,
event: MemoryChange['event'],
): Promise<MemoryChange[]> {
const changes: MemoryChange[] = [];
for (const { id } of results) {
const record = await getRecord(id);
if (record) changes.push({ event, record });
}
return changes;
}
/** mem0's `langchain` embedder provider accepts any object with these two methods. */
function asLangchainEmbeddings(embedder: TextEmbedder) {
return {
embedQuery: async (text: string): Promise<number[]> => Array.from(await embedder.embed(text)),
embedDocuments: async (texts: string[]): Promise<number[][]> =>
(await embedder.embedBatch(texts)).map((vector) => Array.from(vector)),
};
}
/**
* A deterministic bag-of-words embedder: texts that share words get similar
* vectors. It needs no model and no network, so tests and offline development
* use it; it does not understand meaning beyond shared words.
*/
export function createWordEmbedder(dimensions = 256): TextEmbedder {
const embed = async (text: string): Promise<number[]> => {
const vector = new Array<number>(dimensions).fill(0);
for (const word of text.toLowerCase().match(/[\p{L}\p{N}]{2,}/gu) ?? []) {
let hash = 2166136261;
for (const char of word) hash = Math.imul(hash ^ char.charCodeAt(0), 16777619);
vector[Math.abs(hash) % dimensions] += 1;
}
const norm = Math.hypot(...vector) || 1;
return vector.map((value) => value / norm);
};
return {
dimensions,
id: `words:${dimensions}`,
embed,
embedBatch: (texts) => Promise.all(texts.map(embed)),
};
}