Files
waggle-os/packages/server/src/local/legacy-memory-import.ts
shumeiko.it 784ca770b1 refactor(memory): run agent recall, memory tools and the server on the memory contract (P1.5)
- Orchestrator, memory tools, context loader and pattern write-back use a
  MemoryStore and a workspace scope instead of MindDB layers; the knowledge
  graph tools, cognify, memory linker and feedback handler are gone (graph
  hidden in v1, ADR-0002).
- The server opens one mem0 store in <dataDir>/memory/, embedded by the
  in-process model on first use, and extracts facts about the user in the
  background with the model the user chose (memoryExtraction setting).
- Legacy .mind memories are copied into the store once, in the background;
  the legacy files are only read.
- Weaver consolidation, harvest, wiki, memory import, memory lanes, vector
  backfill and index reconciliation are removed; agent runs, loops, sub-agent
  results, connector fetch and the failed-turn capture write through the
  contract. Workspace sessions no longer hold a mind handle.
- FileIndexer and the install audit move to waggle.db; recall date helpers and
  the reranker are copied into core.
- The CLI REPL opens the same memory store.
2026-10-24 11:56:26 +02:00

174 lines
5.8 KiB
TypeScript

/**
* One-time copy of memories from the legacy `.mind` files into the memory
* store (P1.5). The legacy files are only read: the data directory may be
* shared with the commercial edition. Each scope is copied once; progress is
* kept in waggle.db, so an interrupted copy resumes where it stopped, and the
* store does not write identical text twice in one scope.
*/
import fs from 'node:fs';
import Database from 'better-sqlite3';
import {
MEMORY_IMPORTANCE,
MEMORY_SOURCES,
scopeKey,
type MemoryImportance,
type MemoryKind,
type MemoryScope,
type MemorySource,
type MemoryStore,
} from '@waggle/memory-contract';
import type { WaggleDB } from '@waggle/core';
export interface LegacyMemorySource {
/** A legacy `.mind` file. */
path: string;
scope: MemoryScope;
}
export interface LegacyMemoryImportReport {
scope: string;
status: 'imported' | 'already-imported' | 'missing' | 'failed';
copied: number;
skipped: number;
error?: string;
}
interface LegacyFrameRow {
id: number;
content: string;
importance: string;
source: string | null;
created_at: string | null;
}
/** Content prefixes the legacy engine used as types, mapped to contract kinds. */
const PREFIX_KINDS: ReadonlyArray<[RegExp, MemoryKind, boolean]> = [
// [pattern, kind, strip the first line (a header)]
[/^\[mind-profile [^\]]*\]/, 'profile', true],
[/^\[mind-fact\]/, 'fact', true],
[/^\[mind-event\]/, 'event', true],
[/^(User preference:|Style note:)/, 'preference', false],
[/^Decision:/, 'decision', false],
[/^(\[Session summary|Research findings:)/, 'summary', false],
[/^(\[Loop:|\[Sub-agent:|User asked:|Work completed:|Recommendation:)/, 'note', false],
];
/** Raw dialogue turns were an engine-specific recall lane; they are not memories. */
const SKIPPED_PREFIX = /^\[mind-rawturn /;
const metaKey = (scope: MemoryScope): string => `memory-import:${scopeKey(scope)}`;
const BATCH = 200;
export async function importLegacyMemories(
memory: MemoryStore,
waggleDb: WaggleDB,
sources: readonly LegacyMemorySource[],
onProgress?: (report: LegacyMemoryImportReport) => void,
isCancelled: () => boolean = () => false,
): Promise<LegacyMemoryImportReport[]> {
const reports: LegacyMemoryImportReport[] = [];
for (const source of sources) {
if (isCancelled()) break;
const report = await importOne(memory, waggleDb, source, isCancelled);
reports.push(report);
onProgress?.(report);
}
return reports;
}
async function importOne(
memory: MemoryStore,
waggleDb: WaggleDB,
source: LegacyMemorySource,
isCancelled: () => boolean,
): Promise<LegacyMemoryImportReport> {
const scope = scopeKey(source.scope);
const report: LegacyMemoryImportReport = { scope, status: 'imported', copied: 0, skipped: 0 };
const state = waggleDb.getMeta(metaKey(source.scope));
if (state === 'done') return { ...report, status: 'already-imported' };
if (!fs.existsSync(source.path) || fs.statSync(source.path).size === 0) {
waggleDb.setMeta(metaKey(source.scope), 'done');
return { ...report, status: 'missing' };
}
let db: Database.Database | null = null;
try {
db = new Database(source.path, { readonly: true, fileMustExist: true });
const hasFrames = db.prepare(
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'memory_frames'",
).get();
if (!hasFrames) {
waggleDb.setMeta(metaKey(source.scope), 'done');
return { ...report, status: 'missing' };
}
let lastId = state ? Number(state) || 0 : 0;
const page = db.prepare(
`SELECT id, content, importance, source, created_at FROM memory_frames
WHERE id > ? ORDER BY id LIMIT ?`,
);
for (;;) {
const rows = page.all(lastId, BATCH) as LegacyFrameRow[];
if (rows.length === 0) break;
for (const row of rows) {
if (isCancelled()) {
waggleDb.setMeta(metaKey(source.scope), String(lastId));
return report;
}
const mapped = mapFrame(row);
if (mapped) {
await memory.remember({
scope: source.scope,
text: mapped.text,
kind: mapped.kind,
importance: mapped.importance,
source: mapped.source,
origin: 'legacy-import',
...(row.created_at && { metadata: { legacy_created_at: row.created_at } }),
});
report.copied++;
} else {
report.skipped++;
}
lastId = row.id;
}
waggleDb.setMeta(metaKey(source.scope), String(lastId));
}
waggleDb.setMeta(metaKey(source.scope), 'done');
return report;
} catch (err) {
return { ...report, status: 'failed', error: err instanceof Error ? err.message : String(err) };
} finally {
db?.close();
}
}
/** Converts a legacy frame, or returns null for one that is not carried over. */
export function mapFrame(row: Pick<LegacyFrameRow, 'content' | 'importance' | 'source'>): {
text: string;
kind: MemoryKind;
importance: MemoryImportance;
source: MemorySource;
} | null {
const content = row.content?.trim() ?? '';
if (!content || row.importance === 'deprecated' || SKIPPED_PREFIX.test(content)) return null;
let kind: MemoryKind = 'fact';
let text = content;
for (const [pattern, mappedKind, stripHeader] of PREFIX_KINDS) {
if (!pattern.test(content)) continue;
kind = mappedKind;
if (stripHeader) {
const newline = content.indexOf('\n');
text = newline >= 0 ? content.slice(newline + 1).trim() : content.replace(pattern, '').trim();
}
break;
}
if (!text) return null;
const importance = (MEMORY_IMPORTANCE as readonly string[]).includes(row.importance)
? row.importance as MemoryImportance
: 'normal';
const source = row.source && (MEMORY_SOURCES as readonly string[]).includes(row.source)
? row.source as MemorySource
: 'import';
return { text, kind, importance, source };
}