refactor: drop the one-time copy of legacy .mind data
The first start copied Waggle's tables and the memories of the commercial edition's .mind files into waggle.db and mem0. The product is still in development and nobody has data to bring over, so both copies, their tests and the legacy creation-time field are gone. ADR-0007 records the change.
This commit is contained in:
@@ -10,7 +10,7 @@ import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify';
|
||||
import cors from '@fastify/cors';
|
||||
import fastifyStatic from '@fastify/static';
|
||||
import websocket from '@fastify/websocket';
|
||||
import { WaggleDB, importLegacyMindData, workspaceDataScope, type LegacyMindSource, WorkspaceManager, WaggleConfig, createEmbeddingProvider, type EmbeddingProviderConfig, type EmbeddingProviderInstance, CronStore, AwarenessLayer, VaultStore, SkillHashStore, TelemetryStore, TELEMETRY_EVENTS, ExecutionTraceStore } from '@waggle/core';
|
||||
import { WaggleDB, workspaceDataScope, WorkspaceManager, WaggleConfig, createEmbeddingProvider, type EmbeddingProviderConfig, type EmbeddingProviderInstance, CronStore, AwarenessLayer, VaultStore, SkillHashStore, TelemetryStore, TELEMETRY_EVENTS, ExecutionTraceStore } from '@waggle/core';
|
||||
import { corsOriginAllowed } from './cors-config.js';
|
||||
import { InstallAuditStore } from './install-audit.js';
|
||||
import { selfBaseUrl, selfRequestHeaders } from './self-request.js';
|
||||
@@ -68,7 +68,6 @@ import type { RiskLevel } from '@waggle/shared';
|
||||
import { runConnectorFetch } from './connector-harvest.js';
|
||||
import { openMemoryStore } from './memory-runtime.js';
|
||||
import { MemoryExtractionQueue } from './memory-extraction.js';
|
||||
import { importLegacyMemories, type LegacyMemorySource } from './legacy-memory-import.js';
|
||||
import { PERSONAL_SCOPE, workspaceScope, type MemoryStore } from '@waggle/memory-contract';
|
||||
import { runLoopTick } from './loop-executor.js';
|
||||
import { enqueueHeldAction } from './held-action-executor.js';
|
||||
@@ -486,36 +485,14 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
const defaultWorkspaceId = defaultWorkspaceConfig.id;
|
||||
server.decorate('workspaceManager', wsManager);
|
||||
|
||||
// The legacy memory file of earlier versions; only the one-time copies of
|
||||
// legacy data below read it, and nothing creates it any more.
|
||||
const personalPath = path.join(fullConfig.dataDir, 'personal.mind');
|
||||
|
||||
// Waggle's own data (automations, notifications, cost tracking, profile and
|
||||
// awareness) lives in waggle.db, apart from memory (ADR-0007). The first
|
||||
// start copies it from the legacy .mind files, which stay untouched.
|
||||
// awareness) lives in waggle.db, apart from memory (ADR-0007).
|
||||
const waggleDb = WaggleDB.openInDataDir(fullConfig.dataDir);
|
||||
server.decorate('waggleDb', waggleDb);
|
||||
const legacySources: LegacyMindSource[] = [{ path: personalPath, scope: 'personal' }];
|
||||
for (const ws of wsManager.list()) {
|
||||
try {
|
||||
legacySources.push({ path: wsManager.getMindPath(ws.id), scope: workspaceDataScope(ws.id) });
|
||||
} catch {
|
||||
// A workspace whose directory fails the path checks has nothing to copy.
|
||||
}
|
||||
}
|
||||
// Install audit store — persistent trail for capability install events. Its
|
||||
// table must exist before the legacy copy below, which fills it.
|
||||
// Install audit store — persistent trail for capability install events.
|
||||
const auditStore = new InstallAuditStore(waggleDb);
|
||||
server.decorate('auditStore', auditStore);
|
||||
|
||||
for (const report of importLegacyMindData(waggleDb, legacySources)) {
|
||||
if (report.status === 'imported') {
|
||||
log.info(`[waggle-db] copied legacy data for ${report.scope}: ${JSON.stringify(report.copied)}`);
|
||||
} else if (report.status === 'failed') {
|
||||
log.warn(`[waggle-db] could not copy legacy data for ${report.scope}: ${report.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Cron store — SQLite-backed schedule persistence for Solo
|
||||
const cronStore = new CronStore(waggleDb);
|
||||
server.decorate('cronStore', cronStore);
|
||||
@@ -2479,10 +2456,8 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
// Stop MCP servers
|
||||
await mcpRuntime.stopAll().catch(() => {});
|
||||
|
||||
// Stop collecting chat turns for fact extraction and the legacy memory copy.
|
||||
// Stop collecting chat turns for fact extraction.
|
||||
memoryExtraction.stop();
|
||||
legacyMemoryImport.cancel();
|
||||
await legacyMemoryImport.done;
|
||||
|
||||
// Close all workspace sessions (new concurrent model)
|
||||
sessionManager.closeAll();
|
||||
@@ -2500,31 +2475,5 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
}
|
||||
});
|
||||
|
||||
// One-time copy of memories from the legacy .mind files into the memory
|
||||
// store, in the background; the legacy files are only read.
|
||||
const legacyMemoryImport = (() => {
|
||||
let cancelled = false;
|
||||
const sources: LegacyMemorySource[] = [{ path: personalPath, scope: PERSONAL_SCOPE }];
|
||||
for (const ws of wsManager.list()) {
|
||||
try {
|
||||
sources.push({ path: wsManager.getMindPath(ws.id), scope: workspaceScope(ws.id) });
|
||||
} catch {
|
||||
// A workspace whose directory fails the path checks has nothing to copy.
|
||||
}
|
||||
}
|
||||
const done = new Promise<void>(resolve => setImmediate(resolve))
|
||||
.then(() => importLegacyMemories(memory, waggleDb, sources, (report) => {
|
||||
if (report.status === 'imported') {
|
||||
log.info(`[memory] copied ${report.copied} legacy memories for ${report.scope} (${report.skipped} skipped)`);
|
||||
} else if (report.status === 'failed') {
|
||||
log.warn(`[memory] could not copy legacy memories for ${report.scope}: ${report.error}`);
|
||||
}
|
||||
}, () => cancelled))
|
||||
.then(() => undefined, (err: unknown) => {
|
||||
log.warn(`[memory] legacy memory copy stopped: ${err instanceof Error ? err.message : String(err)}`);
|
||||
});
|
||||
return { done, cancel: () => { cancelled = true; } };
|
||||
})();
|
||||
|
||||
return server;
|
||||
}
|
||||
|
||||
@@ -1,173 +0,0 @@
|
||||
/**
|
||||
* One-time copy of memories from the legacy `.mind` files into the memory
|
||||
* store (P1.5). The legacy files are only read: the data directory may be
|
||||
* shared with the commercial edition. Each scope is copied once; progress is
|
||||
* kept in waggle.db, so an interrupted copy resumes where it stopped, and the
|
||||
* store does not write identical text twice in one scope.
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import Database from 'better-sqlite3';
|
||||
import {
|
||||
MEMORY_IMPORTANCE,
|
||||
MEMORY_SOURCES,
|
||||
scopeKey,
|
||||
type MemoryImportance,
|
||||
type MemoryKind,
|
||||
type MemoryScope,
|
||||
type MemorySource,
|
||||
type MemoryStore,
|
||||
} from '@waggle/memory-contract';
|
||||
import type { WaggleDB } from '@waggle/core';
|
||||
|
||||
export interface LegacyMemorySource {
|
||||
/** A legacy `.mind` file. */
|
||||
path: string;
|
||||
scope: MemoryScope;
|
||||
}
|
||||
|
||||
export interface LegacyMemoryImportReport {
|
||||
scope: string;
|
||||
status: 'imported' | 'already-imported' | 'missing' | 'failed';
|
||||
copied: number;
|
||||
skipped: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
interface LegacyFrameRow {
|
||||
id: number;
|
||||
content: string;
|
||||
importance: string;
|
||||
source: string | null;
|
||||
created_at: string | null;
|
||||
}
|
||||
|
||||
/** Content prefixes the legacy engine used as types, mapped to contract kinds. */
|
||||
const PREFIX_KINDS: ReadonlyArray<[RegExp, MemoryKind, boolean]> = [
|
||||
// [pattern, kind, strip the first line (a header)]
|
||||
[/^\[mind-profile [^\]]*\]/, 'profile', true],
|
||||
[/^\[mind-fact\]/, 'fact', true],
|
||||
[/^\[mind-event\]/, 'event', true],
|
||||
[/^(User preference:|Style note:)/, 'preference', false],
|
||||
[/^Decision:/, 'decision', false],
|
||||
[/^(\[Session summary|Research findings:)/, 'summary', false],
|
||||
[/^(\[Loop:|\[Sub-agent:|User asked:|Work completed:|Recommendation:)/, 'note', false],
|
||||
];
|
||||
|
||||
/** Raw dialogue turns were an engine-specific recall lane; they are not memories. */
|
||||
const SKIPPED_PREFIX = /^\[mind-rawturn /;
|
||||
|
||||
const metaKey = (scope: MemoryScope): string => `memory-import:${scopeKey(scope)}`;
|
||||
const BATCH = 200;
|
||||
|
||||
export async function importLegacyMemories(
|
||||
memory: MemoryStore,
|
||||
waggleDb: WaggleDB,
|
||||
sources: readonly LegacyMemorySource[],
|
||||
onProgress?: (report: LegacyMemoryImportReport) => void,
|
||||
isCancelled: () => boolean = () => false,
|
||||
): Promise<LegacyMemoryImportReport[]> {
|
||||
const reports: LegacyMemoryImportReport[] = [];
|
||||
for (const source of sources) {
|
||||
if (isCancelled()) break;
|
||||
const report = await importOne(memory, waggleDb, source, isCancelled);
|
||||
reports.push(report);
|
||||
onProgress?.(report);
|
||||
}
|
||||
return reports;
|
||||
}
|
||||
|
||||
async function importOne(
|
||||
memory: MemoryStore,
|
||||
waggleDb: WaggleDB,
|
||||
source: LegacyMemorySource,
|
||||
isCancelled: () => boolean,
|
||||
): Promise<LegacyMemoryImportReport> {
|
||||
const scope = scopeKey(source.scope);
|
||||
const report: LegacyMemoryImportReport = { scope, status: 'imported', copied: 0, skipped: 0 };
|
||||
const state = waggleDb.getMeta(metaKey(source.scope));
|
||||
if (state === 'done') return { ...report, status: 'already-imported' };
|
||||
if (!fs.existsSync(source.path) || fs.statSync(source.path).size === 0) {
|
||||
waggleDb.setMeta(metaKey(source.scope), 'done');
|
||||
return { ...report, status: 'missing' };
|
||||
}
|
||||
|
||||
let db: Database.Database | null = null;
|
||||
try {
|
||||
db = new Database(source.path, { readonly: true, fileMustExist: true });
|
||||
const hasFrames = db.prepare(
|
||||
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'memory_frames'",
|
||||
).get();
|
||||
if (!hasFrames) {
|
||||
waggleDb.setMeta(metaKey(source.scope), 'done');
|
||||
return { ...report, status: 'missing' };
|
||||
}
|
||||
let lastId = state ? Number(state) || 0 : 0;
|
||||
const page = db.prepare(
|
||||
`SELECT id, content, importance, source, created_at FROM memory_frames
|
||||
WHERE id > ? ORDER BY id LIMIT ?`,
|
||||
);
|
||||
for (;;) {
|
||||
const rows = page.all(lastId, BATCH) as LegacyFrameRow[];
|
||||
if (rows.length === 0) break;
|
||||
for (const row of rows) {
|
||||
if (isCancelled()) {
|
||||
waggleDb.setMeta(metaKey(source.scope), String(lastId));
|
||||
return report;
|
||||
}
|
||||
const mapped = mapFrame(row);
|
||||
if (mapped) {
|
||||
await memory.remember({
|
||||
scope: source.scope,
|
||||
text: mapped.text,
|
||||
kind: mapped.kind,
|
||||
importance: mapped.importance,
|
||||
source: mapped.source,
|
||||
origin: 'legacy-import',
|
||||
...(row.created_at && { metadata: { legacy_created_at: row.created_at } }),
|
||||
});
|
||||
report.copied++;
|
||||
} else {
|
||||
report.skipped++;
|
||||
}
|
||||
lastId = row.id;
|
||||
}
|
||||
waggleDb.setMeta(metaKey(source.scope), String(lastId));
|
||||
}
|
||||
waggleDb.setMeta(metaKey(source.scope), 'done');
|
||||
return report;
|
||||
} catch (err) {
|
||||
return { ...report, status: 'failed', error: err instanceof Error ? err.message : String(err) };
|
||||
} finally {
|
||||
db?.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** Converts a legacy frame, or returns null for one that is not carried over. */
|
||||
export function mapFrame(row: Pick<LegacyFrameRow, 'content' | 'importance' | 'source'>): {
|
||||
text: string;
|
||||
kind: MemoryKind;
|
||||
importance: MemoryImportance;
|
||||
source: MemorySource;
|
||||
} | null {
|
||||
const content = row.content?.trim() ?? '';
|
||||
if (!content || row.importance === 'deprecated' || SKIPPED_PREFIX.test(content)) return null;
|
||||
let kind: MemoryKind = 'fact';
|
||||
let text = content;
|
||||
for (const [pattern, mappedKind, stripHeader] of PREFIX_KINDS) {
|
||||
if (!pattern.test(content)) continue;
|
||||
kind = mappedKind;
|
||||
if (stripHeader) {
|
||||
const newline = content.indexOf('\n');
|
||||
text = newline >= 0 ? content.slice(newline + 1).trim() : content.replace(pattern, '').trim();
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (!text) return null;
|
||||
const importance = (MEMORY_IMPORTANCE as readonly string[]).includes(row.importance)
|
||||
? row.importance as MemoryImportance
|
||||
: 'normal';
|
||||
const source = row.source && (MEMORY_SOURCES as readonly string[]).includes(row.source)
|
||||
? row.source as MemorySource
|
||||
: 'import';
|
||||
return { text, kind, importance, source };
|
||||
}
|
||||
@@ -7,7 +7,7 @@
|
||||
* importance it had.
|
||||
* - `confirmedAt` (ISO 8601): the user confirmed a memory extracted in the background.
|
||||
* - `title`, `tags`, `evidence`, `relatedMemoryIds`, `mergedIntoId`.
|
||||
* Chat write-back stores `trace_id`; the legacy import stores `legacy_created_at`.
|
||||
* Chat write-back stores `trace_id`.
|
||||
*/
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
import {
|
||||
@@ -76,18 +76,3 @@ export function memoryLifecycle(record: MemoryRecord): MemoryLifecycle {
|
||||
if (record.metadata.archived === true) return 'archived';
|
||||
return awaitsReview(record) ? 'unreviewed' : 'active';
|
||||
}
|
||||
|
||||
/**
|
||||
* When the memory was first written. Memories copied from the legacy engine
|
||||
* keep their original time in metadata; the store's own time is the copy time.
|
||||
*/
|
||||
export function memoryCreatedAt(record: MemoryRecord): string {
|
||||
const legacy = record.metadata.legacy_created_at;
|
||||
if (typeof legacy === 'string') {
|
||||
// SQLite's datetime('now') is UTC without a zone: "YYYY-MM-DD HH:MM:SS".
|
||||
const iso = /^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d$/.test(legacy) ? `${legacy.replace(' ', 'T')}Z` : legacy;
|
||||
const time = Date.parse(iso);
|
||||
if (Number.isFinite(time)) return new Date(time).toISOString();
|
||||
}
|
||||
return record.createdAt;
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ import {
|
||||
asImportance,
|
||||
asKind,
|
||||
awaitsReview,
|
||||
memoryCreatedAt,
|
||||
memoryLifecycle,
|
||||
workspaceIdOf,
|
||||
writeScopeForWorkspace,
|
||||
@@ -100,7 +99,7 @@ export function recordToMemory(record: MemoryRecord): Memory {
|
||||
tags: stringArray(meta.tags),
|
||||
relatedMemoryIds: stringArray(meta.relatedMemoryIds),
|
||||
status: memoryLifecycle(record),
|
||||
createdAt: memoryCreatedAt(record),
|
||||
createdAt: record.createdAt,
|
||||
...(record.updatedAt && { updatedAt: record.updatedAt }),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -22,7 +22,6 @@ import {
|
||||
allScopes,
|
||||
asImportance,
|
||||
asKind,
|
||||
memoryCreatedAt,
|
||||
memoryLifecycle,
|
||||
scopeForWorkspace,
|
||||
workspaceIdOf,
|
||||
@@ -193,7 +192,7 @@ export function toFrameResponse(record: MemoryRecord | MemoryHit, workspaceName?
|
||||
mind: record.scope.kind,
|
||||
workspaceId: workspaceIdOf(record) ?? null,
|
||||
status: memoryLifecycle(record),
|
||||
timestamp: memoryCreatedAt(record),
|
||||
timestamp: record.createdAt,
|
||||
...('score' in record ? { score: record.score } : {}),
|
||||
...(workspaceName ? { workspaceName } : {}),
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user