refactor: drop the one-time copy of legacy .mind data
The first start copied Waggle's tables and the memories of the commercial edition's .mind files into waggle.db and mem0. The product is still in development and nobody has data to bring over, so both copies, their tests and the legacy creation-time field are gone. ADR-0007 records the change.
This commit is contained in:
@@ -52,7 +52,6 @@ export {
|
||||
type TracePayload, type TraceToolCall, type TraceReasoningStep,
|
||||
type StartTraceInput, type FinalizeTraceInput, type TraceQueryFilter,
|
||||
} from './execution-traces.js';
|
||||
export { importLegacyMindData, type LegacyMindSource, type LegacyImportReport } from './waggle-db-import.js';
|
||||
|
||||
// ── Waggle-specific orchestration ──
|
||||
export { WaggleConfig, type ProviderEntry } from './config.js';
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
/**
|
||||
* One-time copy of Waggle's own tables from the legacy memory databases
|
||||
* (personal.mind, workspace.mind) into waggle.db.
|
||||
*
|
||||
* The legacy files are only read: the data directory may be shared with the
|
||||
* commercial edition, which keeps using them. Each source is copied once,
|
||||
* recorded in waggle_meta, and a table is copied only while its target is
|
||||
* empty, so a re-run never duplicates rows.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import Database from 'better-sqlite3';
|
||||
import type { Database as DatabaseType } from 'better-sqlite3';
|
||||
import { AwarenessLayer } from './awareness.js';
|
||||
import { CronStore } from './cron-store.js';
|
||||
import { ExecutionTraceStore } from './execution-traces.js';
|
||||
import { IdentityLayer } from './identity.js';
|
||||
import { SkillHashStore } from './skill-hashes.js';
|
||||
import type { DataScope, WaggleDB } from './waggle-db.js';
|
||||
|
||||
/** Tables that only ever lived in personal.mind, copied with their ids. */
|
||||
const PERSONAL_TABLES = [
|
||||
'cron_schedules',
|
||||
'cron_execution_history',
|
||||
'cron_run_leases',
|
||||
'notifications',
|
||||
'pending_actions',
|
||||
'skill_hashes',
|
||||
'feedback_entries',
|
||||
'execution_traces',
|
||||
'execution_trace_spend',
|
||||
'execution_trace_spend_reservations',
|
||||
'install_audit',
|
||||
] as const;
|
||||
|
||||
const PROFILE_COLUMNS = ['name', 'role', 'department', 'personality', 'capabilities', 'system_prompt', 'created_at', 'updated_at'];
|
||||
const AWARENESS_COLUMNS = ['category', 'content', 'priority', 'metadata', 'created_at', 'expires_at'];
|
||||
|
||||
export interface LegacyMindSource {
|
||||
/** Path to a legacy .mind file. */
|
||||
path: string;
|
||||
scope: DataScope;
|
||||
}
|
||||
|
||||
export interface LegacyImportReport {
|
||||
scope: DataScope;
|
||||
status: 'imported' | 'already-imported' | 'missing' | 'failed';
|
||||
copied: Record<string, number>;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
function importKey(scope: DataScope): string {
|
||||
return `import:${scope}`;
|
||||
}
|
||||
|
||||
function tableExists(db: DatabaseType, table: string): boolean {
|
||||
return db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(table) !== undefined;
|
||||
}
|
||||
|
||||
function columnsOf(db: DatabaseType, table: string): string[] {
|
||||
return (db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map(c => c.name);
|
||||
}
|
||||
|
||||
function copyRows(
|
||||
source: DatabaseType,
|
||||
target: DatabaseType,
|
||||
sourceTable: string,
|
||||
targetTable: string,
|
||||
columns: string[],
|
||||
extra: Record<string, unknown> = {},
|
||||
): number {
|
||||
const rows = source.prepare(`SELECT ${columns.join(', ')} FROM ${sourceTable}`).all() as Array<Record<string, unknown>>;
|
||||
if (rows.length === 0) return 0;
|
||||
const allColumns = [...columns, ...Object.keys(extra)];
|
||||
const insert = target.prepare(
|
||||
`INSERT INTO ${targetTable} (${allColumns.join(', ')}) VALUES (${allColumns.map(() => '?').join(', ')})`,
|
||||
);
|
||||
for (const row of rows) {
|
||||
insert.run(...columns.map(c => row[c]), ...Object.values(extra));
|
||||
}
|
||||
return rows.length;
|
||||
}
|
||||
|
||||
function copyPersonalTables(source: DatabaseType, target: DatabaseType, copied: Record<string, number>): void {
|
||||
for (const table of PERSONAL_TABLES) {
|
||||
if (!tableExists(source, table)) continue;
|
||||
if (!tableExists(target, table)) {
|
||||
// Lazily created tables (feedback_entries) take the legacy definition.
|
||||
const ddl = source.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(table) as { sql: string };
|
||||
target.exec(ddl.sql);
|
||||
}
|
||||
const empty = target.prepare(`SELECT 1 FROM ${table} LIMIT 1`).get() === undefined;
|
||||
if (!empty) continue;
|
||||
const targetColumns = new Set(columnsOf(target, table));
|
||||
const shared = columnsOf(source, table).filter(c => targetColumns.has(c));
|
||||
copied[table] = copyRows(source, target, table, table, shared);
|
||||
}
|
||||
}
|
||||
|
||||
function copyScopedTables(source: DatabaseType, target: DatabaseType, scope: DataScope, copied: Record<string, number>): void {
|
||||
if (tableExists(source, 'identity')) {
|
||||
const hasProfile = target.prepare('SELECT 1 FROM profiles WHERE scope = ?').get(scope) !== undefined;
|
||||
if (!hasProfile) {
|
||||
const sourceColumns = new Set(columnsOf(source, 'identity'));
|
||||
const shared = PROFILE_COLUMNS.filter(c => sourceColumns.has(c));
|
||||
copied.profiles = copyRows(source, target, 'identity', 'profiles', shared, { scope });
|
||||
}
|
||||
}
|
||||
if (tableExists(source, 'awareness')) {
|
||||
const hasItems = target.prepare('SELECT 1 FROM awareness_items WHERE scope = ? LIMIT 1').get(scope) !== undefined;
|
||||
if (!hasItems) {
|
||||
const sourceColumns = new Set(columnsOf(source, 'awareness'));
|
||||
const shared = AWARENESS_COLUMNS.filter(c => sourceColumns.has(c));
|
||||
copied.awareness_items = copyRows(source, target, 'awareness', 'awareness_items', shared, { scope });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies Waggle data from each legacy source into `waggleDb`, once per scope.
|
||||
* The personal source also brings the app-wide tables; workspace sources bring
|
||||
* their profile and awareness items.
|
||||
*
|
||||
* Tables owned outside core (install_audit, owned by the server) must be created
|
||||
* by their owner before this call; otherwise they take the legacy definition.
|
||||
*/
|
||||
export function importLegacyMindData(waggleDb: WaggleDB, sources: LegacyMindSource[]): LegacyImportReport[] {
|
||||
// Create every core target table with its current schema first.
|
||||
new CronStore(waggleDb);
|
||||
new SkillHashStore(waggleDb);
|
||||
new ExecutionTraceStore(waggleDb);
|
||||
new IdentityLayer(waggleDb);
|
||||
new AwarenessLayer(waggleDb);
|
||||
|
||||
const target = waggleDb.getDatabase();
|
||||
const reports: LegacyImportReport[] = [];
|
||||
for (const { path: sourcePath, scope } of sources) {
|
||||
const copied: Record<string, number> = {};
|
||||
if (waggleDb.getMeta(importKey(scope))) {
|
||||
reports.push({ scope, status: 'already-imported', copied });
|
||||
continue;
|
||||
}
|
||||
const stat = fs.statSync(sourcePath, { throwIfNoEntry: false });
|
||||
if (!stat || stat.size === 0) {
|
||||
reports.push({ scope, status: 'missing', copied });
|
||||
continue;
|
||||
}
|
||||
let source: DatabaseType | null = null;
|
||||
try {
|
||||
source = new Database(sourcePath, { fileMustExist: true });
|
||||
const db = source;
|
||||
target.transaction(() => {
|
||||
if (scope === 'personal') copyPersonalTables(db, target, copied);
|
||||
copyScopedTables(db, target, scope, copied);
|
||||
})();
|
||||
waggleDb.setMeta(importKey(scope), new Date().toISOString());
|
||||
reports.push({ scope, status: 'imported', copied });
|
||||
} catch (err) {
|
||||
reports.push({ scope, status: 'failed', copied: {}, error: err instanceof Error ? err.message : String(err) });
|
||||
} finally {
|
||||
source?.close();
|
||||
}
|
||||
}
|
||||
return reports;
|
||||
}
|
||||
@@ -1,14 +1,10 @@
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import Database from 'better-sqlite3';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { AwarenessLayer } from '../src/awareness.js';
|
||||
import { CronStore } from '../src/cron-store.js';
|
||||
import { ExecutionTraceStore } from '../src/execution-traces.js';
|
||||
import { IdentityLayer } from '../src/identity.js';
|
||||
import { WaggleDB, workspaceDataScope } from '../src/waggle-db.js';
|
||||
import { importLegacyMindData } from '../src/waggle-db-import.js';
|
||||
|
||||
let tmpDir: string;
|
||||
|
||||
@@ -65,65 +61,3 @@ describe('profile and awareness scopes', () => {
|
||||
db.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe('importLegacyMindData', () => {
|
||||
function legacyMind(file: string, setup: (db: Database.Database) => void): string {
|
||||
const p = path.join(tmpDir, file);
|
||||
const db = new Database(p);
|
||||
setup(db);
|
||||
db.close();
|
||||
return p;
|
||||
}
|
||||
|
||||
it('copies app tables, the profile and awareness once, and leaves the source unchanged', () => {
|
||||
const personalPath = legacyMind('personal.mind', (db) => {
|
||||
db.exec(`
|
||||
CREATE TABLE identity (id INTEGER PRIMARY KEY CHECK (id = 1), name TEXT NOT NULL, role TEXT NOT NULL DEFAULT '',
|
||||
department TEXT NOT NULL DEFAULT '', personality TEXT NOT NULL DEFAULT '', capabilities TEXT NOT NULL DEFAULT '',
|
||||
system_prompt TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')));
|
||||
INSERT INTO identity (id, name, role) VALUES (1, 'Ana', 'Founder');
|
||||
CREATE TABLE awareness (id INTEGER PRIMARY KEY AUTOINCREMENT, category TEXT NOT NULL, content TEXT NOT NULL,
|
||||
priority INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL DEFAULT (datetime('now')), expires_at TEXT);
|
||||
INSERT INTO awareness (category, content) VALUES ('task', 'Draft the plan');
|
||||
CREATE TABLE feedback_entries (id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT NOT NULL,
|
||||
message_index INTEGER NOT NULL, rating TEXT NOT NULL, reason TEXT, detail TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')));
|
||||
INSERT INTO feedback_entries (session_id, message_index, rating) VALUES ('s1', 2, 'up');
|
||||
`);
|
||||
// App tables created by the real stores, as the legacy personal.mind had them.
|
||||
const legacy = { getDatabase: () => db } as unknown as WaggleDB;
|
||||
new CronStore(legacy).create({ name: 'Morning briefing', cronExpr: '0 8 * * *', jobType: 'proactive' });
|
||||
new ExecutionTraceStore(legacy).start({ sessionId: 's1', input: 'hi' });
|
||||
});
|
||||
const wsPath = legacyMind('workspace.mind', (db) => {
|
||||
db.exec(`CREATE TABLE awareness (id INTEGER PRIMARY KEY AUTOINCREMENT, category TEXT NOT NULL, content TEXT NOT NULL,
|
||||
priority INTEGER NOT NULL DEFAULT 0, metadata TEXT NOT NULL DEFAULT '{}', created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
expires_at TEXT);
|
||||
INSERT INTO awareness (category, content) VALUES ('pending', 'Review the contract');`);
|
||||
});
|
||||
const before = fs.readFileSync(personalPath);
|
||||
|
||||
const db = new WaggleDB(path.join(tmpDir, 'waggle.db'));
|
||||
const sources = [
|
||||
{ path: personalPath, scope: 'personal' as const },
|
||||
{ path: wsPath, scope: workspaceDataScope('alpha') },
|
||||
{ path: path.join(tmpDir, 'missing.mind'), scope: workspaceDataScope('ghost') },
|
||||
];
|
||||
const reports = importLegacyMindData(db, sources);
|
||||
|
||||
expect(reports.map(r => r.status)).toEqual(['imported', 'imported', 'missing']);
|
||||
expect(new IdentityLayer(db).get()).toMatchObject({ name: 'Ana', role: 'Founder', scope: 'personal' });
|
||||
expect(new AwarenessLayer(db).getAll().map(i => i.content)).toEqual(['Draft the plan']);
|
||||
expect(new AwarenessLayer(db, workspaceDataScope('alpha')).getAll().map(i => i.content)).toEqual(['Review the contract']);
|
||||
expect(new CronStore(db).list().map(s => s.name)).toEqual(['Morning briefing']);
|
||||
expect(db.getDatabase().prepare('SELECT COUNT(*) AS n FROM feedback_entries').get()).toEqual({ n: 1 });
|
||||
expect(db.getDatabase().prepare('SELECT COUNT(*) AS n FROM execution_traces').get()).toEqual({ n: 1 });
|
||||
expect(fs.readFileSync(personalPath).equals(before)).toBe(true);
|
||||
|
||||
const again = importLegacyMindData(db, sources);
|
||||
expect(again.map(r => r.status)).toEqual(['already-imported', 'already-imported', 'missing']);
|
||||
expect(new CronStore(db).list()).toHaveLength(1);
|
||||
db.close();
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user