feat(core): give Waggle its own database, apart from memory (P1.5)
Waggle kept its own tables in the memory database. Before the memory
engine can be replaced by mem0 they need a home of their own.
- add WaggleDB (<dataDir>/waggle.db). Automations, notifications, held
actions, skill hashes, thumbs feedback, cost tracking, the profile card
and awareness items live there. Their stores take a WaggleDB, so passing
a memory database is a type error.
- the profile card and awareness items are now per scope ('personal' or
'workspace:<id>') in one table each, instead of one table per .mind file
- on the first start, copy these tables from personal.mind and every
workspace.mind (importLegacyMindData). The legacy files are only read,
each scope is copied once and a table only while its target is empty.
- copy the logger, injection scanner, external-memory ingress guard,
WorkspaceManager and the embedders from @waggle/hive-mind-core into
@waggle/core, which now re-exports its own copies
- point the server, agent orchestrator and CLI at waggle.db; update the
tests that seeded these tables in a memory database
ADR-0007 records the layout and the licensing question about the copied
files.
This commit is contained in:
161
packages/core/src/waggle-db-import.ts
Normal file
161
packages/core/src/waggle-db-import.ts
Normal file
@@ -0,0 +1,161 @@
|
||||
/**
|
||||
* One-time copy of Waggle's own tables from the legacy memory databases
|
||||
* (personal.mind, workspace.mind) into waggle.db.
|
||||
*
|
||||
* The legacy files are only read: the data directory may be shared with the
|
||||
* commercial edition, which keeps using them. Each source is copied once,
|
||||
* recorded in waggle_meta, and a table is copied only while its target is
|
||||
* empty, so a re-run never duplicates rows.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import Database from 'better-sqlite3';
|
||||
import type { Database as DatabaseType } from 'better-sqlite3';
|
||||
import { AwarenessLayer } from './awareness.js';
|
||||
import { CronStore } from './cron-store.js';
|
||||
import { ExecutionTraceStore } from './execution-traces.js';
|
||||
import { IdentityLayer } from './identity.js';
|
||||
import { SkillHashStore } from './skill-hashes.js';
|
||||
import type { DataScope, WaggleDB } from './waggle-db.js';
|
||||
|
||||
/** Tables that only ever lived in personal.mind, copied with their ids. */
|
||||
const PERSONAL_TABLES = [
|
||||
'cron_schedules',
|
||||
'cron_execution_history',
|
||||
'cron_run_leases',
|
||||
'notifications',
|
||||
'pending_actions',
|
||||
'skill_hashes',
|
||||
'feedback_entries',
|
||||
'execution_traces',
|
||||
'execution_trace_spend',
|
||||
'execution_trace_spend_reservations',
|
||||
] as const;
|
||||
|
||||
const PROFILE_COLUMNS = ['name', 'role', 'department', 'personality', 'capabilities', 'system_prompt', 'created_at', 'updated_at'];
|
||||
const AWARENESS_COLUMNS = ['category', 'content', 'priority', 'metadata', 'created_at', 'expires_at'];
|
||||
|
||||
export interface LegacyMindSource {
|
||||
/** Path to a legacy .mind file. */
|
||||
path: string;
|
||||
scope: DataScope;
|
||||
}
|
||||
|
||||
export interface LegacyImportReport {
|
||||
scope: DataScope;
|
||||
status: 'imported' | 'already-imported' | 'missing' | 'failed';
|
||||
copied: Record<string, number>;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
function importKey(scope: DataScope): string {
|
||||
return `import:${scope}`;
|
||||
}
|
||||
|
||||
function tableExists(db: DatabaseType, table: string): boolean {
|
||||
return db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(table) !== undefined;
|
||||
}
|
||||
|
||||
function columnsOf(db: DatabaseType, table: string): string[] {
|
||||
return (db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map(c => c.name);
|
||||
}
|
||||
|
||||
function copyRows(
|
||||
source: DatabaseType,
|
||||
target: DatabaseType,
|
||||
sourceTable: string,
|
||||
targetTable: string,
|
||||
columns: string[],
|
||||
extra: Record<string, unknown> = {},
|
||||
): number {
|
||||
const rows = source.prepare(`SELECT ${columns.join(', ')} FROM ${sourceTable}`).all() as Array<Record<string, unknown>>;
|
||||
if (rows.length === 0) return 0;
|
||||
const allColumns = [...columns, ...Object.keys(extra)];
|
||||
const insert = target.prepare(
|
||||
`INSERT INTO ${targetTable} (${allColumns.join(', ')}) VALUES (${allColumns.map(() => '?').join(', ')})`,
|
||||
);
|
||||
for (const row of rows) {
|
||||
insert.run(...columns.map(c => row[c]), ...Object.values(extra));
|
||||
}
|
||||
return rows.length;
|
||||
}
|
||||
|
||||
function copyPersonalTables(source: DatabaseType, target: DatabaseType, copied: Record<string, number>): void {
|
||||
for (const table of PERSONAL_TABLES) {
|
||||
if (!tableExists(source, table)) continue;
|
||||
if (!tableExists(target, table)) {
|
||||
// Lazily created tables (feedback_entries) take the legacy definition.
|
||||
const ddl = source.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(table) as { sql: string };
|
||||
target.exec(ddl.sql);
|
||||
}
|
||||
const empty = target.prepare(`SELECT 1 FROM ${table} LIMIT 1`).get() === undefined;
|
||||
if (!empty) continue;
|
||||
const targetColumns = new Set(columnsOf(target, table));
|
||||
const shared = columnsOf(source, table).filter(c => targetColumns.has(c));
|
||||
copied[table] = copyRows(source, target, table, table, shared);
|
||||
}
|
||||
}
|
||||
|
||||
function copyScopedTables(source: DatabaseType, target: DatabaseType, scope: DataScope, copied: Record<string, number>): void {
|
||||
if (tableExists(source, 'identity')) {
|
||||
const hasProfile = target.prepare('SELECT 1 FROM profiles WHERE scope = ?').get(scope) !== undefined;
|
||||
if (!hasProfile) {
|
||||
const sourceColumns = new Set(columnsOf(source, 'identity'));
|
||||
const shared = PROFILE_COLUMNS.filter(c => sourceColumns.has(c));
|
||||
copied.profiles = copyRows(source, target, 'identity', 'profiles', shared, { scope });
|
||||
}
|
||||
}
|
||||
if (tableExists(source, 'awareness')) {
|
||||
const hasItems = target.prepare('SELECT 1 FROM awareness_items WHERE scope = ? LIMIT 1').get(scope) !== undefined;
|
||||
if (!hasItems) {
|
||||
const sourceColumns = new Set(columnsOf(source, 'awareness'));
|
||||
const shared = AWARENESS_COLUMNS.filter(c => sourceColumns.has(c));
|
||||
copied.awareness_items = copyRows(source, target, 'awareness', 'awareness_items', shared, { scope });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies Waggle data from each legacy source into `waggleDb`, once per scope.
|
||||
* The personal source also brings the app-wide tables; workspace sources bring
|
||||
* their profile and awareness items.
|
||||
*/
|
||||
export function importLegacyMindData(waggleDb: WaggleDB, sources: LegacyMindSource[]): LegacyImportReport[] {
|
||||
// Create every target table with its current schema first.
|
||||
new CronStore(waggleDb);
|
||||
new SkillHashStore(waggleDb);
|
||||
new ExecutionTraceStore(waggleDb);
|
||||
new IdentityLayer(waggleDb);
|
||||
new AwarenessLayer(waggleDb);
|
||||
|
||||
const target = waggleDb.getDatabase();
|
||||
const reports: LegacyImportReport[] = [];
|
||||
for (const { path: sourcePath, scope } of sources) {
|
||||
const copied: Record<string, number> = {};
|
||||
if (waggleDb.getMeta(importKey(scope))) {
|
||||
reports.push({ scope, status: 'already-imported', copied });
|
||||
continue;
|
||||
}
|
||||
const stat = fs.statSync(sourcePath, { throwIfNoEntry: false });
|
||||
if (!stat || stat.size === 0) {
|
||||
reports.push({ scope, status: 'missing', copied });
|
||||
continue;
|
||||
}
|
||||
let source: DatabaseType | null = null;
|
||||
try {
|
||||
source = new Database(sourcePath, { fileMustExist: true });
|
||||
const db = source;
|
||||
target.transaction(() => {
|
||||
if (scope === 'personal') copyPersonalTables(db, target, copied);
|
||||
copyScopedTables(db, target, scope, copied);
|
||||
})();
|
||||
waggleDb.setMeta(importKey(scope), new Date().toISOString());
|
||||
reports.push({ scope, status: 'imported', copied });
|
||||
} catch (err) {
|
||||
reports.push({ scope, status: 'failed', copied: {}, error: err instanceof Error ? err.message : String(err) });
|
||||
} finally {
|
||||
source?.close();
|
||||
}
|
||||
}
|
||||
return reports;
|
||||
}
|
||||
Reference in New Issue
Block a user