Files
waggle-os/packages/core/src/waggle-db-import.ts
shumeiko.it 2e08f2ae96 refactor(server): move the install audit trail from core to the server (P1.2, UF-040)
Core keeps only shared infrastructure, so InstallAuditStore leaves it and
lives in packages/server/src/local/install-audit.ts. The trail itself stays:
the Audit and Activity tabs, the extend and export routes, and the threat
model entry keep working on the same waggle.db table.

- @waggle/shared gets RecordAuditInput and the narrow InstallAuditRecorder
  (record only), which the agent's skill tools now depend on.
- The server creates the audit table before the one-time legacy copy, which
  no longer constructs it in core.
- UF-040: GET /api/debug/logs called prepare() on the store's private db
  wrapper, which has none, so the support bundle never carried recent
  installs. It now reads them through getRecentSummary().
2026-10-25 21:32:38 +01:00

166 lines
6.4 KiB
TypeScript

/**
* One-time copy of Waggle's own tables from the legacy memory databases
* (personal.mind, workspace.mind) into waggle.db.
*
* The legacy files are only read: the data directory may be shared with the
* commercial edition, which keeps using them. Each source is copied once,
* recorded in waggle_meta, and a table is copied only while its target is
* empty, so a re-run never duplicates rows.
*/
import fs from 'node:fs';
import Database from 'better-sqlite3';
import type { Database as DatabaseType } from 'better-sqlite3';
import { AwarenessLayer } from './awareness.js';
import { CronStore } from './cron-store.js';
import { ExecutionTraceStore } from './execution-traces.js';
import { IdentityLayer } from './identity.js';
import { SkillHashStore } from './skill-hashes.js';
import type { DataScope, WaggleDB } from './waggle-db.js';
/** Tables that only ever lived in personal.mind, copied with their ids. */
const PERSONAL_TABLES = [
'cron_schedules',
'cron_execution_history',
'cron_run_leases',
'notifications',
'pending_actions',
'skill_hashes',
'feedback_entries',
'execution_traces',
'execution_trace_spend',
'execution_trace_spend_reservations',
'install_audit',
] as const;
const PROFILE_COLUMNS = ['name', 'role', 'department', 'personality', 'capabilities', 'system_prompt', 'created_at', 'updated_at'];
const AWARENESS_COLUMNS = ['category', 'content', 'priority', 'metadata', 'created_at', 'expires_at'];
export interface LegacyMindSource {
/** Path to a legacy .mind file. */
path: string;
scope: DataScope;
}
export interface LegacyImportReport {
scope: DataScope;
status: 'imported' | 'already-imported' | 'missing' | 'failed';
copied: Record<string, number>;
error?: string;
}
function importKey(scope: DataScope): string {
return `import:${scope}`;
}
function tableExists(db: DatabaseType, table: string): boolean {
return db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(table) !== undefined;
}
function columnsOf(db: DatabaseType, table: string): string[] {
return (db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map(c => c.name);
}
function copyRows(
source: DatabaseType,
target: DatabaseType,
sourceTable: string,
targetTable: string,
columns: string[],
extra: Record<string, unknown> = {},
): number {
const rows = source.prepare(`SELECT ${columns.join(', ')} FROM ${sourceTable}`).all() as Array<Record<string, unknown>>;
if (rows.length === 0) return 0;
const allColumns = [...columns, ...Object.keys(extra)];
const insert = target.prepare(
`INSERT INTO ${targetTable} (${allColumns.join(', ')}) VALUES (${allColumns.map(() => '?').join(', ')})`,
);
for (const row of rows) {
insert.run(...columns.map(c => row[c]), ...Object.values(extra));
}
return rows.length;
}
function copyPersonalTables(source: DatabaseType, target: DatabaseType, copied: Record<string, number>): void {
for (const table of PERSONAL_TABLES) {
if (!tableExists(source, table)) continue;
if (!tableExists(target, table)) {
// Lazily created tables (feedback_entries) take the legacy definition.
const ddl = source.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(table) as { sql: string };
target.exec(ddl.sql);
}
const empty = target.prepare(`SELECT 1 FROM ${table} LIMIT 1`).get() === undefined;
if (!empty) continue;
const targetColumns = new Set(columnsOf(target, table));
const shared = columnsOf(source, table).filter(c => targetColumns.has(c));
copied[table] = copyRows(source, target, table, table, shared);
}
}
function copyScopedTables(source: DatabaseType, target: DatabaseType, scope: DataScope, copied: Record<string, number>): void {
if (tableExists(source, 'identity')) {
const hasProfile = target.prepare('SELECT 1 FROM profiles WHERE scope = ?').get(scope) !== undefined;
if (!hasProfile) {
const sourceColumns = new Set(columnsOf(source, 'identity'));
const shared = PROFILE_COLUMNS.filter(c => sourceColumns.has(c));
copied.profiles = copyRows(source, target, 'identity', 'profiles', shared, { scope });
}
}
if (tableExists(source, 'awareness')) {
const hasItems = target.prepare('SELECT 1 FROM awareness_items WHERE scope = ? LIMIT 1').get(scope) !== undefined;
if (!hasItems) {
const sourceColumns = new Set(columnsOf(source, 'awareness'));
const shared = AWARENESS_COLUMNS.filter(c => sourceColumns.has(c));
copied.awareness_items = copyRows(source, target, 'awareness', 'awareness_items', shared, { scope });
}
}
}
/**
* Copies Waggle data from each legacy source into `waggleDb`, once per scope.
* The personal source also brings the app-wide tables; workspace sources bring
* their profile and awareness items.
*
* Tables owned outside core (install_audit, owned by the server) must be created
* by their owner before this call; otherwise they take the legacy definition.
*/
export function importLegacyMindData(waggleDb: WaggleDB, sources: LegacyMindSource[]): LegacyImportReport[] {
// Create every core target table with its current schema first.
new CronStore(waggleDb);
new SkillHashStore(waggleDb);
new ExecutionTraceStore(waggleDb);
new IdentityLayer(waggleDb);
new AwarenessLayer(waggleDb);
const target = waggleDb.getDatabase();
const reports: LegacyImportReport[] = [];
for (const { path: sourcePath, scope } of sources) {
const copied: Record<string, number> = {};
if (waggleDb.getMeta(importKey(scope))) {
reports.push({ scope, status: 'already-imported', copied });
continue;
}
const stat = fs.statSync(sourcePath, { throwIfNoEntry: false });
if (!stat || stat.size === 0) {
reports.push({ scope, status: 'missing', copied });
continue;
}
let source: DatabaseType | null = null;
try {
source = new Database(sourcePath, { fileMustExist: true });
const db = source;
target.transaction(() => {
if (scope === 'personal') copyPersonalTables(db, target, copied);
copyScopedTables(db, target, scope, copied);
})();
waggleDb.setMeta(importKey(scope), new Date().toISOString());
reports.push({ scope, status: 'imported', copied });
} catch (err) {
reports.push({ scope, status: 'failed', copied: {}, error: err instanceof Error ? err.message : String(err) });
} finally {
source?.close();
}
}
return reports;
}