feat(core): give Waggle its own database, apart from memory (P1.5)

Waggle kept its own tables in the memory database. Before the memory
engine can be replaced by mem0 they need a home of their own.

- add WaggleDB (<dataDir>/waggle.db). Automations, notifications, held
  actions, skill hashes, thumbs feedback, cost tracking, the profile card
  and awareness items live there. Their stores take a WaggleDB, so passing
  a memory database is a type error.
- the profile card and awareness items are now per scope ('personal' or
  'workspace:<id>') in one table each, instead of one table per .mind file
- on the first start, copy these tables from personal.mind and every
  workspace.mind (importLegacyMindData). The legacy files are only read,
  each scope is copied once and a table only while its target is empty.
- copy the logger, injection scanner, external-memory ingress guard,
  WorkspaceManager and the embedders from @waggle/hive-mind-core into
  @waggle/core, which now re-exports its own copies
- point the server, agent orchestrator and CLI at waggle.db; update the
  tests that seeded these tables in a memory database

ADR-0007 records the layout and the licensing question about the copied
files.
This commit is contained in:
shumeiko.it
2026-10-23 14:31:28 +02:00
parent 20567ecb10
commit ad2ca875cb
65 changed files with 6131 additions and 201 deletions

View File

@@ -10,7 +10,7 @@ import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify';
import cors from '@fastify/cors';
import fastifyStatic from '@fastify/static';
import websocket from '@fastify/websocket';
import { MindDB, MultiMind, MultiMindCache, WorkspaceManager, WaggleConfig, createEmbeddingProvider, type EmbeddingProviderConfig, type EmbeddingProviderInstance, FrameStore, SessionStore, SuppressionStore, InstallAuditStore, CronStore, AwarenessLayer, VaultStore, SkillHashStore, HarvestSourceStore, ClaudeCodeAdapter, reconcileIndexes, TeamSync, TelemetryStore, TELEMETRY_EVENTS, ExecutionTraceStore, harvestSetHash, type WorkspaceConfig } from '@waggle/core';
import { MindDB, MultiMind, WaggleDB, importLegacyMindData, workspaceDataScope, type LegacyMindSource, MultiMindCache, WorkspaceManager, WaggleConfig, createEmbeddingProvider, type EmbeddingProviderConfig, type EmbeddingProviderInstance, FrameStore, SessionStore, SuppressionStore, InstallAuditStore, CronStore, AwarenessLayer, VaultStore, SkillHashStore, HarvestSourceStore, ClaudeCodeAdapter, reconcileIndexes, TeamSync, TelemetryStore, TELEMETRY_EVENTS, ExecutionTraceStore, harvestSetHash, type WorkspaceConfig } from '@waggle/core';
import { corsOriginAllowed } from './cors-config.js';
import { getBoundTeamServer } from './team-server-binding.js';
import { fetchTeamServer } from './team-server-egress.js';
@@ -382,6 +382,7 @@ declare module 'fastify' {
agentState: AgentState;
auditStore: import('@waggle/core').InstallAuditStore;
cronStore: import('@waggle/core').CronStore;
waggleDb: import('@waggle/core').WaggleDB;
traceStore: import('@waggle/core').ExecutionTraceStore;
/** Active behavioral spec — the compiled `BEHAVIORAL_SPEC` baseline. */
activeBehavioralSpec: ReturnType<typeof import('@waggle/agent').buildActiveBehavioralSpec>;
@@ -505,18 +506,39 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
const multiMind = new MultiMind(personalPath);
server.decorate('multiMind', multiMind);
// Waggle's own data (automations, notifications, cost tracking, profile and
// awareness) lives in waggle.db, apart from memory (ADR-0007). The first
// start copies it from the legacy .mind files, which stay untouched.
const waggleDb = WaggleDB.openInDataDir(fullConfig.dataDir);
server.decorate('waggleDb', waggleDb);
const legacySources: LegacyMindSource[] = [{ path: personalPath, scope: 'personal' }];
for (const ws of wsManager.list()) {
try {
legacySources.push({ path: wsManager.getMindPath(ws.id), scope: workspaceDataScope(ws.id) });
} catch {
// A workspace whose directory fails the path checks has nothing to copy.
}
}
for (const report of importLegacyMindData(waggleDb, legacySources)) {
if (report.status === 'imported') {
log.info(`[waggle-db] copied legacy data for ${report.scope}: ${JSON.stringify(report.copied)}`);
} else if (report.status === 'failed') {
log.warn(`[waggle-db] could not copy legacy data for ${report.scope}: ${report.error}`);
}
}
// Install audit store — persistent trail for capability install events
const auditStore = new InstallAuditStore(multiMind.personal);
server.decorate('auditStore', auditStore);
// Cron store — SQLite-backed schedule persistence for Solo
const cronStore = new CronStore(multiMind.personal);
const cronStore = new CronStore(waggleDb);
server.decorate('cronStore', cronStore);
seedDefaultCrons(cronStore);
// Execution trace store — persistent log of agent runs (cost tracking, run history)
const traceStore = new ExecutionTraceStore(multiMind.personal);
const traceStore = new ExecutionTraceStore(waggleDb);
server.decorate('traceStore', traceStore);
// Workflow-harness → trace bridge. Any harness phase that completes or
@@ -691,6 +713,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
// Orchestrator — connects to personal .mind
const orchestrator = new Orchestrator({
db: multiMind.personal,
waggleDb,
embedder,
mode: 'local',
version: '0.4',
@@ -902,7 +925,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
const skills = loadSkills(waggleHome);
// Skill hash store — detect skill changes on disk
const skillHashStore = new SkillHashStore(multiMind.personal);
const skillHashStore = new SkillHashStore(waggleDb);
server.decorate('skillHashStore', skillHashStore);
// Check for changed skills at startup
@@ -1180,6 +1203,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
const createSessionOrchestrator = (workspaceMind?: MindDB): Orchestrator => {
const sessionOrch = new Orchestrator({
db: multiMind.personal,
waggleDb,
embedder,
mode: 'local',
version: '0.4',
@@ -1859,7 +1883,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
return age > 7 * 24 * 60 * 60 * 1000;
}).length;
if (staleCount > 0) {
const wsAwareness = new AwarenessLayer(wsDb);
const wsAwareness = new AwarenessLayer(waggleDb, workspaceDataScope(ws.id));
wsAwareness.add('flag', `Workspace "${ws.name}" has ${staleCount} stale memory frames (>7 days untouched)`, 0);
}
}
@@ -1872,6 +1896,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
dataDir: fullConfig.dataDir,
workspaceManager: wsManager,
getWorkspaceMindDb,
waggleDb,
};
/** Emit a ProactiveMessage through the delivery router pipeline. */
@@ -2152,7 +2177,8 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
const body = await resp.json() as unknown;
return parseOpenAiTextCompletion(body).content;
};
const loopResult = await runLoopTick({ schedule, mindDb, embedder, chat: loopChat, log });
const loopAwareness = new AwarenessLayer(waggleDb, wsId && wsId !== '*' ? workspaceDataScope(wsId) : 'personal');
const loopResult = await runLoopTick({ schedule, mindDb, awareness: loopAwareness, embedder, chat: loopChat, log });
if (!loopResult.skipped) {
emitNotification(server, {
title: `Loop: ${schedule.name}`,
@@ -2981,6 +3007,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
// Close all cached workspace minds
mindCache.closeAll();
multiMind.close();
waggleDb.close();
// Close audit DB, teams DB, and telemetry
closeAuditDb();