feat(core): give Waggle its own database, apart from memory (P1.5)

Waggle kept its own tables in the memory database. Before the memory
engine can be replaced by mem0 they need a home of their own.

- add WaggleDB (<dataDir>/waggle.db). Automations, notifications, held
  actions, skill hashes, thumbs feedback, cost tracking, the profile card
  and awareness items live there. Their stores take a WaggleDB, so passing
  a memory database is a type error.
- the profile card and awareness items are now per scope ('personal' or
  'workspace:<id>') in one table each, instead of one table per .mind file
- on the first start, copy these tables from personal.mind and every
  workspace.mind (importLegacyMindData). The legacy files are only read,
  each scope is copied once and a table only while its target is empty.
- copy the logger, injection scanner, external-memory ingress guard,
  WorkspaceManager and the embedders from @waggle/hive-mind-core into
  @waggle/core, which now re-exports its own copies
- point the server, agent orchestrator and CLI at waggle.db; update the
  tests that seeded these tables in a memory database

ADR-0007 records the layout and the licensing question about the copied
files.
This commit is contained in:
shumeiko.it
2026-10-23 14:31:28 +02:00
parent 20567ecb10
commit ad2ca875cb
65 changed files with 6131 additions and 201 deletions

View File

@@ -10,7 +10,7 @@ import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify';
import cors from '@fastify/cors';
import fastifyStatic from '@fastify/static';
import websocket from '@fastify/websocket';
import { MindDB, MultiMind, MultiMindCache, WorkspaceManager, WaggleConfig, createEmbeddingProvider, type EmbeddingProviderConfig, type EmbeddingProviderInstance, FrameStore, SessionStore, SuppressionStore, InstallAuditStore, CronStore, AwarenessLayer, VaultStore, SkillHashStore, HarvestSourceStore, ClaudeCodeAdapter, reconcileIndexes, TeamSync, TelemetryStore, TELEMETRY_EVENTS, ExecutionTraceStore, harvestSetHash, type WorkspaceConfig } from '@waggle/core';
import { MindDB, MultiMind, WaggleDB, importLegacyMindData, workspaceDataScope, type LegacyMindSource, MultiMindCache, WorkspaceManager, WaggleConfig, createEmbeddingProvider, type EmbeddingProviderConfig, type EmbeddingProviderInstance, FrameStore, SessionStore, SuppressionStore, InstallAuditStore, CronStore, AwarenessLayer, VaultStore, SkillHashStore, HarvestSourceStore, ClaudeCodeAdapter, reconcileIndexes, TeamSync, TelemetryStore, TELEMETRY_EVENTS, ExecutionTraceStore, harvestSetHash, type WorkspaceConfig } from '@waggle/core';
import { corsOriginAllowed } from './cors-config.js';
import { getBoundTeamServer } from './team-server-binding.js';
import { fetchTeamServer } from './team-server-egress.js';
@@ -382,6 +382,7 @@ declare module 'fastify' {
agentState: AgentState;
auditStore: import('@waggle/core').InstallAuditStore;
cronStore: import('@waggle/core').CronStore;
waggleDb: import('@waggle/core').WaggleDB;
traceStore: import('@waggle/core').ExecutionTraceStore;
/** Active behavioral spec — the compiled `BEHAVIORAL_SPEC` baseline. */
activeBehavioralSpec: ReturnType<typeof import('@waggle/agent').buildActiveBehavioralSpec>;
@@ -505,18 +506,39 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
const multiMind = new MultiMind(personalPath);
server.decorate('multiMind', multiMind);
// Waggle's own data (automations, notifications, cost tracking, profile and
// awareness) lives in waggle.db, apart from memory (ADR-0007). The first
// start copies it from the legacy .mind files, which stay untouched.
const waggleDb = WaggleDB.openInDataDir(fullConfig.dataDir);
server.decorate('waggleDb', waggleDb);
const legacySources: LegacyMindSource[] = [{ path: personalPath, scope: 'personal' }];
for (const ws of wsManager.list()) {
try {
legacySources.push({ path: wsManager.getMindPath(ws.id), scope: workspaceDataScope(ws.id) });
} catch {
// A workspace whose directory fails the path checks has nothing to copy.
}
}
for (const report of importLegacyMindData(waggleDb, legacySources)) {
if (report.status === 'imported') {
log.info(`[waggle-db] copied legacy data for ${report.scope}: ${JSON.stringify(report.copied)}`);
} else if (report.status === 'failed') {
log.warn(`[waggle-db] could not copy legacy data for ${report.scope}: ${report.error}`);
}
}
// Install audit store — persistent trail for capability install events
const auditStore = new InstallAuditStore(multiMind.personal);
server.decorate('auditStore', auditStore);
// Cron store — SQLite-backed schedule persistence for Solo
const cronStore = new CronStore(multiMind.personal);
const cronStore = new CronStore(waggleDb);
server.decorate('cronStore', cronStore);
seedDefaultCrons(cronStore);
// Execution trace store — persistent log of agent runs (cost tracking, run history)
const traceStore = new ExecutionTraceStore(multiMind.personal);
const traceStore = new ExecutionTraceStore(waggleDb);
server.decorate('traceStore', traceStore);
// Workflow-harness → trace bridge. Any harness phase that completes or
@@ -691,6 +713,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
// Orchestrator — connects to personal .mind
const orchestrator = new Orchestrator({
db: multiMind.personal,
waggleDb,
embedder,
mode: 'local',
version: '0.4',
@@ -902,7 +925,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
const skills = loadSkills(waggleHome);
// Skill hash store — detect skill changes on disk
const skillHashStore = new SkillHashStore(multiMind.personal);
const skillHashStore = new SkillHashStore(waggleDb);
server.decorate('skillHashStore', skillHashStore);
// Check for changed skills at startup
@@ -1180,6 +1203,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
const createSessionOrchestrator = (workspaceMind?: MindDB): Orchestrator => {
const sessionOrch = new Orchestrator({
db: multiMind.personal,
waggleDb,
embedder,
mode: 'local',
version: '0.4',
@@ -1859,7 +1883,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
return age > 7 * 24 * 60 * 60 * 1000;
}).length;
if (staleCount > 0) {
const wsAwareness = new AwarenessLayer(wsDb);
const wsAwareness = new AwarenessLayer(waggleDb, workspaceDataScope(ws.id));
wsAwareness.add('flag', `Workspace "${ws.name}" has ${staleCount} stale memory frames (>7 days untouched)`, 0);
}
}
@@ -1872,6 +1896,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
dataDir: fullConfig.dataDir,
workspaceManager: wsManager,
getWorkspaceMindDb,
waggleDb,
};
/** Emit a ProactiveMessage through the delivery router pipeline. */
@@ -2152,7 +2177,8 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
const body = await resp.json() as unknown;
return parseOpenAiTextCompletion(body).content;
};
const loopResult = await runLoopTick({ schedule, mindDb, embedder, chat: loopChat, log });
const loopAwareness = new AwarenessLayer(waggleDb, wsId && wsId !== '*' ? workspaceDataScope(wsId) : 'personal');
const loopResult = await runLoopTick({ schedule, mindDb, awareness: loopAwareness, embedder, chat: loopChat, log });
if (!loopResult.skipped) {
emitNotification(server, {
title: `Loop: ${schedule.name}`,
@@ -2981,6 +3007,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
// Close all cached workspace minds
mindCache.closeAll();
multiMind.close();
waggleDb.close();
// Close audit DB, teams DB, and telemetry
closeAuditDb();

View File

@@ -52,6 +52,8 @@ export interface LoopTickDeps {
schedule: LoopSchedule;
/** The already-resolved workspace (or personal) mind for this loop. */
mindDb: MindDB;
/** Awareness items of the same scope, in waggle.db; the loop keeps its state there. */
awareness: AwarenessLayer;
/** Embedder for recall-by-meaning. */
embedder: Embedder;
/** Toolless completion against the local proxy/LiteLLM. */
@@ -201,7 +203,7 @@ export function buildMakerPrompt(opts: {
* memory-write failures ARE caught (best-effort, must not fail the whole tick).
*/
export async function runLoopTick(deps: LoopTickDeps): Promise<LoopTickResult> {
const { schedule, mindDb, embedder, chat, log } = deps;
const { schedule, mindDb, awareness, embedder, chat, log } = deps;
const spec = parseLoopSpec(schedule.job_config);
if (!spec) {
@@ -210,7 +212,6 @@ export async function runLoopTick(deps: LoopTickDeps): Promise<LoopTickResult> {
}
const stateKey = `loop:${schedule.id}`; // namespaced so loops don't collide on awareness.status
const awareness = new AwarenessLayer(mindDb);
const prior = awareness.getByStatus(stateKey)[0];
const priorMeta = prior ? awareness.parseMetadata(prior) : undefined;
let priorResult = priorMeta ? String(priorMeta.result ?? '') : '';

View File

@@ -12,7 +12,7 @@
import fs from 'node:fs';
import path from 'node:path';
import type { WorkspaceManager, WorkspaceConfig } from '@waggle/core';
import { AwarenessLayer, workspaceDataScope, type WorkspaceManager, type WorkspaceConfig } from '@waggle/core';
import { materialFingerprint } from './notification-gate.js';
// ── Types ──────────────────────────────────────────────────────────────
@@ -34,6 +34,8 @@ export interface ProactiveContext {
workspaceManager: WorkspaceManager;
/** Get a cached workspace MindDB (opens on demand). Returns null if workspace not found. */
getWorkspaceMindDb: (workspaceId: string) => import('@waggle/core').MindDB | null;
/** waggle.db, which holds awareness items. */
waggleDb: import('@waggle/core').WaggleDB;
}
// ── Helpers ────────────────────────────────────────────────────────────
@@ -57,16 +59,10 @@ function getLastSessionActivity(dataDir: string, workspaceId: string): Date | nu
return latest > 0 ? new Date(latest) : null;
}
/** Count pending awareness items in a workspace mind. */
/** Count open task and pending awareness items of a workspace. */
function countPendingAwareness(ctx: ProactiveContext, workspaceId: string): number {
const db = ctx.getWorkspaceMindDb(workspaceId);
if (!db) return 0;
try {
const raw = db.getDatabase();
const row = raw.prepare(
"SELECT COUNT(*) as cnt FROM awareness WHERE (category = 'task' OR category = 'pending') AND (expires_at IS NULL OR expires_at > datetime('now'))",
).get() as { cnt: number } | undefined;
return row?.cnt ?? 0;
return new AwarenessLayer(ctx.waggleDb, workspaceDataScope(workspaceId)).countOpen();
} catch {
return 0;
}

View File

@@ -1034,6 +1034,7 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
try {
const nowBlock = buildWorkspaceNowBlock({
dataDir: server.localConfig.dataDir,
waggleDb: server.waggleDb,
workspaceId,
wsManager: server.workspaceManager,
activateWorkspaceMind: server.agentState.activateWorkspaceMind,
@@ -1741,6 +1742,7 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
if (!effectiveWorkspace) return 'No workspace state available.';
const block = buildWorkspaceNowBlock({
dataDir: server.localConfig.dataDir,
waggleDb: server.waggleDb,
workspaceId: effectiveWorkspace,
wsManager: server.workspaceManager,
activateWorkspaceMind: server.agentState.activateWorkspaceMind,

View File

@@ -252,6 +252,7 @@ export const commandRoutes: FastifyPluginAsync = async (server) => {
getWorkspaceState: async (): Promise<string> => {
const block = buildWorkspaceNowBlock({
dataDir: server.localConfig.dataDir,
waggleDb: server.waggleDb,
workspaceId: effectiveWorkspaceId,
wsManager: server.workspaceManager,
activateWorkspaceMind: server.agentState.activateWorkspaceMind,
@@ -289,6 +290,7 @@ export const commandRoutes: FastifyPluginAsync = async (server) => {
try {
const block = buildWorkspaceNowBlock({
dataDir: server.localConfig.dataDir,
waggleDb: server.waggleDb,
workspaceId: workspaceId ?? 'default',
wsManager: server.workspaceManager,
activateWorkspaceMind: server.agentState.activateWorkspaceMind,

View File

@@ -64,6 +64,7 @@ export const commandRoutes: FastifyPluginAsync = async (server) => {
getWorkspaceState: async (): Promise<string> => {
const block = buildWorkspaceNowBlock({
dataDir: server.localConfig.dataDir,
waggleDb: server.waggleDb,
workspaceId: effectiveWorkspaceId,
wsManager: server.workspaceManager,
activateWorkspaceMind: server.agentState.activateWorkspaceMind,

View File

@@ -76,7 +76,7 @@ export async function feedbackRoutes(fastify: FastifyInstance) {
}
try {
const db = fastify.multiMind.personal.getDatabase();
const db = fastify.waggleDb.getDatabase();
ensureFeedbackTable(db);
db.prepare(`
@@ -94,7 +94,7 @@ export async function feedbackRoutes(fastify: FastifyInstance) {
// GET /api/feedback/stats — return feedback stats
fastify.get('/api/feedback/stats', async (_request, reply) => {
try {
const db = fastify.multiMind.personal.getDatabase();
const db = fastify.waggleDb.getDatabase();
ensureFeedbackTable(db);
// Total feedback count

View File

@@ -260,7 +260,7 @@ export const homeRoutes: FastifyPluginAsync = async (server) => {
// ── Greeting + identity name (B8) ─────────────────────────────
let userName: string | undefined;
try {
const layer = new IdentityLayer(server.multiMind.personal);
const layer = new IdentityLayer(server.waggleDb);
if (layer.exists()) {
const name = layer.get().name?.trim();
if (name) userName = name;
@@ -302,6 +302,7 @@ export const homeRoutes: FastifyPluginAsync = async (server) => {
try {
const state = buildWorkspaceState({
dataDir: server.localConfig.dataDir,
waggleDb: server.waggleDb,
workspaceId: ws.id,
wsManager: server.workspaceManager,
activateWorkspaceMind: server.agentState.activateWorkspaceMind,

View File

@@ -9,8 +9,8 @@
// UI sees a single consistent shape regardless of whether the user has
// completed the onboarding identity questionnaire yet.
import type { FastifyPluginAsync } from 'fastify';
import { IdentityLayer } from '@waggle/core';
import type { FastifyInstance, FastifyPluginAsync } from 'fastify';
import { IdentityLayer, workspaceDataScope, type DataScope } from '@waggle/core';
interface IdentityResponseShape {
configured: boolean;
@@ -25,6 +25,12 @@ interface IdentityResponseShape {
_note?: string;
}
/** Profile scope for a request: a known workspace, else the personal profile. */
function identityScope(server: FastifyInstance, wsId: string | undefined): DataScope {
if (wsId && wsId !== 'personal' && server.workspaceManager?.get(wsId)) return workspaceDataScope(wsId);
return 'personal';
}
function placeholder(note?: string): IdentityResponseShape {
return {
configured: false,
@@ -63,20 +69,12 @@ export const identityRoutes: FastifyPluginAsync = async (server) => {
'/api/identity',
async (request, reply) => {
const wsId = request.query.workspace ?? request.query.workspaceId;
const personalMindDb = server.multiMind?.personal;
if (!personalMindDb) {
return reply.send(placeholder('multi-mind not initialized'));
if (!server.waggleDb) {
return reply.send(placeholder('waggle.db not initialized'));
}
// Use the workspace mind if requested + available; else personal.
const targetMindDb =
wsId && wsId !== 'personal'
? server.agentState?.getWorkspaceMindDb?.(wsId) ?? personalMindDb
: personalMindDb;
try {
const layer = new IdentityLayer(targetMindDb);
const layer = new IdentityLayer(server.waggleDb, identityScope(server, wsId));
if (!layer.exists()) {
return reply.send(placeholder('identity not configured for this mind'));
}
@@ -115,18 +113,11 @@ export const identityRoutes: FastifyPluginAsync = async (server) => {
}>('/api/identity', async (request, reply) => {
const body = request.body ?? {};
const wsId = body.workspace ?? body.workspaceId;
const personalMindDb = server.multiMind?.personal;
if (!personalMindDb) {
return reply.status(503).send({ error: 'multi-mind not initialized' });
if (!server.waggleDb) {
return reply.status(503).send({ error: 'waggle.db not initialized' });
}
const targetMindDb =
wsId && wsId !== 'personal'
? server.agentState?.getWorkspaceMindDb?.(wsId) ?? personalMindDb
: personalMindDb;
const layer = new IdentityLayer(targetMindDb);
const layer = new IdentityLayer(server.waggleDb, identityScope(server, wsId));
try {
// Merge-on-update: an omitted field keeps its stored value. The prior
// `body.x ?? ''` semantics meant a partial write (e.g. the onboarding

View File

@@ -2,6 +2,7 @@ import type { FastifyPluginAsync } from 'fastify';
import type { SearchScope, Importance, FrameSource, MemoryFrame } from '@waggle/core';
import {
AwarenessLayer,
workspaceDataScope,
evaluateExternalMemoryIngress,
FrameStore,
KnowledgeGraph,
@@ -796,7 +797,8 @@ export const memoryRoutes: FastifyPluginAsync = async (server) => {
let awarenessId: number | undefined;
if (kind === 'task') {
try {
awarenessId = new AwarenessLayer(targetDb).add('task', content, 0, undefined, {
const awarenessScope = mindLabel === 'workspace' && workspaceId ? workspaceDataScope(workspaceId) : 'personal';
awarenessId = new AwarenessLayer(server.waggleDb, awarenessScope).add('task', content, 0, undefined, {
context: 'quick-capture',
status: 'open',
}).id;

View File

@@ -1,6 +1,6 @@
import fs from 'node:fs';
import path from 'node:path';
import { MindDB } from '@waggle/core';
import { MindDB, type WaggleDB } from '@waggle/core';
import { assertSafeSegment } from './validate.js';
import { extractProgressItems, type ProgressItem } from './sessions.js';
import {
@@ -225,12 +225,14 @@ export function buildWorkspaceNowBlock(opts: {
dataDir: string;
workspaceId: string;
wsManager: WsManagerLike;
/** waggle.db, which holds the workspace's awareness items. */
waggleDb: WaggleDB;
/** @deprecated Read-only context construction no longer mutates global workspace state. */
activateWorkspaceMind?: (id: string) => boolean;
/** Optional cron schedules for upcoming schedule display */
cronSchedules?: CronScheduleLike[];
}): WorkspaceNowBlock | null {
const { dataDir, workspaceId, wsManager } = opts;
const { dataDir, workspaceId, wsManager, waggleDb } = opts;
// Path-traversal guard: workspaceId becomes a path segment below
// (getMindPath + dataDir/workspaces/<workspaceId>/sessions). Reject any
@@ -248,6 +250,7 @@ export function buildWorkspaceNowBlock(opts: {
dataDir,
workspaceId,
wsManager,
waggleDb,
});
if (structuredState) {

View File

@@ -772,6 +772,7 @@ export const workspaceRoutes: FastifyPluginAsync = async (server) => {
try {
workspaceState = buildWorkspaceState({
dataDir: server.localConfig.dataDir,
waggleDb: server.waggleDb,
workspaceId: id,
wsManager: server.workspaceManager,
activateWorkspaceMind: server.agentState.activateWorkspaceMind,
@@ -897,6 +898,7 @@ export const workspaceRoutes: FastifyPluginAsync = async (server) => {
try {
state = buildWorkspaceState({
dataDir: server.localConfig.dataDir,
waggleDb: server.waggleDb,
workspaceId: id,
wsManager: server.workspaceManager,
activateWorkspaceMind: server.agentState.activateWorkspaceMind,

View File

@@ -12,7 +12,7 @@
import fs from 'node:fs';
import path from 'node:path';
import { MindDB } from '@waggle/core';
import { AwarenessLayer, MindDB, WaggleDB, workspaceDataScope } from '@waggle/core';
import {
extractProgressItems,
extractOpenQuestions,
@@ -112,19 +112,8 @@ function extractDecisionItems(
// ── Active items from awareness layer ──────────────────────────────────
interface AwarenessRow {
id: number;
category: string;
content: string;
created_at: string;
}
function extractAwarenessItems(raw: ReturnType<MindDB['getDatabase']>): StateItem[] {
const items = raw.prepare(
`SELECT id, category, content, created_at FROM awareness
WHERE expires_at IS NULL OR expires_at > datetime('now')
ORDER BY priority DESC LIMIT 10`,
).all() as AwarenessRow[];
function extractAwarenessItems(waggleDb: WaggleDB, workspaceId: string): StateItem[] {
const items = new AwarenessLayer(waggleDb, workspaceDataScope(workspaceId)).getAll();
return items
.filter(i => i.category === 'task' || i.category === 'action' || i.category === 'pending')
@@ -228,12 +217,14 @@ export interface BuildWorkspaceStateOpts {
dataDir: string;
workspaceId: string;
wsManager: WsManagerLike;
/** waggle.db, which holds the workspace's awareness items. */
waggleDb: WaggleDB;
/** @deprecated Read-only state construction no longer mutates global workspace state. */
activateWorkspaceMind?: (id: string) => boolean;
}
export function buildWorkspaceState(opts: BuildWorkspaceStateOpts): WorkspaceState | null {
const { dataDir, workspaceId, wsManager } = opts;
const { dataDir, workspaceId, wsManager, waggleDb } = opts;
const ws = wsManager.get(workspaceId);
if (!ws) return null;
@@ -257,7 +248,6 @@ export function buildWorkspaceState(opts: BuildWorkspaceStateOpts): WorkspaceSta
}
recentDecisions = extractDecisionItems(raw, 5);
awarenessItems = extractAwarenessItems(raw);
wsDb.close();
wsDb = null;
@@ -266,6 +256,10 @@ export function buildWorkspaceState(opts: BuildWorkspaceStateOpts): WorkspaceSta
return null;
}
// Awareness items live in waggle.db; failing to read them must not drop the
// memory-sourced state above.
try { awarenessItems = extractAwarenessItems(waggleDb, workspaceId); } catch { /* non-blocking */ }
// ── Session-sourced state ────────────────────────────────────────
const sessionsDir = path.join(dataDir, 'workspaces', workspaceId, 'sessions');