feat(core): give Waggle its own database, apart from memory (P1.5)

Waggle kept its own tables in the memory database. Before the memory
engine can be replaced by mem0 they need a home of their own.

- add WaggleDB (<dataDir>/waggle.db). Automations, notifications, held
  actions, skill hashes, thumbs feedback, cost tracking, the profile card
  and awareness items live there. Their stores take a WaggleDB, so passing
  a memory database is a type error.
- the profile card and awareness items are now per scope ('personal' or
  'workspace:<id>') in one table each, instead of one table per .mind file
- on the first start, copy these tables from personal.mind and every
  workspace.mind (importLegacyMindData). The legacy files are only read,
  each scope is copied once and a table only while its target is empty.
- copy the logger, injection scanner, external-memory ingress guard,
  WorkspaceManager and the embedders from @waggle/hive-mind-core into
  @waggle/core, which now re-exports its own copies
- point the server, agent orchestrator and CLI at waggle.db; update the
  tests that seeded these tables in a memory database

ADR-0007 records the layout and the licensing question about the copied
files.
This commit is contained in:
shumeiko.it
2026-10-23 14:31:28 +02:00
parent 20567ecb10
commit ad2ca875cb
65 changed files with 6131 additions and 201 deletions

View File

@@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { MindDB, AwarenessLayer, type Embedder } from '@waggle/core';
import { MindDB, WaggleDB, AwarenessLayer, type Embedder } from '@waggle/core';
import {
runLoopTick,
parseLoopSpec,
@@ -44,8 +44,8 @@ function framesIn(db: MindDB, gop: string): Array<{ content: string; source: str
}
/** Seed a prior-tick awareness row so the cost floor has a lastTickAt to read. */
function seedPriorTick(db: MindDB, scheduleId: number, lastTickAt: string) {
new AwarenessLayer(db).add('pending', `Loop ${scheduleId}`, 0, undefined, {
function seedPriorTick(awareness: AwarenessLayer, scheduleId: number, lastTickAt: string) {
awareness.add('pending', `Loop ${scheduleId}`, 0, undefined, {
status: `loop:${scheduleId}`, result: 'previous report', lastTickAt,
});
}
@@ -53,14 +53,19 @@ function seedPriorTick(db: MindDB, scheduleId: number, lastTickAt: string) {
describe('runLoopTick (Loop v0 — L1 report-only)', () => {
let tmpDir: string;
let db: MindDB;
let waggleDb: WaggleDB;
let awareness: AwarenessLayer;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-loop-'));
db = new MindDB(path.join(tmpDir, 'test.mind'));
waggleDb = new WaggleDB(':memory:');
awareness = new AwarenessLayer(waggleDb);
});
afterEach(() => {
db.close();
waggleDb.close();
fs.rmSync(tmpDir, { recursive: true, force: true });
});
@@ -76,7 +81,7 @@ describe('runLoopTick (Loop v0 — L1 report-only)', () => {
it('writes a report frame to the loop gop with source agent_inferred', async () => {
const chat = makeChat();
const res = await runLoopTick({ schedule: schedule(), mindDb: db, embedder, chat, log: silentLog });
const res = await runLoopTick({ schedule: schedule(), mindDb: db, awareness, embedder, chat, log: silentLog });
expect(res.skipped).toBeFalsy();
expect(res.wrote).toBe(true);
@@ -87,7 +92,7 @@ describe('runLoopTick (Loop v0 — L1 report-only)', () => {
});
it('records the judge score and surfaces it in the result', async () => {
const res = await runLoopTick({ schedule: schedule(), mindDb: db, embedder, chat: makeChat(), log: silentLog });
const res = await runLoopTick({ schedule: schedule(), mindDb: db, awareness, embedder, chat: makeChat(), log: silentLog });
// weighted 0.8*0.5 + 0.7*0.3 + 0.9*0.2 = 0.79 (length penalty 1 for a short report)
expect(res.score).toBeCloseTo(0.79, 2);
});
@@ -97,29 +102,29 @@ describe('runLoopTick (Loop v0 — L1 report-only)', () => {
// lastTickAt = now, which would otherwise throttle the second).
const noFloor = JSON.stringify({ prompt: 'Observe the pipeline.', minIntervalMs: 0 });
const chat1 = makeChat('First run: 3 deals at risk.');
await runLoopTick({ schedule: schedule({ job_config: noFloor }), mindDb: db, embedder, chat: chat1, log: silentLog });
await runLoopTick({ schedule: schedule({ job_config: noFloor }), mindDb: db, awareness, embedder, chat: chat1, log: silentLog });
// Prior state lives under the namespaced status key.
const items = new AwarenessLayer(db).getByStatus('loop:1');
// Prior state lives in waggle.db under the namespaced status key.
const items = new AwarenessLayer(waggleDb).getByStatus('loop:1');
expect(items).toHaveLength(1);
expect(new AwarenessLayer(db).parseMetadata(items[0]).result).toContain('First run: 3 deals at risk');
expect(new AwarenessLayer(waggleDb).parseMetadata(items[0]).result).toContain('First run: 3 deals at risk');
// Second tick: the maker prompt must carry the prior report.
const chat2 = makeChat('Second run: 1 new at-risk deal.');
await runLoopTick({ schedule: schedule({ job_config: noFloor }), mindDb: db, embedder, chat: chat2, log: silentLog });
await runLoopTick({ schedule: schedule({ job_config: noFloor }), mindDb: db, awareness, embedder, chat: chat2, log: silentLog });
const makerCall = chat2.mock.calls.find(c => !String(c[0]).includes('Return the JSON now'));
expect(makerCall?.[0]).toContain('First run: 3 deals at risk');
// State merged in place (still a single awareness item), result advanced.
const after = new AwarenessLayer(db).getByStatus('loop:1');
const after = new AwarenessLayer(waggleDb).getByStatus('loop:1');
expect(after).toHaveLength(1);
expect(new AwarenessLayer(db).parseMetadata(after[0]).result).toContain('Second run');
expect(new AwarenessLayer(waggleDb).parseMetadata(after[0]).result).toContain('Second run');
});
it('still reports when the judge output is unparseable (score undefined, no throw)', async () => {
// Judge returns junk → LLMJudge yields parsed:false, score stays undefined.
const chat = vi.fn(async () => 'not json at all');
const res = await runLoopTick({ schedule: schedule(), mindDb: db, embedder, chat, log: silentLog });
const res = await runLoopTick({ schedule: schedule(), mindDb: db, awareness, embedder, chat, log: silentLog });
expect(res.skipped).toBeFalsy();
expect(res.score).toBeUndefined();
expect(framesIn(db, 'loop')).toHaveLength(1); // the report itself is "not json at all" — still written
@@ -127,8 +132,8 @@ describe('runLoopTick (Loop v0 — L1 report-only)', () => {
it('throttles a tick whose last real run (awareness lastTickAt) is within the cost floor', async () => {
const chat = makeChat();
seedPriorTick(db, 1, new Date(Date.now() - 60_000).toISOString()); // ran 1 min ago
const res = await runLoopTick({ schedule: schedule(), mindDb: db, embedder, chat, log: silentLog });
seedPriorTick(awareness, 1, new Date(Date.now() - 60_000).toISOString()); // ran 1 min ago
const res = await runLoopTick({ schedule: schedule(), mindDb: db, awareness, embedder, chat, log: silentLog });
expect(res.skipped).toBe(true);
expect(res.reason).toBe('within min interval');
expect(chat).not.toHaveBeenCalled();
@@ -137,8 +142,8 @@ describe('runLoopTick (Loop v0 — L1 report-only)', () => {
it('runs when the last real run is older than the cost floor', async () => {
const chat = makeChat();
seedPriorTick(db, 1, new Date(Date.now() - (LOOP_MIN_INTERVAL_MS + 60_000)).toISOString());
const res = await runLoopTick({ schedule: schedule(), mindDb: db, embedder, chat, log: silentLog });
seedPriorTick(awareness, 1, new Date(Date.now() - (LOOP_MIN_INTERVAL_MS + 60_000)).toISOString());
const res = await runLoopTick({ schedule: schedule(), mindDb: db, awareness, embedder, chat, log: silentLog });
expect(res.skipped).toBeFalsy();
expect(chat).toHaveBeenCalled();
});
@@ -150,7 +155,7 @@ describe('runLoopTick (Loop v0 — L1 report-only)', () => {
// when last_run_at looks "just now".
const chat = makeChat();
const justNowSqlite = new Date().toISOString().slice(0, 19).replace('T', ' '); // 'YYYY-MM-DD HH:MM:SS'
const res = await runLoopTick({ schedule: schedule({ last_run_at: justNowSqlite }), mindDb: db, embedder, chat, log: silentLog });
const res = await runLoopTick({ schedule: schedule({ last_run_at: justNowSqlite }), mindDb: db, awareness, embedder, chat, log: silentLog });
expect(res.skipped).toBeFalsy();
expect(chat).toHaveBeenCalled();
});
@@ -159,7 +164,7 @@ describe('runLoopTick (Loop v0 — L1 report-only)', () => {
const chat = makeChat();
const res = await runLoopTick({
schedule: schedule({ job_config: '{}' }),
mindDb: db, embedder, chat, log: silentLog,
mindDb: db, awareness, embedder, chat, log: silentLog,
});
expect(res.skipped).toBe(true);
expect(res.reason).toBe('no prompt');
@@ -170,7 +175,7 @@ describe('runLoopTick (Loop v0 — L1 report-only)', () => {
const chat = makeChat();
const res = await runLoopTick({
schedule: schedule({ job_config: JSON.stringify({ prompt: 'observe', writeToMemory: false }) }),
mindDb: db, embedder, chat, log: silentLog,
mindDb: db, awareness, embedder, chat, log: silentLog,
});
expect(res.skipped).toBeFalsy();
expect(res.wrote).toBeFalsy();
@@ -237,11 +242,15 @@ describe('parseProposal', () => {
describe('runLoopTick — assist mode (L2 proposals)', () => {
let tmpDir: string;
let db: MindDB;
let waggleDb: WaggleDB;
let awareness: AwarenessLayer;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-loop-a-'));
db = new MindDB(path.join(tmpDir, 'test.mind'));
waggleDb = new WaggleDB(':memory:');
awareness = new AwarenessLayer(waggleDb);
});
afterEach(() => { db.close(); fs.rmSync(tmpDir, { recursive: true, force: true }); });
afterEach(() => { db.close(); waggleDb.close(); fs.rmSync(tmpDir, { recursive: true, force: true }); });
function assistChat(report: string, fence: string) {
return vi.fn(async (prompt: string) => {
@@ -255,7 +264,7 @@ describe('runLoopTick — assist mode (L2 proposals)', () => {
it('report mode never proposes an action', async () => {
const chat = vi.fn(async () => 'A plain report, no fence.');
const res = await runLoopTick({ schedule: sched({ prompt: 'sweep' }), mindDb: db, embedder, chat, log: silentLog });
const res = await runLoopTick({ schedule: sched({ prompt: 'sweep' }), mindDb: db, awareness, embedder, chat, log: silentLog });
expect(res.proposedAction).toBeUndefined();
});
@@ -263,7 +272,7 @@ describe('runLoopTick — assist mode (L2 proposals)', () => {
const chat = assistChat('Three deals went quiet.', '{"tool":"send_email","args":{"to":"x@y.z"},"summary":"nudge them"}');
const res = await runLoopTick({
schedule: sched({ prompt: 'sweep', mode: 'assist' }),
mindDb: db, embedder, chat, log: silentLog,
mindDb: db, awareness, embedder, chat, log: silentLog,
});
expect(res.proposedAction).toEqual({ tool: 'send_email', args: { to: 'x@y.z' }, summary: 'nudge them' });
expect(res.summary).toContain('Three deals went quiet');
@@ -276,7 +285,7 @@ describe('runLoopTick — assist mode (L2 proposals)', () => {
it('assist mode with a {none:true} proposal yields no action', async () => {
const chat = assistChat('Nothing actionable today.', '{"none":true}');
const res = await runLoopTick({ schedule: sched({ prompt: 'sweep', mode: 'assist' }), mindDb: db, embedder, chat, log: silentLog });
const res = await runLoopTick({ schedule: sched({ prompt: 'sweep', mode: 'assist' }), mindDb: db, awareness, embedder, chat, log: silentLog });
expect(res.proposedAction).toBeUndefined();
expect(res.summary).toContain('Nothing actionable');
});