refactor(memory): run agent recall, memory tools and the server on the memory contract (P1.5)

- Orchestrator, memory tools, context loader and pattern write-back use a
  MemoryStore and a workspace scope instead of MindDB layers; the knowledge
  graph tools, cognify, memory linker and feedback handler are gone (graph
  hidden in v1, ADR-0002).
- The server opens one mem0 store in <dataDir>/memory/, embedded by the
  in-process model on first use, and extracts facts about the user in the
  background with the model the user chose (memoryExtraction setting).
- Legacy .mind memories are copied into the store once, in the background;
  the legacy files are only read.
- Weaver consolidation, harvest, wiki, memory import, memory lanes, vector
  backfill and index reconciliation are removed; agent runs, loops, sub-agent
  results, connector fetch and the failed-turn capture write through the
  contract. Workspace sessions no longer hold a mind handle.
- FileIndexer and the install audit move to waggle.db; recall date helpers and
  the reranker are copied into core.
- The CLI REPL opens the same memory store.
This commit is contained in:
shumeiko.it
2026-10-24 11:56:26 +02:00
parent f3eb6b6652
commit 784ca770b1
86 changed files with 2520 additions and 8444 deletions

View File

@@ -17,7 +17,6 @@ import { fetchTeamServer } from './team-server-egress.js';
import { getStorageProvider } from './storage/index.js';
import { isLoopbackBind, resolveBindHost } from './net-config.js';
import { isLocalRequest } from './origin-guard.js';
import { MemoryWeaver } from '@waggle/weaver';
import {
Orchestrator,
createSystemTools,
@@ -67,7 +66,10 @@ import { PluginRuntimeManager, getStarterSkillsDir, validatePluginManifest } fro
import { MarketplaceDB, MarketplaceSync, seedMcpServers, seedNewSources } from '@waggle/marketplace';
import type { RiskLevel } from '@waggle/shared';
import { runConnectorFetch } from './connector-harvest.js';
import { writeAutoSyncSummaryFrame } from './harvest-autosync-frame.js';
import { openMemoryStore } from './memory-runtime.js';
import { MemoryExtractionQueue } from './memory-extraction.js';
import { importLegacyMemories, type LegacyMemorySource } from './legacy-memory-import.js';
import { PERSONAL_SCOPE, workspaceScope, type MemoryStore } from '@waggle/memory-contract';
import { runLoopTick } from './loop-executor.js';
import { enqueueHeldAction } from './held-action-executor.js';
import { workspaceRoutes } from './routes/workspaces.js';
@@ -77,11 +79,10 @@ import { memoryCenterRoutes } from './routes/memory-center.js';
import { artifactRoutes } from './routes/artifacts.js';
import { settingsRoutes } from './routes/settings.js';
import { embeddingRoutes, buildEmbeddingStatusPayload } from './routes/embedding.js';
import { sessionRoutes, findUndistilledSessions, markSessionDistilled } from './routes/sessions.js';
import { sessionRoutes } from './routes/sessions.js';
import { knowledgeRoutes } from './routes/knowledge.js';
import { litellmRoutes } from './routes/litellm.js';
import { runMemoryLaneExtraction } from './memory-lane-cron.js';
import { VectorEnrichmentService } from './services/vector-enrichment-service.js';
import { waggleDanceRoutes } from './routes/waggle-dance.js';
import { ingestRoutes, readFileRegistry } from './routes/ingest.js';
import { mindRoutes } from './routes/mind.js';
import { agentRoutes } from './routes/agent.js';
@@ -97,7 +98,6 @@ import { capabilitiesRoutes } from './routes/capabilities.js';
import { toolsRoutes } from './routes/tools.js';
import { externalToolRunRoutes } from './routes/external-tool-runs.js';
import { routeProposalRoutes } from './routes/route-proposals.js';
import { waggleDanceRoutes } from './routes/waggle-dance.js';
import { commandRoutes } from './routes/commands.js';
import { commandRoutes as commandCenterRoutes } from './routes/command.js';
import { homeRoutes } from './routes/home.js';
@@ -118,7 +118,6 @@ import { extendRoutes } from './routes/extend.js';
import { populateMcpRuntimeFromConfig } from './mcp-config.js';
import { scheduleMarketplaceBackgroundSync } from './marketplace-background-sync.js';
import { fleetRoutes } from './routes/fleet.js';
import { importRoutes } from './routes/import.js';
import { vaultRoutes } from './routes/vault.js';
import { personaRoutes } from './routes/personas.js';
import { feedbackRoutes } from './routes/feedback.js';
@@ -129,7 +128,6 @@ import { dataEraseRoutes } from './routes/data-erase.js';
import { costRoutes } from './routes/cost.js';
import { backupRoutes } from './routes/backup.js';
import { offlineRoutes } from './routes/offline.js';
import { weaverRoutes } from './routes/weaver.js';
import { eventRoutes, closeAuditDb, cleanupAuditEvents, emitAuditEvent } from './routes/events.js';
import { closeTeamsDb } from './routes/team.js';
import { pinRoutes } from './routes/pins.js';
@@ -159,8 +157,6 @@ import { agentGroupRoutes } from './routes/agent-groups.js';
import { localJobRoutes } from './routes/jobs.js';
import { agentEntityRoutes } from './routes/agents.js';
import { automationRoutes } from './routes/automations.js';
import { harvestRoutes } from './routes/harvest.js';
import { wikiRoutes } from './routes/wiki.js';
import { identityRoutes } from './routes/identity.js';
import { agentRunRoutes } from './routes/agent-run.js';
import { agentRunsRoutes } from './routes/agent-runs.js';
@@ -320,10 +316,7 @@ export interface AgentState {
* Create a fresh Orchestrator for a workspace session with the given mind mounted.
* Each session gets its own instance (Option Y fix from docs/plans/phase-a-the-room.md §9).
*/
createSessionOrchestrator: {
(): Orchestrator;
(workspaceMind: import('@waggle/core').MindDB): Orchestrator;
};
createSessionOrchestrator: (workspaceId?: string) => Orchestrator;
/**
* Build the full tool pool for a session, using the session's orchestrator for mind tools.
* When `sourceWorkspaceId` is provided, also injects Phase B.2 cross-workspace read tools
@@ -366,10 +359,6 @@ export interface AgentState {
wsSessionToken: string;
/** SHA-256 hash of the paired Browser Companion credential, or null when unpaired. */
browserCompanionCredentialHash: string | null;
/** Memory-weaver run timestamps for the personal mind. */
weaverState: { lastPersonalConsolidation: string | null; lastPersonalDecay: string | null };
/** Per-workspace memory-weaver run timestamps, keyed by workspace ID. */
workspaceWeaverStatus: Record<string, { lastConsolidation: string | null }>;
}
declare module 'fastify' {
@@ -383,6 +372,10 @@ declare module 'fastify' {
auditStore: import('@waggle/core').InstallAuditStore;
cronStore: import('@waggle/core').CronStore;
waggleDb: import('@waggle/core').WaggleDB;
/** Memory (ADR-0007): personal and workspace memories, reached only through the contract. */
memory: MemoryStore;
/** Background fact extraction from chat turns (ADR-0007, decision 6). */
memoryExtraction: MemoryExtractionQueue;
traceStore: import('@waggle/core').ExecutionTraceStore;
/** Active behavioral spec — the compiled `BEHAVIORAL_SPEC` baseline. */
activeBehavioralSpec: ReturnType<typeof import('@waggle/agent').buildActiveBehavioralSpec>;
@@ -528,7 +521,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
}
// Install audit store — persistent trail for capability install events
const auditStore = new InstallAuditStore(multiMind.personal);
const auditStore = new InstallAuditStore(waggleDb);
server.decorate('auditStore', auditStore);
// Cron store — SQLite-backed schedule persistence for Solo
@@ -710,11 +703,33 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
const waggleHome = fullConfig.dataDir || path.join(os.homedir(), '.waggle');
const rerankerCacheDir = path.join(waggleHome, 'models', 'reranker');
// Orchestrator — connects to personal .mind
// Memory (ADR-0007): one mem0 store in <dataDir>/memory/, embedded by the
// in-process model. Facts are extracted with the model the user chose,
// through the built-in proxy (or LiteLLM when the user runs it).
const memory = await openMemoryStore({
dataDir: fullConfig.dataDir,
modelCacheDir: embeddingConfig.inprocess?.cacheDir ?? path.join(waggleHome, 'models'),
embeddingModel: embeddingConfig.inprocess?.model,
extraction: {
baseURL: fullConfig.useBuiltInProxy
? `http://127.0.0.1:${fullConfig.port}/v1`
: fullConfig.litellmUrl,
apiKey: litellmApiKey,
model: () => server.agentState?.currentModel || 'claude-sonnet-4-6',
},
});
server.decorate('memory', memory);
const memoryExtraction = new MemoryExtractionQueue({
memory,
isEnabled: () => new WaggleConfig(fullConfig.dataDir).getMemoryExtraction(),
onError: (err) => log.warn(`[memory] fact extraction failed: ${err instanceof Error ? err.message : String(err)}`),
});
server.decorate('memoryExtraction', memoryExtraction);
// Orchestrator — personal memory; a workspace is added with setWorkspace()
const orchestrator = new Orchestrator({
db: multiMind.personal,
memory,
waggleDb,
embedder,
mode: 'local',
version: '0.4',
rerankerCacheDir,
@@ -858,27 +873,18 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
// main-agent-visible return path stays intact.
try {
const wsId = server.agentState.activeWorkspaceId;
const wsDb = wsId ? mindCache.getOrOpen(wsId) : null;
const targetDb = wsDb ?? multiMind.personal;
const mindLabel = wsDb ? `workspace:${wsId}` : 'personal';
const scope = wsId ? workspaceScope(wsId) : PERSONAL_SCOPE;
// Structured content: tag provenance so search can filter
// for sub-agent-derived frames and so humans can see where it came from.
// Structured content: tag provenance so humans can see where it came from.
const content = `[Sub-agent: ${result.role}] ${result.agentName}\n\n${result.response}\n\n— tools: ${result.toolsUsed.join(', ') || 'none'} · ${(result.duration / 1000).toFixed(1)}s · ${result.usage.inputTokens + result.usage.outputTokens} tokens`;
// Use raw frames — cognify requires a per-mind pipeline we don't
// construct here; the decayFrames weaver will pick these up later.
const frames = new FrameStore(targetDb);
const sessions = new SessionStore(targetDb);
const active = sessions.getActive();
const gopId = active.length > 0 ? active[0].gop_id : sessions.create().gop_id;
const latestI = frames.getLatestIFrame(gopId);
if (latestI) {
frames.createPFrame(gopId, content, latestI.id, 'normal', 'agent_inferred');
} else {
frames.createIFrame(gopId, content, 'normal', 'agent_inferred');
}
log.debug(`Gap L: persisted sub-agent result ${result.agentId} to ${mindLabel} mind`);
await memory.remember({
scope,
text: content,
kind: 'note',
source: 'agent_inferred',
origin: 'sub-agent',
});
log.debug(`Gap L: persisted sub-agent result ${result.agentId} to ${wsId ? `workspace:${wsId}` : 'personal'} memory`);
} catch (err) {
log.debug('Gap L: sub-agent persistence failed (non-blocking)', err);
}
@@ -1200,16 +1206,15 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
*
* See docs/plans/phase-a-the-room.md §9 for the Option X vs Option Y decision.
*/
const createSessionOrchestrator = (workspaceMind?: MindDB): Orchestrator => {
const createSessionOrchestrator = (workspaceId?: string): Orchestrator => {
const sessionOrch = new Orchestrator({
db: multiMind.personal,
memory,
waggleDb,
embedder,
mode: 'local',
version: '0.4',
rerankerCacheDir,
});
if (workspaceMind) sessionOrch.setWorkspaceMind(workspaceMind);
if (workspaceId) sessionOrch.setWorkspace(workspaceId);
return sessionOrch;
};
@@ -1231,8 +1236,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
const crossTools = createCrossWorkspaceTools({
sourceWorkspaceId,
embedder,
getMindForWorkspace: (id) => mindCache.getOrOpen(id),
memory,
listWorkspaces: () => wsManager.list().map(w => ({ id: w.id, name: w.name })),
listWorkspaceFiles: async (id, subPath) => {
try {
@@ -1268,14 +1272,6 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
allowedRoot: path.join(fullConfig.dataDir, 'workspaces'),
});
server.decorate('mindCache', mindCache);
const vectorEnrichmentService = new VectorEnrichmentService({
personalMind: multiMind.personal,
embeddingProvider,
listWorkspaceIds: () => wsManager.list().map(workspace => workspace.id),
acquireWorkspaceMind: workspaceId => mindCache.acquire(workspaceId),
releaseWorkspaceMind: workspaceId => mindCache.release(workspaceId),
log: (level, message) => log[level](message),
});
let activeWorkspaceId: string | null = null;
const setActiveWorkspaceId = (workspaceId: string | null): void => {
activeWorkspaceId = workspaceId;
@@ -1283,271 +1279,16 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
if (decorated) decorated.activeWorkspaceId = workspaceId;
};
// ── TeamSync cache — one TeamSync instance per team workspace ──
const teamSyncCache = new Map<string, { sync: TeamSync; binding: string }>();
const serializeTeamSyncBinding = (
teamServer: NonNullable<ReturnType<WaggleConfig['getTeamServer']>>,
teamId: string,
): string => JSON.stringify({
url: teamServer.url,
teamId,
token: crypto.createHash('sha256').update(teamServer.token ?? '').digest('hex'),
userId: teamServer.userId ?? 'local-user',
displayName: teamServer.displayName ?? 'You',
});
function getTeamSync(workspaceId: string, wsConfig: WorkspaceConfig | null, waggleConfig: WaggleConfig): TeamSync | null {
if (!wsConfig?.teamId || !wsConfig?.teamServerUrl) {
teamSyncCache.delete(workspaceId);
return null;
}
const teamServer = getBoundTeamServer(wsConfig.teamServerUrl, waggleConfig.getTeamServer());
if (!teamServer?.token) {
teamSyncCache.delete(workspaceId);
return null;
}
const binding = serializeTeamSyncBinding(teamServer, wsConfig.teamId);
const cached = teamSyncCache.get(workspaceId);
if (cached?.binding === binding) return cached.sync;
const sync = new TeamSync({
teamServerUrl: teamServer.url,
teamSlug: wsConfig.teamId, // teamId is used as slug
authToken: teamServer.token,
userId: teamServer.userId ?? 'local-user',
displayName: teamServer.displayName ?? 'You',
}, fetchTeamServer);
const pushFrame = sync.pushFrame.bind(sync);
sync.pushFrame = async (frame) => {
const currentWorkspace = wsManager.get(workspaceId);
const currentWaggleConfig = new WaggleConfig(fullConfig.dataDir);
const currentTeamServer = getBoundTeamServer(
currentWorkspace?.teamServerUrl,
currentWaggleConfig.getTeamServer(),
);
const currentBinding = currentWorkspace?.teamId && currentTeamServer?.token
? serializeTeamSyncBinding(currentTeamServer, currentWorkspace.teamId)
: null;
const cached = teamSyncCache.get(workspaceId);
if (cached?.sync !== sync) {
return cached && currentBinding === cached.binding
? cached.sync.pushFrame(frame)
: null;
}
if (currentBinding === binding) return pushFrame(frame);
teamSyncCache.delete(workspaceId);
if (currentWorkspace?.teamId && currentTeamServer?.token) {
const replacement = getTeamSync(workspaceId, currentWorkspace, currentWaggleConfig);
if (replacement) {
if (activeWorkspaceId === workspaceId) orchestrator.setTeamSync(replacement);
return replacement.pushFrame(frame);
}
}
if (activeWorkspaceId === workspaceId) orchestrator.setTeamSync(null);
return null;
};
teamSyncCache.set(workspaceId, { sync, binding });
return sync;
}
/**
* Activate a workspace's .mind file on the orchestrator.
* Opens the mind if not cached, then calls setWorkspaceMind().
*/
/** Make a workspace's memory active on the shared orchestrator. */
const activateWorkspaceMind = (workspaceId: string): boolean => {
if (activeWorkspaceId === workspaceId) return true; // already active
if (!wsManager.get(workspaceId)) return false;
const wsDb = mindCache.getOrOpen(workspaceId);
if (!wsDb) return false;
orchestrator.setWorkspaceMind(wsDb);
orchestrator.setWorkspace(workspaceId);
setActiveWorkspaceId(workspaceId);
return true;
};
// ── Memory Weaver — background consolidation & decay (A1 fix) ────
// Runs on the personal mind at startup. Workspace minds get weavers when activated.
const personalFrames = new FrameStore(multiMind.personal);
const personalSessions = new SessionStore(multiMind.personal);
const personalWeaver = new MemoryWeaver(multiMind.personal, personalFrames, personalSessions);
const weaverTimers: NodeJS.Timeout[] = [];
const workspaceWeavers = new Map<string, { weaver: MemoryWeaver; timers: NodeJS.Timeout[] }>();
// F2: Track weaver last-run timestamps for status endpoint
const weaverState: { lastPersonalConsolidation: string | null; lastPersonalDecay: string | null } = {
lastPersonalConsolidation: null,
lastPersonalDecay: null,
};
const workspaceWeaverStatus: Record<string, { lastConsolidation: string | null }> = {};
// Run personal mind weaver on intervals
const runPersonalConsolidation = () => {
try {
const active = personalSessions.getActive();
for (const s of active) personalWeaver.consolidateGop(s.gop_id);
weaverState.lastPersonalConsolidation = new Date().toISOString();
} catch (err) { log.debug('Personal consolidation skipped', err); }
};
const runPersonalDecay = () => {
try {
personalWeaver.decayFrames();
personalWeaver.strengthenFrames();
weaverState.lastPersonalDecay = new Date().toISOString();
} catch (err) { log.debug('Personal decay skipped', err); }
};
weaverTimers.push(setInterval(runPersonalConsolidation, 60 * 60 * 1000)); // hourly
weaverTimers.push(setInterval(runPersonalDecay, 24 * 60 * 60 * 1000)); // daily
// ── Harvest auto-sync — poll stale sources every 30 min ──
const runHarvestAutoSync = async () => {
try {
const harvestStore = new HarvestSourceStore(multiMind.personal);
const stale = harvestStore.getStale();
if (stale.length === 0) return;
const { ClaudeCodeAdapter: CCA } = await import('@waggle/core');
const personalSessionStore = new SessionStore(multiMind.personal);
personalSessionStore.ensure('harvest', 'harvest', 'Imported memory from external sources');
const personalFrameStore = new FrameStore(multiMind.personal);
// #7 sticky erasure: don't re-materialize an erased subject on auto-sync.
const suppression = new SuppressionStore(multiMind.personal);
for (const src of stale) {
try {
// Only auto-sync filesystem-scannable sources
if (src.source !== 'claude-code') continue;
const homedir = (await import('node:os')).default.homedir();
const scanDir = (await import('node:path')).default.join(homedir, '.claude');
const adapter = new CCA();
const items = adapter.scan(scanDir);
if (items.length === 0) continue;
let saved = 0;
let couldNotVerify = 0;
let skippedUnsafe = 0;
for (const item of items) {
// #7 sticky erasure: distinguish a confirmed erasure MATCH from a
// fail-closed read ERROR — both skip the write, but a broken-DB read
// must be reported as "could not verify", not silently as "erased".
const supp = suppression.checkSuppressed(item.source, item.id);
if (supp.suppressed) {
if (supp.reason === 'error') couldNotVerify++;
continue; // #7 Art.17 erased (or unverifiable — fail-closed skip)
}
// #7 Art.17: stamp the subject key (metadata.sourceId) so a subject-mode
// DSAR can reach this auto-synced summary — shared with the cron path.
if (!writeAutoSyncSummaryFrame(personalFrameStore, item)) {
skippedUnsafe++;
continue;
}
saved++;
}
// R3-004: store the content digest so the manual harvest route can
// skip an unchanged re-scan on the next sync.
harvestStore.recordSync(src.source, items.length, saved, harvestSetHash(items));
log.info(`[harvest-auto-sync] ${src.source}: imported ${saved} items`
+ (couldNotVerify > 0 ? ` (${couldNotVerify} could not be verified against the erasure list — skipped, fail-closed)` : '')
+ (skippedUnsafe > 0 ? ` (${skippedUnsafe} unsafe items skipped)` : ''));
} catch (err) {
log.debug(`[harvest-auto-sync] ${src.source} failed:`, err);
}
}
} catch (err) {
log.debug('[harvest-auto-sync] skipped:', err);
}
};
weaverTimers.push(setInterval(runHarvestAutoSync, 30 * 60 * 1000)); // every 30 min
// Extend activateWorkspaceMind to also start a weaver for the workspace
// and distill any undistilled sessions into durable memory frames.
const baseActivateWorkspaceMind = activateWorkspaceMind;
const activateWorkspaceMindWithWeaver = (workspaceId: string): boolean => {
const result = baseActivateWorkspaceMind(workspaceId);
// E4: Workspace topic tracking REMOVED — was saving to personal mind, causing
// cross-workspace leakage. Workspace names are available from workspace config
// and don't need to be duplicated into memory frames.
if (result && !workspaceWeavers.has(workspaceId)) {
const wsDb = mindCache.getIfOpen(workspaceId);
if (wsDb) {
const wsFrames = new FrameStore(wsDb);
const wsSessions = new SessionStore(wsDb);
const wsWeaver = new MemoryWeaver(wsDb, wsFrames, wsSessions);
const timers: NodeJS.Timeout[] = [];
timers.push(setInterval(() => {
try {
const active = wsSessions.getActive();
for (const s of active) wsWeaver.consolidateGop(s.gop_id);
} catch { /* non-blocking */ }
}, 60 * 60 * 1000));
timers.push(setInterval(() => {
try { wsWeaver.decayFrames(); wsWeaver.strengthenFrames(); } catch (err) { log.debug('Workspace memory decay skipped', err); }
}, 24 * 60 * 60 * 1000));
workspaceWeavers.set(workspaceId, { weaver: wsWeaver, timers });
// E2: Distill undistilled sessions into durable memory frames on activation
try {
const sessionsDir = path.join(fullConfig.dataDir, 'workspaces', workspaceId, 'sessions');
const undistilled = findUndistilledSessions(sessionsDir);
for (const session of undistilled) {
const distilled = wsWeaver.distillSessionContent(session.date, session.summary, session.keyPoints);
if (distilled) markSessionDistilled(session.filePath);
}
} catch (err) { log.debug('Workspace distillation skipped', err); }
}
}
// After mind activation for team workspaces — pull remote frames
if (result) {
const wsConfig = wsManager.get(workspaceId);
if (wsConfig?.teamId) {
const sync = getTeamSync(workspaceId, wsConfig, new WaggleConfig(fullConfig.dataDir));
if (sync) {
// Set TeamSync on the orchestrator for push-on-write
orchestrator.setTeamSync(sync);
// Pull team frames into local workspace mind
const wsDb = mindCache.getIfOpen(workspaceId);
if (wsDb) {
const wsSessionStore = new SessionStore(wsDb);
wsSessionStore.ensure('team-sync', 'team-sync', 'Frames synced from team server');
const wsFrameStore = new FrameStore(wsDb);
sync.pullFrames(sync.getLastSyncTimestamp() ?? undefined).then(pulled => {
if (pulled.length === 0) return;
let inserted = 0;
for (const f of pulled) {
try {
// Provenance is carried in the content prefix ([Team:author]); the source
// column must be schema-valid, so use 'import' (these frames are imported
// from the team server). Previously passed 'team_sync', which violated the
// memory_frames.source CHECK → every pulled frame was silently skipped.
wsFrameStore.createIFrame('team-sync', `[Team:${f.authorName}] ${f.content}`, f.importance, 'import');
inserted++;
} catch { /* duplicate or constraint — skip */ }
}
if (inserted > 0) {
log.info(`TeamSync: pulled ${inserted} new frames for workspace ${workspaceId}`);
}
}).catch(err => {
log.warn(`TeamSync pull failed:`, err.message);
});
}
} else {
orchestrator.setTeamSync(null);
}
} else {
// Non-team workspace — clear TeamSync
orchestrator.setTeamSync(null);
}
}
return result;
};
// Helper: get a cached workspace MindDB (opens on demand)
// A6: Close and remove workspace mind DB from cache before deletion
const closeWorkspaceMind = (workspaceId: string): void => {
@@ -1559,10 +1300,8 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
// and this seam closed the same MindDB.
sessionManager.close(workspaceId);
mindCache.close(workspaceId);
teamSyncCache.delete(workspaceId);
if (activeWorkspaceId === workspaceId) {
orchestrator.setTeamSync(null);
orchestrator.clearWorkspaceMind();
orchestrator.setWorkspace(null);
setActiveWorkspaceId(null);
}
};
@@ -1590,13 +1329,11 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
createSessionOrchestrator,
buildToolsForSession,
bindWorkspaceCollaborationTools,
activateWorkspaceMind: activateWorkspaceMindWithWeaver,
activateWorkspaceMind: activateWorkspaceMind,
getWorkspaceMindDb,
closeWorkspaceMind,
listWorkspaces: () => wsManager.list().map(w => ({ id: w.id, name: w.name })),
activeWorkspaceId,
weaverState,
workspaceWeaverStatus,
subagentOrchestrator: null,
pluginRuntimeManager,
mcpRuntime,
@@ -1611,7 +1348,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
wsSessionToken,
browserCompanionCredentialHash,
});
activateWorkspaceMindWithWeaver(defaultWorkspaceId);
activateWorkspaceMind(defaultWorkspaceId);
// Wire up skill hot-reload callback
const reloadSkills = (fresh: LoadedSkill[]) => {
@@ -1642,33 +1379,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
case 'memory_consolidation': {
// Check if this is a marketplace sync (uses memory_consolidation type for system-level scheduling)
const mcJobConfig = JSON.parse(schedule.job_config || '{}');
if (mcJobConfig.action === 'index_reconcile') {
try {
// Reconcile personal mind FTS/vec indexes
const result = await reconcileIndexes(multiMind.personal);
if (result.ftsFixed > 0 || result.vecFixed > 0) {
log.info(`[cron] Index reconciliation: FTS=${result.ftsFixed} vec=${result.vecFixed} fixed (personal)`);
}
// Reconcile workspace minds
let ftsTotal = result.ftsFixed;
let vecTotal = result.vecFixed;
const workspaces = wsManager.list();
for (const ws of workspaces) {
const wsDb = getWorkspaceMindDb(ws.id);
if (wsDb) {
const wsResult = await reconcileIndexes(wsDb);
if (wsResult.ftsFixed > 0 || wsResult.vecFixed > 0) {
log.info(`[cron] Index reconciliation: FTS=${wsResult.ftsFixed} vec=${wsResult.vecFixed} fixed (workspace "${ws.name}")`);
}
ftsTotal += wsResult.ftsFixed;
vecTotal += wsResult.vecFixed;
}
}
dreamJournal.record('index_reconcile', { ftsFixed: ftsTotal, vecFixed: vecTotal });
} catch (err) {
log.warn(`[cron] Index reconciliation failed: ${(err as Error).message}`);
}
} else if (mcJobConfig.action === 'marketplace_sync' && marketplaceDb) {
if (mcJobConfig.action === 'marketplace_sync' && marketplaceDb) {
try {
const vaultLookup = server.vault ? (key: string) => server.vault!.get(key)?.value ?? null : undefined;
const sync = new MarketplaceSync(marketplaceDb, vaultLookup, (url, init) => safeFetch(url, init));
@@ -1688,178 +1399,11 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
} catch (err) {
log.warn(`[cron] Marketplace sync failed: ${(err as Error).message}`);
}
} else if (mcJobConfig.action === 'harvest_sync') {
// Phase 7: Memory Harvest — scan registered filesystem sources for new data.
// Only filesystem-based sources (currently claude-code) can auto-sync; API-based
// sources like chatgpt/gemini require user-uploaded exports.
try {
// Frames have a FK to sessions(gop_id) — ensure the stable
// `harvest` session row exists before creating any frames.
new SessionStore(multiMind.personal).ensure('harvest', 'harvest', 'Imported memory from external sources');
const harvestStore = new HarvestSourceStore(multiMind.personal);
const personalFrames = new FrameStore(multiMind.personal);
// #7 sticky erasure: don't re-materialize an erased subject on cron sync.
const suppression = new SuppressionStore(multiMind.personal);
const stale = harvestStore.getStale();
let totalItems = 0;
let totalFrames = 0;
let sourcesScanned = 0;
let totalCouldNotVerify = 0;
let totalSkippedUnsafe = 0;
for (const src of stale) {
if (src.source !== 'claude-code' || !src.sourcePath) continue;
try {
const adapter = new ClaudeCodeAdapter();
const items = adapter.scan(src.sourcePath);
let saved = 0;
for (const item of items) {
// #7 sticky erasure: a fail-closed read ERROR is skipped like a
// MATCH but tallied separately (report "could not verify", not "erased").
const supp = suppression.checkSuppressed(item.source, item.id);
if (supp.suppressed) {
if (supp.reason === 'error') totalCouldNotVerify++;
continue; // #7 Art.17 erased (or unverifiable — fail-closed skip)
}
// #7 Art.17: stamp the subject key (metadata.sourceId) so a
// subject-mode DSAR reaches this cron-synced summary — shared helper.
if (!writeAutoSyncSummaryFrame(personalFrames, item)) {
totalSkippedUnsafe++;
continue;
}
saved++;
}
// R3-004: store the content digest for next-sync skip.
harvestStore.recordSync(src.source, items.length, saved, harvestSetHash(items));
totalItems += items.length;
totalFrames += saved;
sourcesScanned++;
} catch (innerErr) {
log.warn(`[cron] Harvest sync: source "${src.source}" failed: ${(innerErr as Error).message}`);
}
}
if (sourcesScanned > 0) {
log.info(`[cron] Harvest sync: ${totalFrames} frames from ${totalItems} items across ${sourcesScanned} source(s)`
+ (totalCouldNotVerify > 0 ? ` (${totalCouldNotVerify} could not be verified against the erasure list — skipped, fail-closed)` : '')
+ (totalSkippedUnsafe > 0 ? ` (${totalSkippedUnsafe} unsafe items skipped)` : ''));
}
dreamJournal.record('harvest_sync', {
framesSaved: totalFrames,
itemsScanned: totalItems,
sourcesScanned,
couldNotVerify: totalCouldNotVerify,
});
} catch (err) {
log.warn(`[cron] Harvest sync failed: ${(err as Error).message}`);
}
} else if (mcJobConfig.action === 'memory_compact') {
// Phase 9a: Memory compaction — prune temporary/deprecated frames, merge stale P-frames.
// Runs across personal mind and every workspace mind.
try {
const personalResult = new FrameStore(multiMind.personal).compact();
let wsTempPruned = 0;
let wsDepPruned = 0;
let wsMerged = 0;
for (const ws of wsManager.list()) {
const wsDb = getWorkspaceMindDb(ws.id);
if (!wsDb) continue;
try {
const r = new FrameStore(wsDb).compact();
wsTempPruned += r.temporaryPruned;
wsDepPruned += r.deprecatedPruned;
wsMerged += r.pframesMerged;
} catch (innerErr) {
log.warn(`[cron] Memory compaction: workspace "${ws.name}" failed: ${(innerErr as Error).message}`);
}
}
dreamJournal.record('memory_compact', {
temporaryPruned: personalResult.temporaryPruned + wsTempPruned,
deprecatedPruned: personalResult.deprecatedPruned + wsDepPruned,
pframesMerged: personalResult.pframesMerged + wsMerged,
});
const total =
personalResult.temporaryPruned + personalResult.deprecatedPruned + personalResult.pframesMerged +
wsTempPruned + wsDepPruned + wsMerged;
if (total > 0) {
log.info(
`[cron] Memory compaction: personal(temp=${personalResult.temporaryPruned}, dep=${personalResult.deprecatedPruned}, merged=${personalResult.pframesMerged}) ` +
`workspaces(temp=${wsTempPruned}, dep=${wsDepPruned}, merged=${wsMerged})`
);
}
} catch (err) {
log.warn(`[cron] Memory compaction failed: ${(err as Error).message}`);
}
} else if (mcJobConfig.action === 'memory_lane_extract') {
// W4.3d: extraction of the benchmark-proven recall lanes
// (facts/events/profiles) over frames written since the last run.
// Routes through the built-in proxy ('fast' tier = haiku) — the
// same internal-LLM pattern as the other cron LLM calls.
try {
const llmCall = async (prompt: string, _model: 'fast' | 'accurate'): Promise<string> => {
const proxyUrl = `http://127.0.0.1:${fullConfig.port ?? 3333}/v1/chat/completions`;
const res = await fetch(proxyUrl, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
signal: AbortSignal.timeout(90_000),
body: JSON.stringify({
model: 'claude-haiku-4-5',
max_tokens: 4096,
messages: [{ role: 'user', content: prompt }],
}),
});
if (!res.ok) throw new Error(`lane-extract LLM HTTP ${res.status}`);
const data = await res.json() as unknown;
return parseOpenAiTextCompletion(data).content;
};
const minds: Array<{ label: string; db: import('@waggle/core').MindDB }> = [
{ label: 'personal', db: multiMind.personal },
];
for (const ws of wsManager.list()) {
const wsDb = getWorkspaceMindDb(ws.id);
if (wsDb) minds.push({ label: `workspace "${ws.name}"`, db: wsDb });
}
let laneFramesProcessed = 0;
let laneFacts = 0;
let laneEvents = 0;
let laneProfiles = 0;
for (const mind of minds) {
try {
const r = await runMemoryLaneExtraction(mind.db, llmCall);
if (!r.skipped) {
if (r.errors.length > 0) {
log.warn(
`[cron] Memory lanes (${mind.label}) incomplete; watermark held for retry: ` +
r.errors.join('; ').slice(0, 200)
);
continue;
}
log.info(
`[cron] Memory lanes (${mind.label}): ${r.framesProcessed} frames → ` +
`facts=${r.written?.factsWritten ?? 0} events=${r.written?.eventsWritten ?? 0} ` +
`profiles=${r.written?.profilesWritten ?? 0}` +
(r.errors.length ? ` (errors: ${r.errors.join('; ').slice(0, 200)})` : '')
);
laneFramesProcessed += r.framesProcessed;
laneFacts += r.written?.factsWritten ?? 0;
laneEvents += r.written?.eventsWritten ?? 0;
laneProfiles += r.written?.profilesWritten ?? 0;
}
} catch (innerErr) {
log.warn(`[cron] Memory lanes (${mind.label}) failed: ${(innerErr as Error).message}`);
}
}
dreamJournal.record('memory_lane_extract', {
framesProcessed: laneFramesProcessed,
factsWritten: laneFacts,
eventsWritten: laneEvents,
profilesWritten: laneProfiles,
});
} catch (err) {
log.warn(`[cron] Memory lane extraction failed: ${(err as Error).message}`);
}
} else {
// Normal memory consolidation
try { runPersonalConsolidation(); } catch { /* non-blocking */ }
}
// index_reconcile, harvest_sync, memory_compact, memory_lane_extract and
// plain consolidation maintained the legacy memory engine. mem0 indexes
// on write and background extraction replaces the lanes (ADR-0007), so
// schedules left over from the legacy data only run the retention below.
// UX-Refactor Phase 3: cron_execution_history retention. recordExecution
// now writes a row per tick (Journey 16), so prune >30-day rows on the
// same nightly consolidation cadence (mirrors optStore.pruneOlderThan).
@@ -1870,25 +1414,8 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
break;
}
case 'workspace_health':
// Log stale frame count per workspace as awareness flag
try {
const workspaces = wsManager.list();
for (const ws of workspaces) {
const wsDb = getWorkspaceMindDb(ws.id);
if (wsDb) {
const wsFrames = new FrameStore(wsDb);
const staleCount = wsFrames.list({ limit: 100 })
.filter(f => {
const age = Date.now() - new Date(f.last_accessed).getTime();
return age > 7 * 24 * 60 * 60 * 1000;
}).length;
if (staleCount > 0) {
const wsAwareness = new AwarenessLayer(waggleDb, workspaceDataScope(ws.id));
wsAwareness.add('flag', `Workspace "${ws.name}" has ${staleCount} stale memory frames (>7 days untouched)`, 0);
}
}
}
} catch { /* non-blocking */ }
// Flagged memories untouched for a week; mem0 does not track access,
// so there is nothing to flag (ADR-0007).
break;
case 'proactive': {
const proactiveConfig = JSON.parse(schedule.job_config || '{}');
@@ -2007,7 +1534,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
for (const target of targetWorkspaces) {
try {
// Activate workspace mind so agent has context
activateWorkspaceMindWithWeaver(target.id);
activateWorkspaceMind(target.id);
if (taskConfig.mode === 'ai_task') {
// #17: full agent turn through the loopback chat route —
@@ -2115,12 +1642,18 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
// Auto-fetch (§3.C): pull fresh data from opted-in connectors into the
// personal mind. Cost-free — raw frames, no LLM extraction. Free (Solo).
try {
new SessionStore(multiMind.personal).ensure('harvest', 'harvest', 'Imported memory from external sources');
const personalFrames = new FrameStore(multiMind.personal);
const statePath = path.join(fullConfig.dataDir, 'connector-harvest-state.json');
const r = await runConnectorFetch({
connectors: connectorRegistry.getConnected(),
writeFrame: (content) => { personalFrames.createIFrame('harvest', content, 'normal', 'import'); },
writeFrame: async (content) => {
await memory.remember({
scope: PERSONAL_SCOPE,
text: content,
kind: 'note',
source: 'import',
origin: 'connector-fetch',
});
},
loadState: () => {
try {
const raw = JSON.parse(fs.readFileSync(statePath, 'utf-8')) as Partial<{ lastFetchedAt: string; hashes: Record<string, string> }>;
@@ -2154,12 +1687,12 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
// here so the scheduler records the failure and auto-disables after
// repeated failures (recall/write failures are caught inside runLoopTick).
const wsId = schedule.workspace_id;
const mindDb = wsId && wsId !== '*' ? getWorkspaceMindDb(wsId) : multiMind.personal;
if (!mindDb) {
const loopWorkspace = wsId && wsId !== '*' ? wsId : null;
if (loopWorkspace && !wsManager.get(loopWorkspace)) {
log.warn(`[cron] loop "${schedule.name}" references unknown workspace "${wsId}" — skipping`);
break;
}
if (wsId && wsId !== '*') activateWorkspaceMindWithWeaver(wsId);
if (wsId && wsId !== '*') activateWorkspaceMind(wsId);
const loopModel = server.agentState.currentModel ?? 'claude-sonnet-4-6';
const loopProxyUrl = `http://127.0.0.1:${fullConfig.port ?? 3333}/v1/chat/completions`;
const loopChat = async (prompt: string, maxTokens = 2048): Promise<string> => {
@@ -2178,7 +1711,14 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
return parseOpenAiTextCompletion(body).content;
};
const loopAwareness = new AwarenessLayer(waggleDb, wsId && wsId !== '*' ? workspaceDataScope(wsId) : 'personal');
const loopResult = await runLoopTick({ schedule, mindDb, awareness: loopAwareness, embedder, chat: loopChat, log });
const loopResult = await runLoopTick({
schedule,
memory,
scope: loopWorkspace ? workspaceScope(loopWorkspace) : PERSONAL_SCOPE,
awareness: loopAwareness,
chat: loopChat,
log,
});
if (!loopResult.skipped) {
emitNotification(server, {
title: `Loop: ${schedule.name}`,
@@ -2500,7 +2040,6 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
await server.register(mcpRoutes);
await server.register(extendRoutes);
await server.register(fleetRoutes);
await server.register(importRoutes);
await server.register(vaultRoutes);
await server.register(personaRoutes);
await server.register(feedbackRoutes);
@@ -2510,7 +2049,6 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
await server.register(costRoutes);
await server.register(backupRoutes);
await server.register(offlineRoutes);
await server.register(weaverRoutes);
await server.register(eventRoutes);
await server.register(workflowRoutes);
await server.register(pinRoutes);
@@ -2569,8 +2107,6 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
await server.register(telemetryRoutes);
await server.register(agentGroupRoutes);
await server.register(localJobRoutes);
await server.register(harvestRoutes);
await server.register(wikiRoutes);
await server.register(identityRoutes);
await server.register(agentRunRoutes);
await server.register(agentRunsRoutes);
@@ -2887,7 +2423,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
}
const dbHealthy = (() => {
try {
multiMind.personal.getDatabase().prepare('SELECT 1').get();
waggleDb.getDatabase().prepare('SELECT 1').get();
return true;
} catch {
return false;
@@ -2915,35 +2451,23 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
? 'unavailable'
: 'degraded';
// Memory stats — frame count + mind file size
const memoryStats = (() => {
// Memory stats — memory count and the size of the memory directory.
// mem0 embeds every memory on write, so coverage is complete by design.
const memoryStats = await (async () => {
const embeddingProviderStatus = server.embeddingProvider?.getStatus?.() ?? null;
try {
const db = multiMind.personal.getDatabase();
const row = db.prepare('SELECT COUNT(*) as cnt FROM memory_frames').get() as { cnt: number } | undefined;
const frameCount = row?.cnt ?? 0;
// Get mind file size from the known personal mind path
const scopes = [PERSONAL_SCOPE, ...wsManager.list().map(ws => workspaceScope(ws.id))];
const frameCount = await memory.count({ scopes });
let mindSizeBytes = 0;
try {
const stat = fs.statSync(personalPath);
mindSizeBytes = stat.size;
} catch { /* file may not exist yet */ }
// Embedding coverage — frames with vectors vs total frames
let embeddingCoverage = 0;
try {
const totalRow = db.prepare('SELECT COUNT(*) as cnt FROM memory_frames').get() as { cnt: number };
const vecRow = db.prepare('SELECT COUNT(*) as cnt FROM memory_frames_vec').get() as { cnt: number } | undefined;
const vecCount = vecRow?.cnt ?? 0;
if (totalRow.cnt > 0) {
embeddingCoverage = Math.round((vecCount / totalRow.cnt) * 100);
const memoryDir = path.join(fullConfig.dataDir, 'memory');
for (const file of fs.readdirSync(memoryDir)) {
mindSizeBytes += fs.statSync(path.join(memoryDir, file)).size;
}
} catch { /* vec table may not exist */ }
const embeddingProviderStatus = server.embeddingProvider?.getStatus?.() ?? null;
return { frameCount, mindSizeBytes, embeddingCoverage, embeddingProvider: embeddingProviderStatus };
} catch { /* nothing stored yet */ }
return { frameCount, mindSizeBytes, embeddingCoverage: 100, embeddingProvider: embeddingProviderStatus };
} catch {
return { frameCount: 0, mindSizeBytes: 0, embeddingCoverage: 0, embeddingProvider: null };
return { frameCount: 0, mindSizeBytes: 0, embeddingCoverage: 0, embeddingProvider: embeddingProviderStatus };
}
})();
@@ -2991,22 +2515,18 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
// Stop MCP servers
await mcpRuntime.stopAll().catch(() => {});
// Stop weaver timers
for (const t of weaverTimers) clearInterval(t);
for (const [, ww] of workspaceWeavers) {
for (const t of ww.timers) clearInterval(t);
}
workspaceWeavers.clear();
// Stop collecting chat turns for fact extraction and the legacy memory copy.
memoryExtraction.stop();
legacyMemoryImport.cancel();
await legacyMemoryImport.done;
// Close all workspace sessions (new concurrent model)
sessionManager.closeAll();
// Let the active bounded pass finish before closing any borrowed mind.
await vectorEnrichmentService.stop();
// Close all cached workspace minds
mindCache.closeAll();
multiMind.close();
await memory.close();
waggleDb.close();
// Close audit DB, teams DB, and telemetry
@@ -3020,7 +2540,31 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
}
});
vectorEnrichmentService.start();
// One-time copy of memories from the legacy .mind files into the memory
// store, in the background; the legacy files are only read.
const legacyMemoryImport = (() => {
let cancelled = false;
const sources: LegacyMemorySource[] = [{ path: personalPath, scope: PERSONAL_SCOPE }];
for (const ws of wsManager.list()) {
try {
sources.push({ path: wsManager.getMindPath(ws.id), scope: workspaceScope(ws.id) });
} catch {
// A workspace whose directory fails the path checks has nothing to copy.
}
}
const done = new Promise<void>(resolve => setImmediate(resolve))
.then(() => importLegacyMemories(memory, waggleDb, sources, (report) => {
if (report.status === 'imported') {
log.info(`[memory] copied ${report.copied} legacy memories for ${report.scope} (${report.skipped} skipped)`);
} else if (report.status === 'failed') {
log.warn(`[memory] could not copy legacy memories for ${report.scope}: ${report.error}`);
}
}, () => cancelled))
.then(() => undefined, (err: unknown) => {
log.warn(`[memory] legacy memory copy stopped: ${err instanceof Error ? err.message : String(err)}`);
});
return { done, cancel: () => { cancelled = true; } };
})();
// ── Start Teams server if PostgreSQL is configured (TEAMS tier) ──
// Skip in test environment to avoid port conflicts and addHook-after-listen errors