refactor(memory): run agent recall, memory tools and the server on the memory contract (P1.5)
- Orchestrator, memory tools, context loader and pattern write-back use a MemoryStore and a workspace scope instead of MindDB layers; the knowledge graph tools, cognify, memory linker and feedback handler are gone (graph hidden in v1, ADR-0002). - The server opens one mem0 store in <dataDir>/memory/, embedded by the in-process model on first use, and extracts facts about the user in the background with the model the user chose (memoryExtraction setting). - Legacy .mind memories are copied into the store once, in the background; the legacy files are only read. - Weaver consolidation, harvest, wiki, memory import, memory lanes, vector backfill and index reconciliation are removed; agent runs, loops, sub-agent results, connector fetch and the failed-turn capture write through the contract. Workspace sessions no longer hold a mind handle. - FileIndexer and the install audit move to waggle.db; recall date helpers and the reranker are copied into core. - The CLI REPL opens the same memory store.
This commit is contained in:
@@ -17,7 +17,6 @@ import { fetchTeamServer } from './team-server-egress.js';
|
||||
import { getStorageProvider } from './storage/index.js';
|
||||
import { isLoopbackBind, resolveBindHost } from './net-config.js';
|
||||
import { isLocalRequest } from './origin-guard.js';
|
||||
import { MemoryWeaver } from '@waggle/weaver';
|
||||
import {
|
||||
Orchestrator,
|
||||
createSystemTools,
|
||||
@@ -67,7 +66,10 @@ import { PluginRuntimeManager, getStarterSkillsDir, validatePluginManifest } fro
|
||||
import { MarketplaceDB, MarketplaceSync, seedMcpServers, seedNewSources } from '@waggle/marketplace';
|
||||
import type { RiskLevel } from '@waggle/shared';
|
||||
import { runConnectorFetch } from './connector-harvest.js';
|
||||
import { writeAutoSyncSummaryFrame } from './harvest-autosync-frame.js';
|
||||
import { openMemoryStore } from './memory-runtime.js';
|
||||
import { MemoryExtractionQueue } from './memory-extraction.js';
|
||||
import { importLegacyMemories, type LegacyMemorySource } from './legacy-memory-import.js';
|
||||
import { PERSONAL_SCOPE, workspaceScope, type MemoryStore } from '@waggle/memory-contract';
|
||||
import { runLoopTick } from './loop-executor.js';
|
||||
import { enqueueHeldAction } from './held-action-executor.js';
|
||||
import { workspaceRoutes } from './routes/workspaces.js';
|
||||
@@ -77,11 +79,10 @@ import { memoryCenterRoutes } from './routes/memory-center.js';
|
||||
import { artifactRoutes } from './routes/artifacts.js';
|
||||
import { settingsRoutes } from './routes/settings.js';
|
||||
import { embeddingRoutes, buildEmbeddingStatusPayload } from './routes/embedding.js';
|
||||
import { sessionRoutes, findUndistilledSessions, markSessionDistilled } from './routes/sessions.js';
|
||||
import { sessionRoutes } from './routes/sessions.js';
|
||||
import { knowledgeRoutes } from './routes/knowledge.js';
|
||||
import { litellmRoutes } from './routes/litellm.js';
|
||||
import { runMemoryLaneExtraction } from './memory-lane-cron.js';
|
||||
import { VectorEnrichmentService } from './services/vector-enrichment-service.js';
|
||||
import { waggleDanceRoutes } from './routes/waggle-dance.js';
|
||||
import { ingestRoutes, readFileRegistry } from './routes/ingest.js';
|
||||
import { mindRoutes } from './routes/mind.js';
|
||||
import { agentRoutes } from './routes/agent.js';
|
||||
@@ -97,7 +98,6 @@ import { capabilitiesRoutes } from './routes/capabilities.js';
|
||||
import { toolsRoutes } from './routes/tools.js';
|
||||
import { externalToolRunRoutes } from './routes/external-tool-runs.js';
|
||||
import { routeProposalRoutes } from './routes/route-proposals.js';
|
||||
import { waggleDanceRoutes } from './routes/waggle-dance.js';
|
||||
import { commandRoutes } from './routes/commands.js';
|
||||
import { commandRoutes as commandCenterRoutes } from './routes/command.js';
|
||||
import { homeRoutes } from './routes/home.js';
|
||||
@@ -118,7 +118,6 @@ import { extendRoutes } from './routes/extend.js';
|
||||
import { populateMcpRuntimeFromConfig } from './mcp-config.js';
|
||||
import { scheduleMarketplaceBackgroundSync } from './marketplace-background-sync.js';
|
||||
import { fleetRoutes } from './routes/fleet.js';
|
||||
import { importRoutes } from './routes/import.js';
|
||||
import { vaultRoutes } from './routes/vault.js';
|
||||
import { personaRoutes } from './routes/personas.js';
|
||||
import { feedbackRoutes } from './routes/feedback.js';
|
||||
@@ -129,7 +128,6 @@ import { dataEraseRoutes } from './routes/data-erase.js';
|
||||
import { costRoutes } from './routes/cost.js';
|
||||
import { backupRoutes } from './routes/backup.js';
|
||||
import { offlineRoutes } from './routes/offline.js';
|
||||
import { weaverRoutes } from './routes/weaver.js';
|
||||
import { eventRoutes, closeAuditDb, cleanupAuditEvents, emitAuditEvent } from './routes/events.js';
|
||||
import { closeTeamsDb } from './routes/team.js';
|
||||
import { pinRoutes } from './routes/pins.js';
|
||||
@@ -159,8 +157,6 @@ import { agentGroupRoutes } from './routes/agent-groups.js';
|
||||
import { localJobRoutes } from './routes/jobs.js';
|
||||
import { agentEntityRoutes } from './routes/agents.js';
|
||||
import { automationRoutes } from './routes/automations.js';
|
||||
import { harvestRoutes } from './routes/harvest.js';
|
||||
import { wikiRoutes } from './routes/wiki.js';
|
||||
import { identityRoutes } from './routes/identity.js';
|
||||
import { agentRunRoutes } from './routes/agent-run.js';
|
||||
import { agentRunsRoutes } from './routes/agent-runs.js';
|
||||
@@ -320,10 +316,7 @@ export interface AgentState {
|
||||
* Create a fresh Orchestrator for a workspace session with the given mind mounted.
|
||||
* Each session gets its own instance (Option Y fix from docs/plans/phase-a-the-room.md §9).
|
||||
*/
|
||||
createSessionOrchestrator: {
|
||||
(): Orchestrator;
|
||||
(workspaceMind: import('@waggle/core').MindDB): Orchestrator;
|
||||
};
|
||||
createSessionOrchestrator: (workspaceId?: string) => Orchestrator;
|
||||
/**
|
||||
* Build the full tool pool for a session, using the session's orchestrator for mind tools.
|
||||
* When `sourceWorkspaceId` is provided, also injects Phase B.2 cross-workspace read tools
|
||||
@@ -366,10 +359,6 @@ export interface AgentState {
|
||||
wsSessionToken: string;
|
||||
/** SHA-256 hash of the paired Browser Companion credential, or null when unpaired. */
|
||||
browserCompanionCredentialHash: string | null;
|
||||
/** Memory-weaver run timestamps for the personal mind. */
|
||||
weaverState: { lastPersonalConsolidation: string | null; lastPersonalDecay: string | null };
|
||||
/** Per-workspace memory-weaver run timestamps, keyed by workspace ID. */
|
||||
workspaceWeaverStatus: Record<string, { lastConsolidation: string | null }>;
|
||||
}
|
||||
|
||||
declare module 'fastify' {
|
||||
@@ -383,6 +372,10 @@ declare module 'fastify' {
|
||||
auditStore: import('@waggle/core').InstallAuditStore;
|
||||
cronStore: import('@waggle/core').CronStore;
|
||||
waggleDb: import('@waggle/core').WaggleDB;
|
||||
/** Memory (ADR-0007): personal and workspace memories, reached only through the contract. */
|
||||
memory: MemoryStore;
|
||||
/** Background fact extraction from chat turns (ADR-0007, decision 6). */
|
||||
memoryExtraction: MemoryExtractionQueue;
|
||||
traceStore: import('@waggle/core').ExecutionTraceStore;
|
||||
/** Active behavioral spec — the compiled `BEHAVIORAL_SPEC` baseline. */
|
||||
activeBehavioralSpec: ReturnType<typeof import('@waggle/agent').buildActiveBehavioralSpec>;
|
||||
@@ -528,7 +521,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
}
|
||||
|
||||
// Install audit store — persistent trail for capability install events
|
||||
const auditStore = new InstallAuditStore(multiMind.personal);
|
||||
const auditStore = new InstallAuditStore(waggleDb);
|
||||
server.decorate('auditStore', auditStore);
|
||||
|
||||
// Cron store — SQLite-backed schedule persistence for Solo
|
||||
@@ -710,11 +703,33 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
const waggleHome = fullConfig.dataDir || path.join(os.homedir(), '.waggle');
|
||||
const rerankerCacheDir = path.join(waggleHome, 'models', 'reranker');
|
||||
|
||||
// Orchestrator — connects to personal .mind
|
||||
// Memory (ADR-0007): one mem0 store in <dataDir>/memory/, embedded by the
|
||||
// in-process model. Facts are extracted with the model the user chose,
|
||||
// through the built-in proxy (or LiteLLM when the user runs it).
|
||||
const memory = await openMemoryStore({
|
||||
dataDir: fullConfig.dataDir,
|
||||
modelCacheDir: embeddingConfig.inprocess?.cacheDir ?? path.join(waggleHome, 'models'),
|
||||
embeddingModel: embeddingConfig.inprocess?.model,
|
||||
extraction: {
|
||||
baseURL: fullConfig.useBuiltInProxy
|
||||
? `http://127.0.0.1:${fullConfig.port}/v1`
|
||||
: fullConfig.litellmUrl,
|
||||
apiKey: litellmApiKey,
|
||||
model: () => server.agentState?.currentModel || 'claude-sonnet-4-6',
|
||||
},
|
||||
});
|
||||
server.decorate('memory', memory);
|
||||
const memoryExtraction = new MemoryExtractionQueue({
|
||||
memory,
|
||||
isEnabled: () => new WaggleConfig(fullConfig.dataDir).getMemoryExtraction(),
|
||||
onError: (err) => log.warn(`[memory] fact extraction failed: ${err instanceof Error ? err.message : String(err)}`),
|
||||
});
|
||||
server.decorate('memoryExtraction', memoryExtraction);
|
||||
|
||||
// Orchestrator — personal memory; a workspace is added with setWorkspace()
|
||||
const orchestrator = new Orchestrator({
|
||||
db: multiMind.personal,
|
||||
memory,
|
||||
waggleDb,
|
||||
embedder,
|
||||
mode: 'local',
|
||||
version: '0.4',
|
||||
rerankerCacheDir,
|
||||
@@ -858,27 +873,18 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
// main-agent-visible return path stays intact.
|
||||
try {
|
||||
const wsId = server.agentState.activeWorkspaceId;
|
||||
const wsDb = wsId ? mindCache.getOrOpen(wsId) : null;
|
||||
const targetDb = wsDb ?? multiMind.personal;
|
||||
const mindLabel = wsDb ? `workspace:${wsId}` : 'personal';
|
||||
const scope = wsId ? workspaceScope(wsId) : PERSONAL_SCOPE;
|
||||
|
||||
// Structured content: tag provenance so search can filter
|
||||
// for sub-agent-derived frames and so humans can see where it came from.
|
||||
// Structured content: tag provenance so humans can see where it came from.
|
||||
const content = `[Sub-agent: ${result.role}] ${result.agentName}\n\n${result.response}\n\n— tools: ${result.toolsUsed.join(', ') || 'none'} · ${(result.duration / 1000).toFixed(1)}s · ${result.usage.inputTokens + result.usage.outputTokens} tokens`;
|
||||
|
||||
// Use raw frames — cognify requires a per-mind pipeline we don't
|
||||
// construct here; the decayFrames weaver will pick these up later.
|
||||
const frames = new FrameStore(targetDb);
|
||||
const sessions = new SessionStore(targetDb);
|
||||
const active = sessions.getActive();
|
||||
const gopId = active.length > 0 ? active[0].gop_id : sessions.create().gop_id;
|
||||
const latestI = frames.getLatestIFrame(gopId);
|
||||
if (latestI) {
|
||||
frames.createPFrame(gopId, content, latestI.id, 'normal', 'agent_inferred');
|
||||
} else {
|
||||
frames.createIFrame(gopId, content, 'normal', 'agent_inferred');
|
||||
}
|
||||
log.debug(`Gap L: persisted sub-agent result ${result.agentId} to ${mindLabel} mind`);
|
||||
await memory.remember({
|
||||
scope,
|
||||
text: content,
|
||||
kind: 'note',
|
||||
source: 'agent_inferred',
|
||||
origin: 'sub-agent',
|
||||
});
|
||||
log.debug(`Gap L: persisted sub-agent result ${result.agentId} to ${wsId ? `workspace:${wsId}` : 'personal'} memory`);
|
||||
} catch (err) {
|
||||
log.debug('Gap L: sub-agent persistence failed (non-blocking)', err);
|
||||
}
|
||||
@@ -1200,16 +1206,15 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
*
|
||||
* See docs/plans/phase-a-the-room.md §9 for the Option X vs Option Y decision.
|
||||
*/
|
||||
const createSessionOrchestrator = (workspaceMind?: MindDB): Orchestrator => {
|
||||
const createSessionOrchestrator = (workspaceId?: string): Orchestrator => {
|
||||
const sessionOrch = new Orchestrator({
|
||||
db: multiMind.personal,
|
||||
memory,
|
||||
waggleDb,
|
||||
embedder,
|
||||
mode: 'local',
|
||||
version: '0.4',
|
||||
rerankerCacheDir,
|
||||
});
|
||||
if (workspaceMind) sessionOrch.setWorkspaceMind(workspaceMind);
|
||||
if (workspaceId) sessionOrch.setWorkspace(workspaceId);
|
||||
return sessionOrch;
|
||||
};
|
||||
|
||||
@@ -1231,8 +1236,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
|
||||
const crossTools = createCrossWorkspaceTools({
|
||||
sourceWorkspaceId,
|
||||
embedder,
|
||||
getMindForWorkspace: (id) => mindCache.getOrOpen(id),
|
||||
memory,
|
||||
listWorkspaces: () => wsManager.list().map(w => ({ id: w.id, name: w.name })),
|
||||
listWorkspaceFiles: async (id, subPath) => {
|
||||
try {
|
||||
@@ -1268,14 +1272,6 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
allowedRoot: path.join(fullConfig.dataDir, 'workspaces'),
|
||||
});
|
||||
server.decorate('mindCache', mindCache);
|
||||
const vectorEnrichmentService = new VectorEnrichmentService({
|
||||
personalMind: multiMind.personal,
|
||||
embeddingProvider,
|
||||
listWorkspaceIds: () => wsManager.list().map(workspace => workspace.id),
|
||||
acquireWorkspaceMind: workspaceId => mindCache.acquire(workspaceId),
|
||||
releaseWorkspaceMind: workspaceId => mindCache.release(workspaceId),
|
||||
log: (level, message) => log[level](message),
|
||||
});
|
||||
let activeWorkspaceId: string | null = null;
|
||||
const setActiveWorkspaceId = (workspaceId: string | null): void => {
|
||||
activeWorkspaceId = workspaceId;
|
||||
@@ -1283,271 +1279,16 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
if (decorated) decorated.activeWorkspaceId = workspaceId;
|
||||
};
|
||||
|
||||
// ── TeamSync cache — one TeamSync instance per team workspace ──
|
||||
const teamSyncCache = new Map<string, { sync: TeamSync; binding: string }>();
|
||||
|
||||
const serializeTeamSyncBinding = (
|
||||
teamServer: NonNullable<ReturnType<WaggleConfig['getTeamServer']>>,
|
||||
teamId: string,
|
||||
): string => JSON.stringify({
|
||||
url: teamServer.url,
|
||||
teamId,
|
||||
token: crypto.createHash('sha256').update(teamServer.token ?? '').digest('hex'),
|
||||
userId: teamServer.userId ?? 'local-user',
|
||||
displayName: teamServer.displayName ?? 'You',
|
||||
});
|
||||
|
||||
function getTeamSync(workspaceId: string, wsConfig: WorkspaceConfig | null, waggleConfig: WaggleConfig): TeamSync | null {
|
||||
if (!wsConfig?.teamId || !wsConfig?.teamServerUrl) {
|
||||
teamSyncCache.delete(workspaceId);
|
||||
return null;
|
||||
}
|
||||
const teamServer = getBoundTeamServer(wsConfig.teamServerUrl, waggleConfig.getTeamServer());
|
||||
if (!teamServer?.token) {
|
||||
teamSyncCache.delete(workspaceId);
|
||||
return null;
|
||||
}
|
||||
const binding = serializeTeamSyncBinding(teamServer, wsConfig.teamId);
|
||||
const cached = teamSyncCache.get(workspaceId);
|
||||
if (cached?.binding === binding) return cached.sync;
|
||||
const sync = new TeamSync({
|
||||
teamServerUrl: teamServer.url,
|
||||
teamSlug: wsConfig.teamId, // teamId is used as slug
|
||||
authToken: teamServer.token,
|
||||
userId: teamServer.userId ?? 'local-user',
|
||||
displayName: teamServer.displayName ?? 'You',
|
||||
}, fetchTeamServer);
|
||||
const pushFrame = sync.pushFrame.bind(sync);
|
||||
sync.pushFrame = async (frame) => {
|
||||
const currentWorkspace = wsManager.get(workspaceId);
|
||||
const currentWaggleConfig = new WaggleConfig(fullConfig.dataDir);
|
||||
const currentTeamServer = getBoundTeamServer(
|
||||
currentWorkspace?.teamServerUrl,
|
||||
currentWaggleConfig.getTeamServer(),
|
||||
);
|
||||
const currentBinding = currentWorkspace?.teamId && currentTeamServer?.token
|
||||
? serializeTeamSyncBinding(currentTeamServer, currentWorkspace.teamId)
|
||||
: null;
|
||||
const cached = teamSyncCache.get(workspaceId);
|
||||
if (cached?.sync !== sync) {
|
||||
return cached && currentBinding === cached.binding
|
||||
? cached.sync.pushFrame(frame)
|
||||
: null;
|
||||
}
|
||||
if (currentBinding === binding) return pushFrame(frame);
|
||||
|
||||
teamSyncCache.delete(workspaceId);
|
||||
if (currentWorkspace?.teamId && currentTeamServer?.token) {
|
||||
const replacement = getTeamSync(workspaceId, currentWorkspace, currentWaggleConfig);
|
||||
if (replacement) {
|
||||
if (activeWorkspaceId === workspaceId) orchestrator.setTeamSync(replacement);
|
||||
return replacement.pushFrame(frame);
|
||||
}
|
||||
}
|
||||
if (activeWorkspaceId === workspaceId) orchestrator.setTeamSync(null);
|
||||
return null;
|
||||
};
|
||||
teamSyncCache.set(workspaceId, { sync, binding });
|
||||
return sync;
|
||||
}
|
||||
|
||||
/**
|
||||
* Activate a workspace's .mind file on the orchestrator.
|
||||
* Opens the mind if not cached, then calls setWorkspaceMind().
|
||||
*/
|
||||
/** Make a workspace's memory active on the shared orchestrator. */
|
||||
const activateWorkspaceMind = (workspaceId: string): boolean => {
|
||||
if (activeWorkspaceId === workspaceId) return true; // already active
|
||||
if (!wsManager.get(workspaceId)) return false;
|
||||
|
||||
const wsDb = mindCache.getOrOpen(workspaceId);
|
||||
if (!wsDb) return false;
|
||||
|
||||
orchestrator.setWorkspaceMind(wsDb);
|
||||
orchestrator.setWorkspace(workspaceId);
|
||||
setActiveWorkspaceId(workspaceId);
|
||||
return true;
|
||||
};
|
||||
|
||||
// ── Memory Weaver — background consolidation & decay (A1 fix) ────
|
||||
// Runs on the personal mind at startup. Workspace minds get weavers when activated.
|
||||
const personalFrames = new FrameStore(multiMind.personal);
|
||||
const personalSessions = new SessionStore(multiMind.personal);
|
||||
const personalWeaver = new MemoryWeaver(multiMind.personal, personalFrames, personalSessions);
|
||||
const weaverTimers: NodeJS.Timeout[] = [];
|
||||
const workspaceWeavers = new Map<string, { weaver: MemoryWeaver; timers: NodeJS.Timeout[] }>();
|
||||
|
||||
// F2: Track weaver last-run timestamps for status endpoint
|
||||
const weaverState: { lastPersonalConsolidation: string | null; lastPersonalDecay: string | null } = {
|
||||
lastPersonalConsolidation: null,
|
||||
lastPersonalDecay: null,
|
||||
};
|
||||
const workspaceWeaverStatus: Record<string, { lastConsolidation: string | null }> = {};
|
||||
|
||||
// Run personal mind weaver on intervals
|
||||
const runPersonalConsolidation = () => {
|
||||
try {
|
||||
const active = personalSessions.getActive();
|
||||
for (const s of active) personalWeaver.consolidateGop(s.gop_id);
|
||||
weaverState.lastPersonalConsolidation = new Date().toISOString();
|
||||
} catch (err) { log.debug('Personal consolidation skipped', err); }
|
||||
};
|
||||
const runPersonalDecay = () => {
|
||||
try {
|
||||
personalWeaver.decayFrames();
|
||||
personalWeaver.strengthenFrames();
|
||||
weaverState.lastPersonalDecay = new Date().toISOString();
|
||||
} catch (err) { log.debug('Personal decay skipped', err); }
|
||||
};
|
||||
weaverTimers.push(setInterval(runPersonalConsolidation, 60 * 60 * 1000)); // hourly
|
||||
weaverTimers.push(setInterval(runPersonalDecay, 24 * 60 * 60 * 1000)); // daily
|
||||
|
||||
// ── Harvest auto-sync — poll stale sources every 30 min ──
|
||||
const runHarvestAutoSync = async () => {
|
||||
try {
|
||||
const harvestStore = new HarvestSourceStore(multiMind.personal);
|
||||
const stale = harvestStore.getStale();
|
||||
if (stale.length === 0) return;
|
||||
|
||||
const { ClaudeCodeAdapter: CCA } = await import('@waggle/core');
|
||||
const personalSessionStore = new SessionStore(multiMind.personal);
|
||||
personalSessionStore.ensure('harvest', 'harvest', 'Imported memory from external sources');
|
||||
const personalFrameStore = new FrameStore(multiMind.personal);
|
||||
// #7 sticky erasure: don't re-materialize an erased subject on auto-sync.
|
||||
const suppression = new SuppressionStore(multiMind.personal);
|
||||
|
||||
for (const src of stale) {
|
||||
try {
|
||||
// Only auto-sync filesystem-scannable sources
|
||||
if (src.source !== 'claude-code') continue;
|
||||
const homedir = (await import('node:os')).default.homedir();
|
||||
const scanDir = (await import('node:path')).default.join(homedir, '.claude');
|
||||
const adapter = new CCA();
|
||||
const items = adapter.scan(scanDir);
|
||||
if (items.length === 0) continue;
|
||||
|
||||
let saved = 0;
|
||||
let couldNotVerify = 0;
|
||||
let skippedUnsafe = 0;
|
||||
for (const item of items) {
|
||||
// #7 sticky erasure: distinguish a confirmed erasure MATCH from a
|
||||
// fail-closed read ERROR — both skip the write, but a broken-DB read
|
||||
// must be reported as "could not verify", not silently as "erased".
|
||||
const supp = suppression.checkSuppressed(item.source, item.id);
|
||||
if (supp.suppressed) {
|
||||
if (supp.reason === 'error') couldNotVerify++;
|
||||
continue; // #7 Art.17 erased (or unverifiable — fail-closed skip)
|
||||
}
|
||||
// #7 Art.17: stamp the subject key (metadata.sourceId) so a subject-mode
|
||||
// DSAR can reach this auto-synced summary — shared with the cron path.
|
||||
if (!writeAutoSyncSummaryFrame(personalFrameStore, item)) {
|
||||
skippedUnsafe++;
|
||||
continue;
|
||||
}
|
||||
saved++;
|
||||
}
|
||||
// R3-004: store the content digest so the manual harvest route can
|
||||
// skip an unchanged re-scan on the next sync.
|
||||
harvestStore.recordSync(src.source, items.length, saved, harvestSetHash(items));
|
||||
log.info(`[harvest-auto-sync] ${src.source}: imported ${saved} items`
|
||||
+ (couldNotVerify > 0 ? ` (${couldNotVerify} could not be verified against the erasure list — skipped, fail-closed)` : '')
|
||||
+ (skippedUnsafe > 0 ? ` (${skippedUnsafe} unsafe items skipped)` : ''));
|
||||
} catch (err) {
|
||||
log.debug(`[harvest-auto-sync] ${src.source} failed:`, err);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
log.debug('[harvest-auto-sync] skipped:', err);
|
||||
}
|
||||
};
|
||||
weaverTimers.push(setInterval(runHarvestAutoSync, 30 * 60 * 1000)); // every 30 min
|
||||
|
||||
// Extend activateWorkspaceMind to also start a weaver for the workspace
|
||||
// and distill any undistilled sessions into durable memory frames.
|
||||
const baseActivateWorkspaceMind = activateWorkspaceMind;
|
||||
const activateWorkspaceMindWithWeaver = (workspaceId: string): boolean => {
|
||||
const result = baseActivateWorkspaceMind(workspaceId);
|
||||
|
||||
// E4: Workspace topic tracking REMOVED — was saving to personal mind, causing
|
||||
// cross-workspace leakage. Workspace names are available from workspace config
|
||||
// and don't need to be duplicated into memory frames.
|
||||
|
||||
if (result && !workspaceWeavers.has(workspaceId)) {
|
||||
const wsDb = mindCache.getIfOpen(workspaceId);
|
||||
if (wsDb) {
|
||||
const wsFrames = new FrameStore(wsDb);
|
||||
const wsSessions = new SessionStore(wsDb);
|
||||
const wsWeaver = new MemoryWeaver(wsDb, wsFrames, wsSessions);
|
||||
const timers: NodeJS.Timeout[] = [];
|
||||
timers.push(setInterval(() => {
|
||||
try {
|
||||
const active = wsSessions.getActive();
|
||||
for (const s of active) wsWeaver.consolidateGop(s.gop_id);
|
||||
} catch { /* non-blocking */ }
|
||||
}, 60 * 60 * 1000));
|
||||
timers.push(setInterval(() => {
|
||||
try { wsWeaver.decayFrames(); wsWeaver.strengthenFrames(); } catch (err) { log.debug('Workspace memory decay skipped', err); }
|
||||
}, 24 * 60 * 60 * 1000));
|
||||
workspaceWeavers.set(workspaceId, { weaver: wsWeaver, timers });
|
||||
|
||||
// E2: Distill undistilled sessions into durable memory frames on activation
|
||||
try {
|
||||
const sessionsDir = path.join(fullConfig.dataDir, 'workspaces', workspaceId, 'sessions');
|
||||
const undistilled = findUndistilledSessions(sessionsDir);
|
||||
for (const session of undistilled) {
|
||||
const distilled = wsWeaver.distillSessionContent(session.date, session.summary, session.keyPoints);
|
||||
if (distilled) markSessionDistilled(session.filePath);
|
||||
}
|
||||
} catch (err) { log.debug('Workspace distillation skipped', err); }
|
||||
}
|
||||
}
|
||||
|
||||
// After mind activation for team workspaces — pull remote frames
|
||||
if (result) {
|
||||
const wsConfig = wsManager.get(workspaceId);
|
||||
if (wsConfig?.teamId) {
|
||||
const sync = getTeamSync(workspaceId, wsConfig, new WaggleConfig(fullConfig.dataDir));
|
||||
if (sync) {
|
||||
// Set TeamSync on the orchestrator for push-on-write
|
||||
orchestrator.setTeamSync(sync);
|
||||
|
||||
// Pull team frames into local workspace mind
|
||||
const wsDb = mindCache.getIfOpen(workspaceId);
|
||||
if (wsDb) {
|
||||
const wsSessionStore = new SessionStore(wsDb);
|
||||
wsSessionStore.ensure('team-sync', 'team-sync', 'Frames synced from team server');
|
||||
const wsFrameStore = new FrameStore(wsDb);
|
||||
|
||||
sync.pullFrames(sync.getLastSyncTimestamp() ?? undefined).then(pulled => {
|
||||
if (pulled.length === 0) return;
|
||||
let inserted = 0;
|
||||
for (const f of pulled) {
|
||||
try {
|
||||
// Provenance is carried in the content prefix ([Team:author]); the source
|
||||
// column must be schema-valid, so use 'import' (these frames are imported
|
||||
// from the team server). Previously passed 'team_sync', which violated the
|
||||
// memory_frames.source CHECK → every pulled frame was silently skipped.
|
||||
wsFrameStore.createIFrame('team-sync', `[Team:${f.authorName}] ${f.content}`, f.importance, 'import');
|
||||
inserted++;
|
||||
} catch { /* duplicate or constraint — skip */ }
|
||||
}
|
||||
if (inserted > 0) {
|
||||
log.info(`TeamSync: pulled ${inserted} new frames for workspace ${workspaceId}`);
|
||||
}
|
||||
}).catch(err => {
|
||||
log.warn(`TeamSync pull failed:`, err.message);
|
||||
});
|
||||
}
|
||||
} else {
|
||||
orchestrator.setTeamSync(null);
|
||||
}
|
||||
} else {
|
||||
// Non-team workspace — clear TeamSync
|
||||
orchestrator.setTeamSync(null);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
// Helper: get a cached workspace MindDB (opens on demand)
|
||||
// A6: Close and remove workspace mind DB from cache before deletion
|
||||
const closeWorkspaceMind = (workspaceId: string): void => {
|
||||
@@ -1559,10 +1300,8 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
// and this seam closed the same MindDB.
|
||||
sessionManager.close(workspaceId);
|
||||
mindCache.close(workspaceId);
|
||||
teamSyncCache.delete(workspaceId);
|
||||
if (activeWorkspaceId === workspaceId) {
|
||||
orchestrator.setTeamSync(null);
|
||||
orchestrator.clearWorkspaceMind();
|
||||
orchestrator.setWorkspace(null);
|
||||
setActiveWorkspaceId(null);
|
||||
}
|
||||
};
|
||||
@@ -1590,13 +1329,11 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
createSessionOrchestrator,
|
||||
buildToolsForSession,
|
||||
bindWorkspaceCollaborationTools,
|
||||
activateWorkspaceMind: activateWorkspaceMindWithWeaver,
|
||||
activateWorkspaceMind: activateWorkspaceMind,
|
||||
getWorkspaceMindDb,
|
||||
closeWorkspaceMind,
|
||||
listWorkspaces: () => wsManager.list().map(w => ({ id: w.id, name: w.name })),
|
||||
activeWorkspaceId,
|
||||
weaverState,
|
||||
workspaceWeaverStatus,
|
||||
subagentOrchestrator: null,
|
||||
pluginRuntimeManager,
|
||||
mcpRuntime,
|
||||
@@ -1611,7 +1348,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
wsSessionToken,
|
||||
browserCompanionCredentialHash,
|
||||
});
|
||||
activateWorkspaceMindWithWeaver(defaultWorkspaceId);
|
||||
activateWorkspaceMind(defaultWorkspaceId);
|
||||
|
||||
// Wire up skill hot-reload callback
|
||||
const reloadSkills = (fresh: LoadedSkill[]) => {
|
||||
@@ -1642,33 +1379,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
case 'memory_consolidation': {
|
||||
// Check if this is a marketplace sync (uses memory_consolidation type for system-level scheduling)
|
||||
const mcJobConfig = JSON.parse(schedule.job_config || '{}');
|
||||
if (mcJobConfig.action === 'index_reconcile') {
|
||||
try {
|
||||
// Reconcile personal mind FTS/vec indexes
|
||||
const result = await reconcileIndexes(multiMind.personal);
|
||||
if (result.ftsFixed > 0 || result.vecFixed > 0) {
|
||||
log.info(`[cron] Index reconciliation: FTS=${result.ftsFixed} vec=${result.vecFixed} fixed (personal)`);
|
||||
}
|
||||
// Reconcile workspace minds
|
||||
let ftsTotal = result.ftsFixed;
|
||||
let vecTotal = result.vecFixed;
|
||||
const workspaces = wsManager.list();
|
||||
for (const ws of workspaces) {
|
||||
const wsDb = getWorkspaceMindDb(ws.id);
|
||||
if (wsDb) {
|
||||
const wsResult = await reconcileIndexes(wsDb);
|
||||
if (wsResult.ftsFixed > 0 || wsResult.vecFixed > 0) {
|
||||
log.info(`[cron] Index reconciliation: FTS=${wsResult.ftsFixed} vec=${wsResult.vecFixed} fixed (workspace "${ws.name}")`);
|
||||
}
|
||||
ftsTotal += wsResult.ftsFixed;
|
||||
vecTotal += wsResult.vecFixed;
|
||||
}
|
||||
}
|
||||
dreamJournal.record('index_reconcile', { ftsFixed: ftsTotal, vecFixed: vecTotal });
|
||||
} catch (err) {
|
||||
log.warn(`[cron] Index reconciliation failed: ${(err as Error).message}`);
|
||||
}
|
||||
} else if (mcJobConfig.action === 'marketplace_sync' && marketplaceDb) {
|
||||
if (mcJobConfig.action === 'marketplace_sync' && marketplaceDb) {
|
||||
try {
|
||||
const vaultLookup = server.vault ? (key: string) => server.vault!.get(key)?.value ?? null : undefined;
|
||||
const sync = new MarketplaceSync(marketplaceDb, vaultLookup, (url, init) => safeFetch(url, init));
|
||||
@@ -1688,178 +1399,11 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
} catch (err) {
|
||||
log.warn(`[cron] Marketplace sync failed: ${(err as Error).message}`);
|
||||
}
|
||||
} else if (mcJobConfig.action === 'harvest_sync') {
|
||||
// Phase 7: Memory Harvest — scan registered filesystem sources for new data.
|
||||
// Only filesystem-based sources (currently claude-code) can auto-sync; API-based
|
||||
// sources like chatgpt/gemini require user-uploaded exports.
|
||||
try {
|
||||
// Frames have a FK to sessions(gop_id) — ensure the stable
|
||||
// `harvest` session row exists before creating any frames.
|
||||
new SessionStore(multiMind.personal).ensure('harvest', 'harvest', 'Imported memory from external sources');
|
||||
const harvestStore = new HarvestSourceStore(multiMind.personal);
|
||||
const personalFrames = new FrameStore(multiMind.personal);
|
||||
// #7 sticky erasure: don't re-materialize an erased subject on cron sync.
|
||||
const suppression = new SuppressionStore(multiMind.personal);
|
||||
const stale = harvestStore.getStale();
|
||||
let totalItems = 0;
|
||||
let totalFrames = 0;
|
||||
let sourcesScanned = 0;
|
||||
let totalCouldNotVerify = 0;
|
||||
let totalSkippedUnsafe = 0;
|
||||
for (const src of stale) {
|
||||
if (src.source !== 'claude-code' || !src.sourcePath) continue;
|
||||
try {
|
||||
const adapter = new ClaudeCodeAdapter();
|
||||
const items = adapter.scan(src.sourcePath);
|
||||
let saved = 0;
|
||||
for (const item of items) {
|
||||
// #7 sticky erasure: a fail-closed read ERROR is skipped like a
|
||||
// MATCH but tallied separately (report "could not verify", not "erased").
|
||||
const supp = suppression.checkSuppressed(item.source, item.id);
|
||||
if (supp.suppressed) {
|
||||
if (supp.reason === 'error') totalCouldNotVerify++;
|
||||
continue; // #7 Art.17 erased (or unverifiable — fail-closed skip)
|
||||
}
|
||||
// #7 Art.17: stamp the subject key (metadata.sourceId) so a
|
||||
// subject-mode DSAR reaches this cron-synced summary — shared helper.
|
||||
if (!writeAutoSyncSummaryFrame(personalFrames, item)) {
|
||||
totalSkippedUnsafe++;
|
||||
continue;
|
||||
}
|
||||
saved++;
|
||||
}
|
||||
// R3-004: store the content digest for next-sync skip.
|
||||
harvestStore.recordSync(src.source, items.length, saved, harvestSetHash(items));
|
||||
totalItems += items.length;
|
||||
totalFrames += saved;
|
||||
sourcesScanned++;
|
||||
} catch (innerErr) {
|
||||
log.warn(`[cron] Harvest sync: source "${src.source}" failed: ${(innerErr as Error).message}`);
|
||||
}
|
||||
}
|
||||
if (sourcesScanned > 0) {
|
||||
log.info(`[cron] Harvest sync: ${totalFrames} frames from ${totalItems} items across ${sourcesScanned} source(s)`
|
||||
+ (totalCouldNotVerify > 0 ? ` (${totalCouldNotVerify} could not be verified against the erasure list — skipped, fail-closed)` : '')
|
||||
+ (totalSkippedUnsafe > 0 ? ` (${totalSkippedUnsafe} unsafe items skipped)` : ''));
|
||||
}
|
||||
dreamJournal.record('harvest_sync', {
|
||||
framesSaved: totalFrames,
|
||||
itemsScanned: totalItems,
|
||||
sourcesScanned,
|
||||
couldNotVerify: totalCouldNotVerify,
|
||||
});
|
||||
} catch (err) {
|
||||
log.warn(`[cron] Harvest sync failed: ${(err as Error).message}`);
|
||||
}
|
||||
} else if (mcJobConfig.action === 'memory_compact') {
|
||||
// Phase 9a: Memory compaction — prune temporary/deprecated frames, merge stale P-frames.
|
||||
// Runs across personal mind and every workspace mind.
|
||||
try {
|
||||
const personalResult = new FrameStore(multiMind.personal).compact();
|
||||
let wsTempPruned = 0;
|
||||
let wsDepPruned = 0;
|
||||
let wsMerged = 0;
|
||||
for (const ws of wsManager.list()) {
|
||||
const wsDb = getWorkspaceMindDb(ws.id);
|
||||
if (!wsDb) continue;
|
||||
try {
|
||||
const r = new FrameStore(wsDb).compact();
|
||||
wsTempPruned += r.temporaryPruned;
|
||||
wsDepPruned += r.deprecatedPruned;
|
||||
wsMerged += r.pframesMerged;
|
||||
} catch (innerErr) {
|
||||
log.warn(`[cron] Memory compaction: workspace "${ws.name}" failed: ${(innerErr as Error).message}`);
|
||||
}
|
||||
}
|
||||
dreamJournal.record('memory_compact', {
|
||||
temporaryPruned: personalResult.temporaryPruned + wsTempPruned,
|
||||
deprecatedPruned: personalResult.deprecatedPruned + wsDepPruned,
|
||||
pframesMerged: personalResult.pframesMerged + wsMerged,
|
||||
});
|
||||
const total =
|
||||
personalResult.temporaryPruned + personalResult.deprecatedPruned + personalResult.pframesMerged +
|
||||
wsTempPruned + wsDepPruned + wsMerged;
|
||||
if (total > 0) {
|
||||
log.info(
|
||||
`[cron] Memory compaction: personal(temp=${personalResult.temporaryPruned}, dep=${personalResult.deprecatedPruned}, merged=${personalResult.pframesMerged}) ` +
|
||||
`workspaces(temp=${wsTempPruned}, dep=${wsDepPruned}, merged=${wsMerged})`
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn(`[cron] Memory compaction failed: ${(err as Error).message}`);
|
||||
}
|
||||
} else if (mcJobConfig.action === 'memory_lane_extract') {
|
||||
// W4.3d: extraction of the benchmark-proven recall lanes
|
||||
// (facts/events/profiles) over frames written since the last run.
|
||||
// Routes through the built-in proxy ('fast' tier = haiku) — the
|
||||
// same internal-LLM pattern as the other cron LLM calls.
|
||||
try {
|
||||
const llmCall = async (prompt: string, _model: 'fast' | 'accurate'): Promise<string> => {
|
||||
const proxyUrl = `http://127.0.0.1:${fullConfig.port ?? 3333}/v1/chat/completions`;
|
||||
const res = await fetch(proxyUrl, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
signal: AbortSignal.timeout(90_000),
|
||||
body: JSON.stringify({
|
||||
model: 'claude-haiku-4-5',
|
||||
max_tokens: 4096,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
}),
|
||||
});
|
||||
if (!res.ok) throw new Error(`lane-extract LLM HTTP ${res.status}`);
|
||||
const data = await res.json() as unknown;
|
||||
return parseOpenAiTextCompletion(data).content;
|
||||
};
|
||||
const minds: Array<{ label: string; db: import('@waggle/core').MindDB }> = [
|
||||
{ label: 'personal', db: multiMind.personal },
|
||||
];
|
||||
for (const ws of wsManager.list()) {
|
||||
const wsDb = getWorkspaceMindDb(ws.id);
|
||||
if (wsDb) minds.push({ label: `workspace "${ws.name}"`, db: wsDb });
|
||||
}
|
||||
let laneFramesProcessed = 0;
|
||||
let laneFacts = 0;
|
||||
let laneEvents = 0;
|
||||
let laneProfiles = 0;
|
||||
for (const mind of minds) {
|
||||
try {
|
||||
const r = await runMemoryLaneExtraction(mind.db, llmCall);
|
||||
if (!r.skipped) {
|
||||
if (r.errors.length > 0) {
|
||||
log.warn(
|
||||
`[cron] Memory lanes (${mind.label}) incomplete; watermark held for retry: ` +
|
||||
r.errors.join('; ').slice(0, 200)
|
||||
);
|
||||
continue;
|
||||
}
|
||||
log.info(
|
||||
`[cron] Memory lanes (${mind.label}): ${r.framesProcessed} frames → ` +
|
||||
`facts=${r.written?.factsWritten ?? 0} events=${r.written?.eventsWritten ?? 0} ` +
|
||||
`profiles=${r.written?.profilesWritten ?? 0}` +
|
||||
(r.errors.length ? ` (errors: ${r.errors.join('; ').slice(0, 200)})` : '')
|
||||
);
|
||||
laneFramesProcessed += r.framesProcessed;
|
||||
laneFacts += r.written?.factsWritten ?? 0;
|
||||
laneEvents += r.written?.eventsWritten ?? 0;
|
||||
laneProfiles += r.written?.profilesWritten ?? 0;
|
||||
}
|
||||
} catch (innerErr) {
|
||||
log.warn(`[cron] Memory lanes (${mind.label}) failed: ${(innerErr as Error).message}`);
|
||||
}
|
||||
}
|
||||
dreamJournal.record('memory_lane_extract', {
|
||||
framesProcessed: laneFramesProcessed,
|
||||
factsWritten: laneFacts,
|
||||
eventsWritten: laneEvents,
|
||||
profilesWritten: laneProfiles,
|
||||
});
|
||||
} catch (err) {
|
||||
log.warn(`[cron] Memory lane extraction failed: ${(err as Error).message}`);
|
||||
}
|
||||
} else {
|
||||
// Normal memory consolidation
|
||||
try { runPersonalConsolidation(); } catch { /* non-blocking */ }
|
||||
}
|
||||
// index_reconcile, harvest_sync, memory_compact, memory_lane_extract and
|
||||
// plain consolidation maintained the legacy memory engine. mem0 indexes
|
||||
// on write and background extraction replaces the lanes (ADR-0007), so
|
||||
// schedules left over from the legacy data only run the retention below.
|
||||
// UX-Refactor Phase 3: cron_execution_history retention. recordExecution
|
||||
// now writes a row per tick (Journey 16), so prune >30-day rows on the
|
||||
// same nightly consolidation cadence (mirrors optStore.pruneOlderThan).
|
||||
@@ -1870,25 +1414,8 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
break;
|
||||
}
|
||||
case 'workspace_health':
|
||||
// Log stale frame count per workspace as awareness flag
|
||||
try {
|
||||
const workspaces = wsManager.list();
|
||||
for (const ws of workspaces) {
|
||||
const wsDb = getWorkspaceMindDb(ws.id);
|
||||
if (wsDb) {
|
||||
const wsFrames = new FrameStore(wsDb);
|
||||
const staleCount = wsFrames.list({ limit: 100 })
|
||||
.filter(f => {
|
||||
const age = Date.now() - new Date(f.last_accessed).getTime();
|
||||
return age > 7 * 24 * 60 * 60 * 1000;
|
||||
}).length;
|
||||
if (staleCount > 0) {
|
||||
const wsAwareness = new AwarenessLayer(waggleDb, workspaceDataScope(ws.id));
|
||||
wsAwareness.add('flag', `Workspace "${ws.name}" has ${staleCount} stale memory frames (>7 days untouched)`, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch { /* non-blocking */ }
|
||||
// Flagged memories untouched for a week; mem0 does not track access,
|
||||
// so there is nothing to flag (ADR-0007).
|
||||
break;
|
||||
case 'proactive': {
|
||||
const proactiveConfig = JSON.parse(schedule.job_config || '{}');
|
||||
@@ -2007,7 +1534,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
for (const target of targetWorkspaces) {
|
||||
try {
|
||||
// Activate workspace mind so agent has context
|
||||
activateWorkspaceMindWithWeaver(target.id);
|
||||
activateWorkspaceMind(target.id);
|
||||
|
||||
if (taskConfig.mode === 'ai_task') {
|
||||
// #17: full agent turn through the loopback chat route —
|
||||
@@ -2115,12 +1642,18 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
// Auto-fetch (§3.C): pull fresh data from opted-in connectors into the
|
||||
// personal mind. Cost-free — raw frames, no LLM extraction. Free (Solo).
|
||||
try {
|
||||
new SessionStore(multiMind.personal).ensure('harvest', 'harvest', 'Imported memory from external sources');
|
||||
const personalFrames = new FrameStore(multiMind.personal);
|
||||
const statePath = path.join(fullConfig.dataDir, 'connector-harvest-state.json');
|
||||
const r = await runConnectorFetch({
|
||||
connectors: connectorRegistry.getConnected(),
|
||||
writeFrame: (content) => { personalFrames.createIFrame('harvest', content, 'normal', 'import'); },
|
||||
writeFrame: async (content) => {
|
||||
await memory.remember({
|
||||
scope: PERSONAL_SCOPE,
|
||||
text: content,
|
||||
kind: 'note',
|
||||
source: 'import',
|
||||
origin: 'connector-fetch',
|
||||
});
|
||||
},
|
||||
loadState: () => {
|
||||
try {
|
||||
const raw = JSON.parse(fs.readFileSync(statePath, 'utf-8')) as Partial<{ lastFetchedAt: string; hashes: Record<string, string> }>;
|
||||
@@ -2154,12 +1687,12 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
// here so the scheduler records the failure and auto-disables after
|
||||
// repeated failures (recall/write failures are caught inside runLoopTick).
|
||||
const wsId = schedule.workspace_id;
|
||||
const mindDb = wsId && wsId !== '*' ? getWorkspaceMindDb(wsId) : multiMind.personal;
|
||||
if (!mindDb) {
|
||||
const loopWorkspace = wsId && wsId !== '*' ? wsId : null;
|
||||
if (loopWorkspace && !wsManager.get(loopWorkspace)) {
|
||||
log.warn(`[cron] loop "${schedule.name}" references unknown workspace "${wsId}" — skipping`);
|
||||
break;
|
||||
}
|
||||
if (wsId && wsId !== '*') activateWorkspaceMindWithWeaver(wsId);
|
||||
if (wsId && wsId !== '*') activateWorkspaceMind(wsId);
|
||||
const loopModel = server.agentState.currentModel ?? 'claude-sonnet-4-6';
|
||||
const loopProxyUrl = `http://127.0.0.1:${fullConfig.port ?? 3333}/v1/chat/completions`;
|
||||
const loopChat = async (prompt: string, maxTokens = 2048): Promise<string> => {
|
||||
@@ -2178,7 +1711,14 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
return parseOpenAiTextCompletion(body).content;
|
||||
};
|
||||
const loopAwareness = new AwarenessLayer(waggleDb, wsId && wsId !== '*' ? workspaceDataScope(wsId) : 'personal');
|
||||
const loopResult = await runLoopTick({ schedule, mindDb, awareness: loopAwareness, embedder, chat: loopChat, log });
|
||||
const loopResult = await runLoopTick({
|
||||
schedule,
|
||||
memory,
|
||||
scope: loopWorkspace ? workspaceScope(loopWorkspace) : PERSONAL_SCOPE,
|
||||
awareness: loopAwareness,
|
||||
chat: loopChat,
|
||||
log,
|
||||
});
|
||||
if (!loopResult.skipped) {
|
||||
emitNotification(server, {
|
||||
title: `Loop: ${schedule.name}`,
|
||||
@@ -2500,7 +2040,6 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
await server.register(mcpRoutes);
|
||||
await server.register(extendRoutes);
|
||||
await server.register(fleetRoutes);
|
||||
await server.register(importRoutes);
|
||||
await server.register(vaultRoutes);
|
||||
await server.register(personaRoutes);
|
||||
await server.register(feedbackRoutes);
|
||||
@@ -2510,7 +2049,6 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
await server.register(costRoutes);
|
||||
await server.register(backupRoutes);
|
||||
await server.register(offlineRoutes);
|
||||
await server.register(weaverRoutes);
|
||||
await server.register(eventRoutes);
|
||||
await server.register(workflowRoutes);
|
||||
await server.register(pinRoutes);
|
||||
@@ -2569,8 +2107,6 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
await server.register(telemetryRoutes);
|
||||
await server.register(agentGroupRoutes);
|
||||
await server.register(localJobRoutes);
|
||||
await server.register(harvestRoutes);
|
||||
await server.register(wikiRoutes);
|
||||
await server.register(identityRoutes);
|
||||
await server.register(agentRunRoutes);
|
||||
await server.register(agentRunsRoutes);
|
||||
@@ -2887,7 +2423,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
}
|
||||
const dbHealthy = (() => {
|
||||
try {
|
||||
multiMind.personal.getDatabase().prepare('SELECT 1').get();
|
||||
waggleDb.getDatabase().prepare('SELECT 1').get();
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
@@ -2915,35 +2451,23 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
? 'unavailable'
|
||||
: 'degraded';
|
||||
|
||||
// Memory stats — frame count + mind file size
|
||||
const memoryStats = (() => {
|
||||
// Memory stats — memory count and the size of the memory directory.
|
||||
// mem0 embeds every memory on write, so coverage is complete by design.
|
||||
const memoryStats = await (async () => {
|
||||
const embeddingProviderStatus = server.embeddingProvider?.getStatus?.() ?? null;
|
||||
try {
|
||||
const db = multiMind.personal.getDatabase();
|
||||
const row = db.prepare('SELECT COUNT(*) as cnt FROM memory_frames').get() as { cnt: number } | undefined;
|
||||
const frameCount = row?.cnt ?? 0;
|
||||
|
||||
// Get mind file size from the known personal mind path
|
||||
const scopes = [PERSONAL_SCOPE, ...wsManager.list().map(ws => workspaceScope(ws.id))];
|
||||
const frameCount = await memory.count({ scopes });
|
||||
let mindSizeBytes = 0;
|
||||
try {
|
||||
const stat = fs.statSync(personalPath);
|
||||
mindSizeBytes = stat.size;
|
||||
} catch { /* file may not exist yet */ }
|
||||
|
||||
// Embedding coverage — frames with vectors vs total frames
|
||||
let embeddingCoverage = 0;
|
||||
try {
|
||||
const totalRow = db.prepare('SELECT COUNT(*) as cnt FROM memory_frames').get() as { cnt: number };
|
||||
const vecRow = db.prepare('SELECT COUNT(*) as cnt FROM memory_frames_vec').get() as { cnt: number } | undefined;
|
||||
const vecCount = vecRow?.cnt ?? 0;
|
||||
if (totalRow.cnt > 0) {
|
||||
embeddingCoverage = Math.round((vecCount / totalRow.cnt) * 100);
|
||||
const memoryDir = path.join(fullConfig.dataDir, 'memory');
|
||||
for (const file of fs.readdirSync(memoryDir)) {
|
||||
mindSizeBytes += fs.statSync(path.join(memoryDir, file)).size;
|
||||
}
|
||||
} catch { /* vec table may not exist */ }
|
||||
|
||||
const embeddingProviderStatus = server.embeddingProvider?.getStatus?.() ?? null;
|
||||
return { frameCount, mindSizeBytes, embeddingCoverage, embeddingProvider: embeddingProviderStatus };
|
||||
} catch { /* nothing stored yet */ }
|
||||
return { frameCount, mindSizeBytes, embeddingCoverage: 100, embeddingProvider: embeddingProviderStatus };
|
||||
} catch {
|
||||
return { frameCount: 0, mindSizeBytes: 0, embeddingCoverage: 0, embeddingProvider: null };
|
||||
return { frameCount: 0, mindSizeBytes: 0, embeddingCoverage: 0, embeddingProvider: embeddingProviderStatus };
|
||||
}
|
||||
})();
|
||||
|
||||
@@ -2991,22 +2515,18 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
// Stop MCP servers
|
||||
await mcpRuntime.stopAll().catch(() => {});
|
||||
|
||||
// Stop weaver timers
|
||||
for (const t of weaverTimers) clearInterval(t);
|
||||
for (const [, ww] of workspaceWeavers) {
|
||||
for (const t of ww.timers) clearInterval(t);
|
||||
}
|
||||
workspaceWeavers.clear();
|
||||
// Stop collecting chat turns for fact extraction and the legacy memory copy.
|
||||
memoryExtraction.stop();
|
||||
legacyMemoryImport.cancel();
|
||||
await legacyMemoryImport.done;
|
||||
|
||||
// Close all workspace sessions (new concurrent model)
|
||||
sessionManager.closeAll();
|
||||
|
||||
// Let the active bounded pass finish before closing any borrowed mind.
|
||||
await vectorEnrichmentService.stop();
|
||||
|
||||
// Close all cached workspace minds
|
||||
mindCache.closeAll();
|
||||
multiMind.close();
|
||||
await memory.close();
|
||||
waggleDb.close();
|
||||
|
||||
// Close audit DB, teams DB, and telemetry
|
||||
@@ -3020,7 +2540,31 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
}
|
||||
});
|
||||
|
||||
vectorEnrichmentService.start();
|
||||
// One-time copy of memories from the legacy .mind files into the memory
|
||||
// store, in the background; the legacy files are only read.
|
||||
const legacyMemoryImport = (() => {
|
||||
let cancelled = false;
|
||||
const sources: LegacyMemorySource[] = [{ path: personalPath, scope: PERSONAL_SCOPE }];
|
||||
for (const ws of wsManager.list()) {
|
||||
try {
|
||||
sources.push({ path: wsManager.getMindPath(ws.id), scope: workspaceScope(ws.id) });
|
||||
} catch {
|
||||
// A workspace whose directory fails the path checks has nothing to copy.
|
||||
}
|
||||
}
|
||||
const done = new Promise<void>(resolve => setImmediate(resolve))
|
||||
.then(() => importLegacyMemories(memory, waggleDb, sources, (report) => {
|
||||
if (report.status === 'imported') {
|
||||
log.info(`[memory] copied ${report.copied} legacy memories for ${report.scope} (${report.skipped} skipped)`);
|
||||
} else if (report.status === 'failed') {
|
||||
log.warn(`[memory] could not copy legacy memories for ${report.scope}: ${report.error}`);
|
||||
}
|
||||
}, () => cancelled))
|
||||
.then(() => undefined, (err: unknown) => {
|
||||
log.warn(`[memory] legacy memory copy stopped: ${err instanceof Error ? err.message : String(err)}`);
|
||||
});
|
||||
return { done, cancel: () => { cancelled = true; } };
|
||||
})();
|
||||
|
||||
// ── Start Teams server if PostgreSQL is configured (TEAMS tier) ──
|
||||
// Skip in test environment to avoid port conflicts and addHook-after-listen errors
|
||||
|
||||
Reference in New Issue
Block a user