refactor: drop the legacy memory engine from core, the server and the build (P1.5)
- @waggle/core no longer re-exports @waggle/hive-mind-core and no longer depends on it. - The server no longer opens personal.mind or a cache of workspace minds; the legacy files are only read by the one-time copies of legacy data. releaseWorkspace closes the workspace's sessions and detaches it from the shared orchestrator. - WorkspaceManager stops creating an empty workspace.mind. - memory-center-legacy.ts had no callers left and is removed. - build:packages no longer builds hive-mind-core and wiki-compiler; the sidecar aliases the memory packages to source instead. - mem0ai stays external in the sidecar and launcher bundles: it imports a client for every provider it supports on demand, and bundling it failed on the optional peers Waggle does not install. The launcher declares mem0ai and no longer needs sqlite-vec.
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@waggle/core",
|
||||
"version": "0.1.0",
|
||||
"description": "Waggle-specific orchestration: config, multi-mind, workspace, vault, telemetry, install-audit, cron, skills, file-store, file-indexer, memory-import. Re-exports substrate from @waggle/hive-mind-core for backward compatibility.",
|
||||
"description": "Waggle infrastructure: config, workspaces, waggle.db stores, secrets, telemetry, file storage and indexing, embeddings and recall helpers.",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
@@ -17,7 +17,6 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@huggingface/transformers": "^3.8.1",
|
||||
"@waggle/hive-mind-core": "*",
|
||||
"@waggle/memory-contract": "*",
|
||||
"@waggle/shared": "*",
|
||||
"better-sqlite3": "^12.6.2",
|
||||
|
||||
@@ -1,23 +1,11 @@
|
||||
// @waggle/core — Waggle-specific orchestration layer.
|
||||
// @waggle/core — Waggle's own infrastructure: config, workspaces, waggle.db
|
||||
// stores (automations, cost tracking, profile, awareness, install audit),
|
||||
// secrets, telemetry, file storage and indexing, embeddings and recall helpers.
|
||||
//
|
||||
// As of CC Sesija B 2026-04-30 (PM Q3 Plan A ratification), the substrate
|
||||
// (mind/ + harvest/ + logger + injection-scanner) lives in @waggle/hive-mind-core
|
||||
// and is distributed as Apache 2.0 OSS through a reviewed, maintainer-curated
|
||||
// forward-port. Raw subtree-split output is never a publication source. This
|
||||
// barrel re-exports
|
||||
// substrate symbols for backward compatibility — existing consumers (packages/agent,
|
||||
// apps/web, packages/server, etc.) keep their `import { ... } from '@waggle/core'`
|
||||
// imports unchanged.
|
||||
//
|
||||
// Waggle-specific (non-extracted) modules below the substrate re-exports stay
|
||||
// in this package: config, multi-mind, workspace orchestration, vault, telemetry,
|
||||
// install-audit, cron-store, skill-hashes, file-store, file-indexer,
|
||||
// memory-import.
|
||||
// Memory is not here: Waggle reaches it only through @waggle/memory-contract
|
||||
// (ADR-0007).
|
||||
|
||||
// ── Waggle infrastructure (not memory) ──
|
||||
// Copied from @waggle/hive-mind-core so the rest of Waggle no longer depends on
|
||||
// the memory substrate for them (P1.5). The substrate keeps its own copies until
|
||||
// the package is removed.
|
||||
export { createCoreLogger, type CoreLogger } from './logger.js';
|
||||
export { scanForInjection, type ScanResult } from './injection-scanner.js';
|
||||
export {
|
||||
@@ -66,80 +54,7 @@ export {
|
||||
} from './execution-traces.js';
|
||||
export { importLegacyMindData, type LegacyMindSource, type LegacyImportReport } from './waggle-db-import.js';
|
||||
|
||||
// ── Substrate re-exports from @waggle/hive-mind-core (Apache 2.0 OSS) ──
|
||||
export {
|
||||
// mind/ — memory substrate
|
||||
MindDB, EmbeddingDimMismatchError,
|
||||
type EmbeddingFingerprint, type FingerprintCheck,
|
||||
FrameStore, stripHmPrefix, hashFrameContent,
|
||||
RawArchive, hashRaw, readArchiveUids, withArchiveUid, type RawArchiveRow, type ArchiveInput,
|
||||
MindErasure, type EraseResult,
|
||||
SuppressionStore, type SuppressedSubject,
|
||||
type MemoryFrame, type FrameType, type Importance, type FrameSource,
|
||||
SessionStore, type Session,
|
||||
HybridSearch, type SearchResult,
|
||||
// D1 (oss-drift triage, 2026-06-11) — chunk-level retrieval (flag-gated, default OFF)
|
||||
chunkRetrievalEnabled, rechunkAllFrames, type RechunkResult,
|
||||
chunkText, type ChunkOptions, type FrameChunk,
|
||||
KnowledgeGraph, type Entity, type Relation, type ValidationSchema,
|
||||
SCHEMA_SQL, VEC_TABLE_SQL, CHUNKS_VEC_TABLE_SQL, SCHEMA_VERSION,
|
||||
vecTableSqlForDim, chunksVecTableSqlForDim,
|
||||
computeRelevance,
|
||||
computeTemporalScore,
|
||||
computePopularityScore,
|
||||
computeContextualScore,
|
||||
computeImportanceScore,
|
||||
SCORING_PROFILES,
|
||||
type ScoringProfile,
|
||||
type ScoringWeights,
|
||||
normalizeEntityName, findDuplicates, isNoiseName, isLikelyAcronym,
|
||||
Ontology, validateEntity, type EntitySchema, type ValidationResult,
|
||||
reconcileIndexes, reconcileFtsIndex, reconcileVecIndex, cleanOrphanVectors, cleanOrphanFts,
|
||||
type ReconcileResult,
|
||||
ConceptTracker, CONCEPT_MASTERY_TABLE_SQL,
|
||||
type ConceptEntry, type ConceptUpdate,
|
||||
// harvest/ — universal memory ingestion pipeline
|
||||
HarvestSourceStore,
|
||||
HarvestRunStore, type HarvestRun, type HarvestRunStatus,
|
||||
ChatGPTAdapter,
|
||||
ClaudeAdapter,
|
||||
ClaudeCodeAdapter,
|
||||
GeminiAdapter,
|
||||
UniversalAdapter,
|
||||
MarkdownAdapter,
|
||||
PlaintextAdapter,
|
||||
UrlAdapter,
|
||||
PdfAdapter,
|
||||
HarvestPipeline, type LLMCallFn, type PipelineOptions,
|
||||
extractMemoryLanes, writeMemoryLaneFrames,
|
||||
MIND_FACT_PREFIX, MIND_EVENT_PREFIX, MIND_PROFILE_PREFIX,
|
||||
type MemoryLaneExtraction, type ExtractedEvent, type ExtractedFact, type ExtractedProfile,
|
||||
type WriteLaneFramesResult,
|
||||
// D2 — LLM-based KG entity extraction (oss-drift triage, 2026-06-11)
|
||||
extractKgEntities, writeKgEntities, KG_ENTITY_TYPES,
|
||||
type KgEntity, type KgEntityType, type KgEntityExtraction, type WriteKgEntitiesResult,
|
||||
// W4.6 — raw-turn storage + RAWDETAIL recall lane
|
||||
writeRawTurnFrames, rawTurnHeader, parseRawTurnHeader, rawTurnConvKey,
|
||||
MIND_RAWTURN_PREFIX, MAX_TURNS_PER_ITEM, RAWDETAIL_KILL_SWITCH,
|
||||
type WriteRawTurnsResult, type ParsedRawTurnHeader,
|
||||
fetchRawDetailLane, rawTurnBody, RAW_DETAIL_K,
|
||||
type RawTurnHit, type RawDetailLaneOptions,
|
||||
dedup, harvestSetHash,
|
||||
HARVEST_FRAME_CONTENT_CAP,
|
||||
type ImportSourceType, type ImportItemType, type UniversalImportItem, type DistilledKnowledge,
|
||||
type HarvestPipelineResult, type HarvestSource, type SourceAdapter, type FilesystemAdapter,
|
||||
type ClassifiedItem, type ExtractedContent, type KnowledgeProvenance,
|
||||
// Plan A AMENDMENT 2026-04-30 — multi-workspace orchestration
|
||||
MultiMind, type MultiMindSearchResult, type MindSource, type SearchScope,
|
||||
MultiMindCache, type MultiMindCacheConfig,
|
||||
// Supersession (P) + bridge (B) frame producer (mono-parity 2026-07-05)
|
||||
detectSupersessionChains, detectEntityGroups, applyConsolidation,
|
||||
collectObservations, getCurrentValues,
|
||||
type ConsolidationLlm, type Observation, type SupersessionChain,
|
||||
type EntityGroup, type ConsolidationResult, type CollectObservationsOptions,
|
||||
} from '@waggle/hive-mind-core';
|
||||
|
||||
// ── Waggle-specific orchestration (stays in @waggle/core) ──
|
||||
// ── Waggle-specific orchestration ──
|
||||
export { WaggleConfig, type ProviderEntry } from './config.js';
|
||||
export { needsMigration, migrateToMultiMind } from './migration.js';
|
||||
export {
|
||||
|
||||
@@ -131,8 +131,8 @@ function isContained(root: string, candidate: string): boolean {
|
||||
* WorkspaceManager manages workspace CRUD, groups, and directory structure.
|
||||
* Each workspace lives under {baseDir}/workspaces/{id}/ with:
|
||||
* - workspace.json (config)
|
||||
* - workspace.mind (SQLite .mind file, created empty)
|
||||
* - sessions/ (JSONL session logs)
|
||||
* Workspace memory lives in the memory store, not in this directory.
|
||||
*/
|
||||
export class WorkspaceManager {
|
||||
private readonly workspacesDir: string;
|
||||
@@ -217,9 +217,6 @@ export class WorkspaceManager {
|
||||
}
|
||||
fs.mkdirSync(path.join(canonicalWorkspace, 'sessions'));
|
||||
|
||||
// Touch workspace.mind — MindDB will init schema when first opened
|
||||
fs.writeFileSync(path.join(canonicalWorkspace, 'workspace.mind'), '', { flag: 'wx' });
|
||||
|
||||
const config: WorkspaceConfig = {
|
||||
id,
|
||||
name: options.name,
|
||||
@@ -355,7 +352,8 @@ export class WorkspaceManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the path to a workspace's .mind file.
|
||||
* Get the path to a workspace's legacy .mind file. Only the one-time copy of
|
||||
* legacy data reads it; new workspaces do not have one.
|
||||
*/
|
||||
getMindPath(id: string): string {
|
||||
this.assertWorkspaceId(id);
|
||||
|
||||
@@ -49,8 +49,8 @@
|
||||
"fastify-plugin": "^5.0.0",
|
||||
"glob": "^13.0.6",
|
||||
"mammoth": "^1.11.0",
|
||||
"mem0ai": "3.3.1",
|
||||
"pdf-parse": "^2.4.5",
|
||||
"sqlite-vec": "^0.1.7-alpha.2",
|
||||
"undici": "^6.27.0",
|
||||
"whatsapp-web.js": "1.34.7",
|
||||
"ws": "^8.21.0",
|
||||
|
||||
@@ -19,5 +19,8 @@ export default defineConfig({
|
||||
'adm-zip',
|
||||
'playwright-core',
|
||||
'chromium-bidi',
|
||||
// mem0 loads a client for each provider it supports on demand; those are
|
||||
// optional peers Waggle does not install (see scripts/build-sidecar.mjs).
|
||||
'mem0ai',
|
||||
],
|
||||
});
|
||||
|
||||
@@ -10,7 +10,7 @@ import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify';
|
||||
import cors from '@fastify/cors';
|
||||
import fastifyStatic from '@fastify/static';
|
||||
import websocket from '@fastify/websocket';
|
||||
import { MindDB, MultiMind, WaggleDB, importLegacyMindData, workspaceDataScope, type LegacyMindSource, MultiMindCache, WorkspaceManager, WaggleConfig, createEmbeddingProvider, type EmbeddingProviderConfig, type EmbeddingProviderInstance, FrameStore, SessionStore, SuppressionStore, InstallAuditStore, CronStore, AwarenessLayer, VaultStore, SkillHashStore, HarvestSourceStore, ClaudeCodeAdapter, reconcileIndexes, TelemetryStore, TELEMETRY_EVENTS, ExecutionTraceStore, harvestSetHash } from '@waggle/core';
|
||||
import { WaggleDB, importLegacyMindData, workspaceDataScope, type LegacyMindSource, WorkspaceManager, WaggleConfig, createEmbeddingProvider, type EmbeddingProviderConfig, type EmbeddingProviderInstance, InstallAuditStore, CronStore, AwarenessLayer, VaultStore, SkillHashStore, TelemetryStore, TELEMETRY_EVENTS, ExecutionTraceStore } from '@waggle/core';
|
||||
import { corsOriginAllowed } from './cors-config.js';
|
||||
import { getStorageProvider } from './storage/index.js';
|
||||
import { isLoopbackBind, resolveBindHost } from './net-config.js';
|
||||
@@ -331,10 +331,6 @@ export interface AgentState {
|
||||
* singleton and is unsafe for concurrent sessions. Being migrated per Phase A.1.
|
||||
*/
|
||||
activateWorkspaceMind: (workspaceId: string) => boolean;
|
||||
/** Get a cached workspace MindDB (opens on demand). Returns null if workspace not found. */
|
||||
getWorkspaceMindDb: (workspaceId: string) => import('@waggle/core').MindDB | null;
|
||||
/** Close and remove a workspace MindDB from cache. Used before workspace deletion to prevent EBUSY. */
|
||||
closeWorkspaceMind: (workspaceId: string) => void;
|
||||
/** Close a workspace's live sessions and detach it from the shared orchestrator. Used before workspace deletion. */
|
||||
releaseWorkspace: (workspaceId: string) => void;
|
||||
/** List all workspaces (id + name). Used by global memory search. */
|
||||
@@ -362,7 +358,6 @@ export interface AgentState {
|
||||
declare module 'fastify' {
|
||||
interface FastifyInstance {
|
||||
localConfig: LocalConfig;
|
||||
multiMind: MultiMind;
|
||||
workspaceManager: WorkspaceManager;
|
||||
eventBus: EventEmitter;
|
||||
agentRunner?: AgentRunner;
|
||||
@@ -382,9 +377,6 @@ declare module 'fastify' {
|
||||
skillHashStore: import('@waggle/core').SkillHashStore;
|
||||
scheduler: import('./cron.js').LocalScheduler;
|
||||
marketplace: import('@waggle/marketplace').MarketplaceDB | null;
|
||||
/** Shared per-workspace mind cache. Routes use this to reach an arbitrary
|
||||
* workspace's mind (e.g. FileIndexer for L-20 auto-indexing). */
|
||||
mindCache: import('@waggle/core').MultiMindCache;
|
||||
offlineManager: OfflineManager;
|
||||
rateLimiter: import('./security-middleware.js').RateLimiter;
|
||||
telemetry: import('@waggle/core').TelemetryStore;
|
||||
@@ -492,10 +484,9 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
const defaultWorkspaceId = defaultWorkspaceConfig.id;
|
||||
server.decorate('workspaceManager', wsManager);
|
||||
|
||||
// MultiMind — open personal mind, no workspace yet (selected via API)
|
||||
// The legacy memory file of earlier versions; only the one-time copies of
|
||||
// legacy data below read it, and nothing creates it any more.
|
||||
const personalPath = path.join(fullConfig.dataDir, 'personal.mind');
|
||||
const multiMind = new MultiMind(personalPath);
|
||||
server.decorate('multiMind', multiMind);
|
||||
|
||||
// Waggle's own data (automations, notifications, cost tracking, profile and
|
||||
// awareness) lives in waggle.db, apart from memory (ADR-0007). The first
|
||||
@@ -1260,16 +1251,6 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
return [...base, ...crossTools];
|
||||
};
|
||||
|
||||
// ── Workspace mind cache (LRU, max 20 open) ──
|
||||
// allowedRoot defends against path-traversal via a crafted workspaceId (review
|
||||
// Critical #2 from cowork/Code-Review_MultiMind_April-2026.md). Any resolved
|
||||
// path outside this root is rejected before MindDB construction.
|
||||
const mindCache = new MultiMindCache({
|
||||
maxOpen: 20,
|
||||
getMindPath: (id) => (wsManager.get(id) ? wsManager.getMindPath(id) : null),
|
||||
allowedRoot: path.join(fullConfig.dataDir, 'workspaces'),
|
||||
});
|
||||
server.decorate('mindCache', mindCache);
|
||||
let activeWorkspaceId: string | null = null;
|
||||
const setActiveWorkspaceId = (workspaceId: string | null): void => {
|
||||
activeWorkspaceId = workspaceId;
|
||||
@@ -1287,30 +1268,15 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
return true;
|
||||
};
|
||||
|
||||
// Helper: get a cached workspace MindDB (opens on demand)
|
||||
// A6: Close and remove workspace mind DB from cache before deletion
|
||||
const closeWorkspaceMind = (workspaceId: string): void => {
|
||||
// Pin-aware: close any live session first so it drops its cache pin (and
|
||||
// stops referencing a handle we're about to force-close). Then force-close
|
||||
// the cache entry — on deletion the workspace is authoritative, so we close
|
||||
// regardless of remaining pins to release the file lock before FS deletion
|
||||
// (A6 EBUSY). Ordering avoids the old double-close where both the session
|
||||
// and this seam closed the same MindDB.
|
||||
// Before a workspace is deleted: close its live sessions and detach it from
|
||||
// the shared orchestrator.
|
||||
const releaseWorkspace = (workspaceId: string): void => {
|
||||
sessionManager.close(workspaceId);
|
||||
mindCache.close(workspaceId);
|
||||
if (activeWorkspaceId === workspaceId) {
|
||||
orchestrator.setWorkspace(null);
|
||||
setActiveWorkspaceId(null);
|
||||
}
|
||||
};
|
||||
// Until the legacy mind cache is removed, releasing a workspace also closes
|
||||
// its cached mind so the workspace directory can be deleted (A6 EBUSY).
|
||||
const releaseWorkspace = closeWorkspaceMind;
|
||||
|
||||
const getWorkspaceMindDb = (workspaceId: string): MindDB | null => {
|
||||
if (!wsManager.get(workspaceId)) return null;
|
||||
return mindCache.getOrOpen(workspaceId);
|
||||
};
|
||||
|
||||
// Decorate with shared agent state
|
||||
server.decorate('agentState', {
|
||||
@@ -1331,8 +1297,6 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
buildToolsForSession,
|
||||
bindWorkspaceCollaborationTools,
|
||||
activateWorkspaceMind: activateWorkspaceMind,
|
||||
getWorkspaceMindDb,
|
||||
closeWorkspaceMind,
|
||||
releaseWorkspace,
|
||||
listWorkspaces: () => wsManager.list().map(w => ({ id: w.id, name: w.name })),
|
||||
activeWorkspaceId,
|
||||
@@ -2523,9 +2487,6 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
// Close all workspace sessions (new concurrent model)
|
||||
sessionManager.closeAll();
|
||||
|
||||
// Close all cached workspace minds
|
||||
mindCache.closeAll();
|
||||
multiMind.close();
|
||||
await memory.close();
|
||||
waggleDb.close();
|
||||
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
import type { MemoryFrame } from '@waggle/core';
|
||||
import type { Memory, MemoryStatus } from '@waggle/shared';
|
||||
import { asKind } from '../memory-records.js';
|
||||
|
||||
/**
|
||||
* @deprecated Projects a legacy frame-store row into the Memory shape. Only
|
||||
* `home.ts` and `artifacts.ts` still read frames; once they read memory through
|
||||
* the contract (`recordToMemory` in memory-center.ts), delete this file.
|
||||
*/
|
||||
export function normalizeToMemory(frame: MemoryFrame, mind: string, workspaceId?: string): Memory {
|
||||
let meta: Record<string, unknown> = {};
|
||||
try {
|
||||
const parsed: unknown = typeof frame.metadata === 'string' && frame.metadata ? JSON.parse(frame.metadata) : {};
|
||||
if (parsed && typeof parsed === 'object') meta = parsed as Record<string, unknown>;
|
||||
} catch { /* malformed metadata reads as empty */ }
|
||||
const content = frame.content ?? '';
|
||||
const importance: Memory['importance'] = frame.importance ?? 'normal';
|
||||
const status: MemoryStatus = typeof meta.status === 'string'
|
||||
? meta.status as MemoryStatus
|
||||
: importance === 'deprecated'
|
||||
? 'deprecated'
|
||||
: importance === 'temporary' && frame.source === 'agent_inferred' ? 'unreviewed' : 'active';
|
||||
const firstLine = content.split('\n').find((l) => l.trim().length > 0)?.trim() ?? content.trim();
|
||||
return {
|
||||
id: String(frame.id),
|
||||
kind: asKind(meta.kind) ?? 'fact',
|
||||
title: typeof meta.title === 'string' && meta.title.trim()
|
||||
? meta.title
|
||||
: firstLine.length > 80 ? `${firstLine.slice(0, 79)}…` : firstLine,
|
||||
content,
|
||||
scope: mind === 'workspace' ? 'workspace' : 'personal',
|
||||
workspaceId,
|
||||
source: frame.source ?? 'user_stated',
|
||||
importance,
|
||||
status,
|
||||
createdAt: frame.created_at ?? new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
@@ -25,8 +25,6 @@ import {
|
||||
workspaceIdOf,
|
||||
} from '../memory-records.js';
|
||||
|
||||
export { normalizeToMemory } from './memory-center-legacy.js';
|
||||
|
||||
/**
|
||||
* Memory Center REST surface (S04, PRD §16.4) on the memory contract (ADR-0007).
|
||||
*
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
"@waggle/agent": ["../agent/src/index.ts"],
|
||||
"@waggle/core": ["../core/src/index.ts"],
|
||||
"@waggle/shared": ["../shared/src/index.ts"],
|
||||
"@waggle/hive-mind-core": ["../hive-mind-core/src/index.ts"],
|
||||
"@waggle/marketplace": ["../marketplace/src/index.ts"],
|
||||
"@waggle/sdk": ["../sdk/src/index.ts"]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user