refactor: drop the legacy memory engine from core, the server and the build (P1.5)

- @waggle/core no longer re-exports @waggle/hive-mind-core and no longer
  depends on it.
- The server no longer opens personal.mind or a cache of workspace minds;
  the legacy files are only read by the one-time copies of legacy data.
  releaseWorkspace closes the workspace's sessions and detaches it from the
  shared orchestrator.
- WorkspaceManager stops creating an empty workspace.mind.
- memory-center-legacy.ts had no callers left and is removed.
- build:packages no longer builds hive-mind-core and wiki-compiler; the
  sidecar aliases the memory packages to source instead.
- mem0ai stays external in the sidecar and launcher bundles: it imports a
  client for every provider it supports on demand, and bundling it failed on
  the optional peers Waggle does not install. The launcher declares mem0ai
  and no longer needs sqlite-vec.
This commit is contained in:
shumeiko.it
2026-10-24 13:34:09 +02:00
parent bde961abcf
commit e435e7da94
12 changed files with 252 additions and 190 deletions

View File

@@ -1,7 +1,7 @@
{
"name": "@waggle/core",
"version": "0.1.0",
"description": "Waggle-specific orchestration: config, multi-mind, workspace, vault, telemetry, install-audit, cron, skills, file-store, file-indexer, memory-import. Re-exports substrate from @waggle/hive-mind-core for backward compatibility.",
"description": "Waggle infrastructure: config, workspaces, waggle.db stores, secrets, telemetry, file storage and indexing, embeddings and recall helpers.",
"type": "module",
"main": "dist/index.js",
"types": "dist/index.d.ts",
@@ -17,7 +17,6 @@
},
"dependencies": {
"@huggingface/transformers": "^3.8.1",
"@waggle/hive-mind-core": "*",
"@waggle/memory-contract": "*",
"@waggle/shared": "*",
"better-sqlite3": "^12.6.2",

View File

@@ -1,23 +1,11 @@
// @waggle/core — Waggle-specific orchestration layer.
// @waggle/core — Waggle's own infrastructure: config, workspaces, waggle.db
// stores (automations, cost tracking, profile, awareness, install audit),
// secrets, telemetry, file storage and indexing, embeddings and recall helpers.
//
// As of CC Sesija B 2026-04-30 (PM Q3 Plan A ratification), the substrate
// (mind/ + harvest/ + logger + injection-scanner) lives in @waggle/hive-mind-core
// and is distributed as Apache 2.0 OSS through a reviewed, maintainer-curated
// forward-port. Raw subtree-split output is never a publication source. This
// barrel re-exports
// substrate symbols for backward compatibility — existing consumers (packages/agent,
// apps/web, packages/server, etc.) keep their `import { ... } from '@waggle/core'`
// imports unchanged.
//
// Waggle-specific (non-extracted) modules below the substrate re-exports stay
// in this package: config, multi-mind, workspace orchestration, vault, telemetry,
// install-audit, cron-store, skill-hashes, file-store, file-indexer,
// memory-import.
// Memory is not here: Waggle reaches it only through @waggle/memory-contract
// (ADR-0007).
// ── Waggle infrastructure (not memory) ──
// Copied from @waggle/hive-mind-core so the rest of Waggle no longer depends on
// the memory substrate for them (P1.5). The substrate keeps its own copies until
// the package is removed.
export { createCoreLogger, type CoreLogger } from './logger.js';
export { scanForInjection, type ScanResult } from './injection-scanner.js';
export {
@@ -66,80 +54,7 @@ export {
} from './execution-traces.js';
export { importLegacyMindData, type LegacyMindSource, type LegacyImportReport } from './waggle-db-import.js';
// ── Substrate re-exports from @waggle/hive-mind-core (Apache 2.0 OSS) ──
export {
// mind/ — memory substrate
MindDB, EmbeddingDimMismatchError,
type EmbeddingFingerprint, type FingerprintCheck,
FrameStore, stripHmPrefix, hashFrameContent,
RawArchive, hashRaw, readArchiveUids, withArchiveUid, type RawArchiveRow, type ArchiveInput,
MindErasure, type EraseResult,
SuppressionStore, type SuppressedSubject,
type MemoryFrame, type FrameType, type Importance, type FrameSource,
SessionStore, type Session,
HybridSearch, type SearchResult,
// D1 (oss-drift triage, 2026-06-11) — chunk-level retrieval (flag-gated, default OFF)
chunkRetrievalEnabled, rechunkAllFrames, type RechunkResult,
chunkText, type ChunkOptions, type FrameChunk,
KnowledgeGraph, type Entity, type Relation, type ValidationSchema,
SCHEMA_SQL, VEC_TABLE_SQL, CHUNKS_VEC_TABLE_SQL, SCHEMA_VERSION,
vecTableSqlForDim, chunksVecTableSqlForDim,
computeRelevance,
computeTemporalScore,
computePopularityScore,
computeContextualScore,
computeImportanceScore,
SCORING_PROFILES,
type ScoringProfile,
type ScoringWeights,
normalizeEntityName, findDuplicates, isNoiseName, isLikelyAcronym,
Ontology, validateEntity, type EntitySchema, type ValidationResult,
reconcileIndexes, reconcileFtsIndex, reconcileVecIndex, cleanOrphanVectors, cleanOrphanFts,
type ReconcileResult,
ConceptTracker, CONCEPT_MASTERY_TABLE_SQL,
type ConceptEntry, type ConceptUpdate,
// harvest/ — universal memory ingestion pipeline
HarvestSourceStore,
HarvestRunStore, type HarvestRun, type HarvestRunStatus,
ChatGPTAdapter,
ClaudeAdapter,
ClaudeCodeAdapter,
GeminiAdapter,
UniversalAdapter,
MarkdownAdapter,
PlaintextAdapter,
UrlAdapter,
PdfAdapter,
HarvestPipeline, type LLMCallFn, type PipelineOptions,
extractMemoryLanes, writeMemoryLaneFrames,
MIND_FACT_PREFIX, MIND_EVENT_PREFIX, MIND_PROFILE_PREFIX,
type MemoryLaneExtraction, type ExtractedEvent, type ExtractedFact, type ExtractedProfile,
type WriteLaneFramesResult,
// D2 — LLM-based KG entity extraction (oss-drift triage, 2026-06-11)
extractKgEntities, writeKgEntities, KG_ENTITY_TYPES,
type KgEntity, type KgEntityType, type KgEntityExtraction, type WriteKgEntitiesResult,
// W4.6 — raw-turn storage + RAWDETAIL recall lane
writeRawTurnFrames, rawTurnHeader, parseRawTurnHeader, rawTurnConvKey,
MIND_RAWTURN_PREFIX, MAX_TURNS_PER_ITEM, RAWDETAIL_KILL_SWITCH,
type WriteRawTurnsResult, type ParsedRawTurnHeader,
fetchRawDetailLane, rawTurnBody, RAW_DETAIL_K,
type RawTurnHit, type RawDetailLaneOptions,
dedup, harvestSetHash,
HARVEST_FRAME_CONTENT_CAP,
type ImportSourceType, type ImportItemType, type UniversalImportItem, type DistilledKnowledge,
type HarvestPipelineResult, type HarvestSource, type SourceAdapter, type FilesystemAdapter,
type ClassifiedItem, type ExtractedContent, type KnowledgeProvenance,
// Plan A AMENDMENT 2026-04-30 — multi-workspace orchestration
MultiMind, type MultiMindSearchResult, type MindSource, type SearchScope,
MultiMindCache, type MultiMindCacheConfig,
// Supersession (P) + bridge (B) frame producer (mono-parity 2026-07-05)
detectSupersessionChains, detectEntityGroups, applyConsolidation,
collectObservations, getCurrentValues,
type ConsolidationLlm, type Observation, type SupersessionChain,
type EntityGroup, type ConsolidationResult, type CollectObservationsOptions,
} from '@waggle/hive-mind-core';
// ── Waggle-specific orchestration (stays in @waggle/core) ──
// ── Waggle-specific orchestration ──
export { WaggleConfig, type ProviderEntry } from './config.js';
export { needsMigration, migrateToMultiMind } from './migration.js';
export {

View File

@@ -131,8 +131,8 @@ function isContained(root: string, candidate: string): boolean {
* WorkspaceManager manages workspace CRUD, groups, and directory structure.
* Each workspace lives under {baseDir}/workspaces/{id}/ with:
* - workspace.json (config)
* - workspace.mind (SQLite .mind file, created empty)
* - sessions/ (JSONL session logs)
* Workspace memory lives in the memory store, not in this directory.
*/
export class WorkspaceManager {
private readonly workspacesDir: string;
@@ -217,9 +217,6 @@ export class WorkspaceManager {
}
fs.mkdirSync(path.join(canonicalWorkspace, 'sessions'));
// Touch workspace.mind — MindDB will init schema when first opened
fs.writeFileSync(path.join(canonicalWorkspace, 'workspace.mind'), '', { flag: 'wx' });
const config: WorkspaceConfig = {
id,
name: options.name,
@@ -355,7 +352,8 @@ export class WorkspaceManager {
}
/**
* Get the path to a workspace's .mind file.
* Get the path to a workspace's legacy .mind file. Only the one-time copy of
* legacy data reads it; new workspaces do not have one.
*/
getMindPath(id: string): string {
this.assertWorkspaceId(id);

View File

@@ -49,8 +49,8 @@
"fastify-plugin": "^5.0.0",
"glob": "^13.0.6",
"mammoth": "^1.11.0",
"mem0ai": "3.3.1",
"pdf-parse": "^2.4.5",
"sqlite-vec": "^0.1.7-alpha.2",
"undici": "^6.27.0",
"whatsapp-web.js": "1.34.7",
"ws": "^8.21.0",

View File

@@ -19,5 +19,8 @@ export default defineConfig({
'adm-zip',
'playwright-core',
'chromium-bidi',
// mem0 loads a client for each provider it supports on demand; those are
// optional peers Waggle does not install (see scripts/build-sidecar.mjs).
'mem0ai',
],
});

View File

@@ -10,7 +10,7 @@ import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify';
import cors from '@fastify/cors';
import fastifyStatic from '@fastify/static';
import websocket from '@fastify/websocket';
import { MindDB, MultiMind, WaggleDB, importLegacyMindData, workspaceDataScope, type LegacyMindSource, MultiMindCache, WorkspaceManager, WaggleConfig, createEmbeddingProvider, type EmbeddingProviderConfig, type EmbeddingProviderInstance, FrameStore, SessionStore, SuppressionStore, InstallAuditStore, CronStore, AwarenessLayer, VaultStore, SkillHashStore, HarvestSourceStore, ClaudeCodeAdapter, reconcileIndexes, TelemetryStore, TELEMETRY_EVENTS, ExecutionTraceStore, harvestSetHash } from '@waggle/core';
import { WaggleDB, importLegacyMindData, workspaceDataScope, type LegacyMindSource, WorkspaceManager, WaggleConfig, createEmbeddingProvider, type EmbeddingProviderConfig, type EmbeddingProviderInstance, InstallAuditStore, CronStore, AwarenessLayer, VaultStore, SkillHashStore, TelemetryStore, TELEMETRY_EVENTS, ExecutionTraceStore } from '@waggle/core';
import { corsOriginAllowed } from './cors-config.js';
import { getStorageProvider } from './storage/index.js';
import { isLoopbackBind, resolveBindHost } from './net-config.js';
@@ -331,10 +331,6 @@ export interface AgentState {
* singleton and is unsafe for concurrent sessions. Being migrated per Phase A.1.
*/
activateWorkspaceMind: (workspaceId: string) => boolean;
/** Get a cached workspace MindDB (opens on demand). Returns null if workspace not found. */
getWorkspaceMindDb: (workspaceId: string) => import('@waggle/core').MindDB | null;
/** Close and remove a workspace MindDB from cache. Used before workspace deletion to prevent EBUSY. */
closeWorkspaceMind: (workspaceId: string) => void;
/** Close a workspace's live sessions and detach it from the shared orchestrator. Used before workspace deletion. */
releaseWorkspace: (workspaceId: string) => void;
/** List all workspaces (id + name). Used by global memory search. */
@@ -362,7 +358,6 @@ export interface AgentState {
declare module 'fastify' {
interface FastifyInstance {
localConfig: LocalConfig;
multiMind: MultiMind;
workspaceManager: WorkspaceManager;
eventBus: EventEmitter;
agentRunner?: AgentRunner;
@@ -382,9 +377,6 @@ declare module 'fastify' {
skillHashStore: import('@waggle/core').SkillHashStore;
scheduler: import('./cron.js').LocalScheduler;
marketplace: import('@waggle/marketplace').MarketplaceDB | null;
/** Shared per-workspace mind cache. Routes use this to reach an arbitrary
* workspace's mind (e.g. FileIndexer for L-20 auto-indexing). */
mindCache: import('@waggle/core').MultiMindCache;
offlineManager: OfflineManager;
rateLimiter: import('./security-middleware.js').RateLimiter;
telemetry: import('@waggle/core').TelemetryStore;
@@ -492,10 +484,9 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
const defaultWorkspaceId = defaultWorkspaceConfig.id;
server.decorate('workspaceManager', wsManager);
// MultiMind — open personal mind, no workspace yet (selected via API)
// The legacy memory file of earlier versions; only the one-time copies of
// legacy data below read it, and nothing creates it any more.
const personalPath = path.join(fullConfig.dataDir, 'personal.mind');
const multiMind = new MultiMind(personalPath);
server.decorate('multiMind', multiMind);
// Waggle's own data (automations, notifications, cost tracking, profile and
// awareness) lives in waggle.db, apart from memory (ADR-0007). The first
@@ -1260,16 +1251,6 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
return [...base, ...crossTools];
};
// ── Workspace mind cache (LRU, max 20 open) ──
// allowedRoot defends against path-traversal via a crafted workspaceId (review
// Critical #2 from cowork/Code-Review_MultiMind_April-2026.md). Any resolved
// path outside this root is rejected before MindDB construction.
const mindCache = new MultiMindCache({
maxOpen: 20,
getMindPath: (id) => (wsManager.get(id) ? wsManager.getMindPath(id) : null),
allowedRoot: path.join(fullConfig.dataDir, 'workspaces'),
});
server.decorate('mindCache', mindCache);
let activeWorkspaceId: string | null = null;
const setActiveWorkspaceId = (workspaceId: string | null): void => {
activeWorkspaceId = workspaceId;
@@ -1287,30 +1268,15 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
return true;
};
// Helper: get a cached workspace MindDB (opens on demand)
// A6: Close and remove workspace mind DB from cache before deletion
const closeWorkspaceMind = (workspaceId: string): void => {
// Pin-aware: close any live session first so it drops its cache pin (and
// stops referencing a handle we're about to force-close). Then force-close
// the cache entry — on deletion the workspace is authoritative, so we close
// regardless of remaining pins to release the file lock before FS deletion
// (A6 EBUSY). Ordering avoids the old double-close where both the session
// and this seam closed the same MindDB.
// Before a workspace is deleted: close its live sessions and detach it from
// the shared orchestrator.
const releaseWorkspace = (workspaceId: string): void => {
sessionManager.close(workspaceId);
mindCache.close(workspaceId);
if (activeWorkspaceId === workspaceId) {
orchestrator.setWorkspace(null);
setActiveWorkspaceId(null);
}
};
// Until the legacy mind cache is removed, releasing a workspace also closes
// its cached mind so the workspace directory can be deleted (A6 EBUSY).
const releaseWorkspace = closeWorkspaceMind;
const getWorkspaceMindDb = (workspaceId: string): MindDB | null => {
if (!wsManager.get(workspaceId)) return null;
return mindCache.getOrOpen(workspaceId);
};
// Decorate with shared agent state
server.decorate('agentState', {
@@ -1331,8 +1297,6 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
buildToolsForSession,
bindWorkspaceCollaborationTools,
activateWorkspaceMind: activateWorkspaceMind,
getWorkspaceMindDb,
closeWorkspaceMind,
releaseWorkspace,
listWorkspaces: () => wsManager.list().map(w => ({ id: w.id, name: w.name })),
activeWorkspaceId,
@@ -2523,9 +2487,6 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
// Close all workspace sessions (new concurrent model)
sessionManager.closeAll();
// Close all cached workspace minds
mindCache.closeAll();
multiMind.close();
await memory.close();
waggleDb.close();

View File

@@ -1,38 +0,0 @@
import type { MemoryFrame } from '@waggle/core';
import type { Memory, MemoryStatus } from '@waggle/shared';
import { asKind } from '../memory-records.js';
/**
* @deprecated Projects a legacy frame-store row into the Memory shape. Only
* `home.ts` and `artifacts.ts` still read frames; once they read memory through
* the contract (`recordToMemory` in memory-center.ts), delete this file.
*/
export function normalizeToMemory(frame: MemoryFrame, mind: string, workspaceId?: string): Memory {
let meta: Record<string, unknown> = {};
try {
const parsed: unknown = typeof frame.metadata === 'string' && frame.metadata ? JSON.parse(frame.metadata) : {};
if (parsed && typeof parsed === 'object') meta = parsed as Record<string, unknown>;
} catch { /* malformed metadata reads as empty */ }
const content = frame.content ?? '';
const importance: Memory['importance'] = frame.importance ?? 'normal';
const status: MemoryStatus = typeof meta.status === 'string'
? meta.status as MemoryStatus
: importance === 'deprecated'
? 'deprecated'
: importance === 'temporary' && frame.source === 'agent_inferred' ? 'unreviewed' : 'active';
const firstLine = content.split('\n').find((l) => l.trim().length > 0)?.trim() ?? content.trim();
return {
id: String(frame.id),
kind: asKind(meta.kind) ?? 'fact',
title: typeof meta.title === 'string' && meta.title.trim()
? meta.title
: firstLine.length > 80 ? `${firstLine.slice(0, 79)}…` : firstLine,
content,
scope: mind === 'workspace' ? 'workspace' : 'personal',
workspaceId,
source: frame.source ?? 'user_stated',
importance,
status,
createdAt: frame.created_at ?? new Date().toISOString(),
};
}

View File

@@ -25,8 +25,6 @@ import {
workspaceIdOf,
} from '../memory-records.js';
export { normalizeToMemory } from './memory-center-legacy.js';
/**
* Memory Center REST surface (S04, PRD §16.4) on the memory contract (ADR-0007).
*

View File

@@ -9,7 +9,6 @@
"@waggle/agent": ["../agent/src/index.ts"],
"@waggle/core": ["../core/src/index.ts"],
"@waggle/shared": ["../shared/src/index.ts"],
"@waggle/hive-mind-core": ["../hive-mind-core/src/index.ts"],
"@waggle/marketplace": ["../marketplace/src/index.ts"],
"@waggle/sdk": ["../sdk/src/index.ts"]
}