This commit is contained in:
233
packages/hive-mind-mcp-server/src/core/setup.ts
Normal file
233
packages/hive-mind-mcp-server/src/core/setup.ts
Normal file
@@ -0,0 +1,233 @@
|
||||
/**
|
||||
* Core setup — initialises MindDB, embedding provider, workspace manager.
|
||||
* All state lives here; tool handlers import from this module.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
|
||||
import {
|
||||
MindDB,
|
||||
FrameStore,
|
||||
HybridSearch,
|
||||
KnowledgeGraph,
|
||||
IdentityLayer,
|
||||
AwarenessLayer,
|
||||
SessionStore,
|
||||
WorkspaceManager,
|
||||
MultiMindCache,
|
||||
createEmbeddingProvider,
|
||||
HarvestSourceStore,
|
||||
ChatGPTAdapter,
|
||||
ClaudeAdapter,
|
||||
ClaudeCodeAdapter,
|
||||
GeminiAdapter,
|
||||
UniversalAdapter,
|
||||
MarkdownAdapter,
|
||||
PlaintextAdapter,
|
||||
UrlAdapter,
|
||||
PdfAdapter,
|
||||
type EmbeddingProviderInstance,
|
||||
type EmbeddingProviderConfig,
|
||||
} from '@waggle/hive-mind-core';
|
||||
|
||||
// ── Data directory ──────────────────────────────────────────────────
|
||||
|
||||
function resolveDataDir(): string {
|
||||
const envDir = process.env.HIVE_MIND_DATA_DIR;
|
||||
if (envDir) {
|
||||
const resolved = envDir.startsWith('~')
|
||||
? path.join(os.homedir(), envDir.slice(1))
|
||||
: envDir;
|
||||
return resolved;
|
||||
}
|
||||
return path.join(os.homedir(), '.hive-mind');
|
||||
}
|
||||
|
||||
// ── Singleton state ─────────────────────────────────────────────────
|
||||
|
||||
let _initialized = false;
|
||||
|
||||
let _dataDir: string;
|
||||
let _personalDb: MindDB;
|
||||
let _frameStore: FrameStore;
|
||||
let _search: HybridSearch;
|
||||
let _knowledgeGraph: KnowledgeGraph;
|
||||
let _identity: IdentityLayer;
|
||||
let _awareness: AwarenessLayer;
|
||||
let _sessions: SessionStore;
|
||||
let _workspaceManager: WorkspaceManager;
|
||||
let _mindCache: MultiMindCache;
|
||||
let _embedder: EmbeddingProviderInstance;
|
||||
let _harvestSourceStore: HarvestSourceStore;
|
||||
|
||||
// ── Harvest adapters (stateless, instantiate once) ──────────────────
|
||||
|
||||
const _chatgptAdapter = new ChatGPTAdapter();
|
||||
const _claudeAdapter = new ClaudeAdapter();
|
||||
const _claudeCodeAdapter = new ClaudeCodeAdapter();
|
||||
const _geminiAdapter = new GeminiAdapter();
|
||||
const _universalAdapter = new UniversalAdapter();
|
||||
const _markdownAdapter = new MarkdownAdapter();
|
||||
const _plaintextAdapter = new PlaintextAdapter();
|
||||
const _urlAdapter = new UrlAdapter();
|
||||
const _pdfAdapter = new PdfAdapter();
|
||||
|
||||
export function getAdapter(source: string) {
|
||||
switch (source) {
|
||||
case 'chatgpt': return _chatgptAdapter;
|
||||
case 'claude': return _claudeAdapter;
|
||||
case 'claude-code': return _claudeCodeAdapter;
|
||||
case 'gemini': return _geminiAdapter;
|
||||
case 'markdown': return _markdownAdapter;
|
||||
case 'plaintext': return _plaintextAdapter;
|
||||
case 'url': return _urlAdapter;
|
||||
case 'pdf': return _pdfAdapter;
|
||||
default: return _universalAdapter;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Initialization ──────────────────────────────────────────────────
|
||||
|
||||
export async function initialize(): Promise<void> {
|
||||
if (_initialized) return;
|
||||
|
||||
_dataDir = resolveDataDir();
|
||||
|
||||
// Ensure data directory exists
|
||||
fs.mkdirSync(_dataDir, { recursive: true });
|
||||
|
||||
const personalMindPath = path.join(_dataDir, 'personal.mind');
|
||||
|
||||
// Open the personal mind database
|
||||
_personalDb = new MindDB(personalMindPath);
|
||||
|
||||
// Initialize layers on personal mind
|
||||
_frameStore = new FrameStore(_personalDb);
|
||||
_knowledgeGraph = new KnowledgeGraph(_personalDb);
|
||||
_identity = new IdentityLayer(_personalDb);
|
||||
_awareness = new AwarenessLayer(_personalDb);
|
||||
_sessions = new SessionStore(_personalDb);
|
||||
_harvestSourceStore = new HarvestSourceStore(_personalDb);
|
||||
|
||||
// Resolve embedding provider — avoid surprise 23MB InProcess download.
|
||||
// Priority: explicit env → Ollama (if URL set) → API keys → mock fallback.
|
||||
// Users opt into InProcess via HIVE_MIND_EMBEDDING_PROVIDER=inprocess or =auto.
|
||||
const explicitProvider = process.env.HIVE_MIND_EMBEDDING_PROVIDER as EmbeddingProviderConfig['provider'] | undefined;
|
||||
let resolvedProvider: EmbeddingProviderConfig['provider'];
|
||||
|
||||
if (explicitProvider) {
|
||||
resolvedProvider = explicitProvider;
|
||||
} else if (process.env.OLLAMA_URL) {
|
||||
resolvedProvider = 'ollama';
|
||||
} else if (process.env.VOYAGE_API_KEY) {
|
||||
resolvedProvider = 'voyage';
|
||||
} else if (process.env.OPENAI_API_KEY) {
|
||||
resolvedProvider = 'openai';
|
||||
} else {
|
||||
resolvedProvider = 'mock';
|
||||
console.error('[hive-mind-memory] No embedding provider configured — using keyword search only.');
|
||||
console.error('[hive-mind-memory] For semantic search, set one of:');
|
||||
console.error('[hive-mind-memory] OLLAMA_URL=http://localhost:11434 (recommended, free)');
|
||||
console.error('[hive-mind-memory] HIVE_MIND_EMBEDDING_PROVIDER=inprocess (downloads 23MB model once)');
|
||||
console.error('[hive-mind-memory] OPENAI_API_KEY=sk-... (remote API)');
|
||||
}
|
||||
|
||||
const embeddingConfig: EmbeddingProviderConfig = {
|
||||
provider: resolvedProvider,
|
||||
targetDimensions: 1024,
|
||||
inprocess: {
|
||||
cacheDir: path.join(_dataDir, 'models'),
|
||||
},
|
||||
ollama: {
|
||||
baseUrl: process.env.OLLAMA_URL,
|
||||
model: process.env.OLLAMA_MODEL,
|
||||
},
|
||||
...(process.env.VOYAGE_API_KEY && {
|
||||
voyage: { apiKey: process.env.VOYAGE_API_KEY },
|
||||
}),
|
||||
...(process.env.OPENAI_API_KEY && {
|
||||
openai: { apiKey: process.env.OPENAI_API_KEY },
|
||||
}),
|
||||
};
|
||||
|
||||
_embedder = await createEmbeddingProvider(embeddingConfig);
|
||||
|
||||
// Initialize hybrid search with the embedding provider
|
||||
_search = new HybridSearch(_personalDb, _embedder);
|
||||
|
||||
// Initialize workspace manager
|
||||
_workspaceManager = new WorkspaceManager(_dataDir);
|
||||
|
||||
// LRU cache of open workspace MindDB handles (max 20)
|
||||
_mindCache = new MultiMindCache({
|
||||
maxOpen: 20,
|
||||
getMindPath: (workspaceId: string) => _workspaceManager.getMindPath(workspaceId),
|
||||
});
|
||||
|
||||
_initialized = true;
|
||||
}
|
||||
|
||||
// ── Accessors ───────────────────────────────────────────────────────
|
||||
|
||||
export function getDataDir(): string { return _dataDir; }
|
||||
export function getPersonalDb(): MindDB { return _personalDb; }
|
||||
export function getFrameStore(): FrameStore { return _frameStore; }
|
||||
export function getSearch(): HybridSearch { return _search; }
|
||||
export function getKnowledgeGraph(): KnowledgeGraph { return _knowledgeGraph; }
|
||||
export function getIdentity(): IdentityLayer { return _identity; }
|
||||
export function getAwareness(): AwarenessLayer { return _awareness; }
|
||||
export function getSessions(): SessionStore { return _sessions; }
|
||||
export function getWorkspaceManager(): WorkspaceManager { return _workspaceManager; }
|
||||
export function getMindCache(): MultiMindCache { return _mindCache; }
|
||||
export function getEmbedder(): EmbeddingProviderInstance { return _embedder; }
|
||||
export function getHarvestSourceStore(): HarvestSourceStore { return _harvestSourceStore; }
|
||||
|
||||
// ── Workspace mind layer cache ──────────────────────────────────────
|
||||
// Avoids re-creating FrameStore/HybridSearch/KnowledgeGraph/SessionStore
|
||||
// on every getWorkspaceMind() call. Invalidates when MindDB reference
|
||||
// changes (i.e. the LRU cache evicted and reopened it).
|
||||
|
||||
interface WorkspaceMindHandle {
|
||||
db: MindDB;
|
||||
frameStore: FrameStore;
|
||||
search: HybridSearch;
|
||||
knowledgeGraph: KnowledgeGraph;
|
||||
sessions: SessionStore;
|
||||
}
|
||||
|
||||
const _workspaceMindLayerCache = new Map<string, WorkspaceMindHandle>();
|
||||
|
||||
/**
|
||||
* Get a workspace's MindDB, FrameStore, HybridSearch, and KnowledgeGraph.
|
||||
* Opens the mind on demand via the LRU cache. Layers are cached and
|
||||
* invalidated when the underlying MindDB handle changes.
|
||||
*/
|
||||
export function getWorkspaceMind(workspaceId: string): WorkspaceMindHandle | null {
|
||||
const db = _mindCache.getOrOpen(workspaceId);
|
||||
if (!db) return null;
|
||||
|
||||
// Return cached layers if the MindDB reference is still the same
|
||||
const cached = _workspaceMindLayerCache.get(workspaceId);
|
||||
if (cached && cached.db === db) return cached;
|
||||
|
||||
const handle: WorkspaceMindHandle = {
|
||||
db,
|
||||
frameStore: new FrameStore(db),
|
||||
search: new HybridSearch(db, _embedder),
|
||||
knowledgeGraph: new KnowledgeGraph(db),
|
||||
sessions: new SessionStore(db),
|
||||
};
|
||||
_workspaceMindLayerCache.set(workspaceId, handle);
|
||||
return handle;
|
||||
}
|
||||
|
||||
// ── Shutdown ────────────────────────────────────────────────────────
|
||||
|
||||
export function shutdown(): void {
|
||||
_workspaceMindLayerCache.clear();
|
||||
_mindCache.closeAll();
|
||||
try { _personalDb?.close(); } catch { /* already closed */ }
|
||||
_initialized = false;
|
||||
}
|
||||
124
packages/hive-mind-mcp-server/src/index.ts
Normal file
124
packages/hive-mind-mcp-server/src/index.ts
Normal file
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Hive Mind Memory MCP Server
|
||||
*
|
||||
* Persistent memory for Claude Code, Claude Desktop, Codex, Hermes, and any
|
||||
* MCP-compatible AI system. Powered by @waggle/hive-mind-core — FrameStore +
|
||||
* HybridSearch + KnowledgeGraph + IdentityLayer + AwarenessLayer.
|
||||
*
|
||||
* Transport: stdio (Claude Code / Claude Desktop / Codex standard)
|
||||
* Data dir: HIVE_MIND_DATA_DIR (default: ~/.hive-mind)
|
||||
*/
|
||||
|
||||
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
||||
|
||||
import { initialize, shutdown } from './core/setup.js';
|
||||
import { registerMemoryTools } from './tools/memory.js';
|
||||
import { registerKnowledgeTools } from './tools/knowledge.js';
|
||||
import { registerIdentityTools } from './tools/identity.js';
|
||||
import { registerAwarenessTools } from './tools/awareness.js';
|
||||
import { registerWorkspaceTools } from './tools/workspace.js';
|
||||
import { registerHarvestTools } from './tools/harvest.js';
|
||||
import { registerCleanupTools } from './tools/cleanup.js';
|
||||
import { registerIngestTools } from './tools/ingest.js';
|
||||
import { registerWikiTools } from './tools/wiki.js';
|
||||
import { registerResources } from './resources/memory.js';
|
||||
import { parseScopes, isFullAccess, scopeGatedServer } from './scope.js';
|
||||
|
||||
// ── Server creation ─────────────────────────────────────────────────
|
||||
|
||||
const server = new McpServer(
|
||||
{
|
||||
name: 'hive-mind-memory',
|
||||
version: '0.1.0',
|
||||
},
|
||||
{
|
||||
capabilities: {
|
||||
resources: {},
|
||||
tools: {},
|
||||
logging: {},
|
||||
},
|
||||
instructions: [
|
||||
'Hive Mind gives you persistent memory across conversations.',
|
||||
'',
|
||||
'Core workflow:',
|
||||
'1. Use recall_memory FIRST to check if relevant context exists',
|
||||
'2. Use save_memory to persist important facts, decisions, and preferences',
|
||||
'3. Use search_entities to explore the knowledge graph',
|
||||
'4. Use get_identity / get_awareness for user context',
|
||||
'',
|
||||
'Wiki compiler:',
|
||||
'5. Use compile_wiki to build a personal wiki from your memories',
|
||||
'6. Use search_wiki / get_page to browse compiled knowledge',
|
||||
'7. Use compile_health to check data quality and find gaps',
|
||||
'8. Use ingest_source to add documents, URLs, or files to memory',
|
||||
'',
|
||||
'Memory is stored locally and persists across sessions.',
|
||||
'Workspaces provide isolated memory spaces for different projects.',
|
||||
].join('\n'),
|
||||
},
|
||||
);
|
||||
|
||||
// ── Scope gate (env-driven) ─────────────────────────────────────────
|
||||
// HIVE_MIND_SCOPES controls which tools register. Default (unset) = full
|
||||
// read+write for backward-compat; "memory:read" registers read tools only,
|
||||
// so a read-only client literally cannot call save/cleanup/ingest/etc.
|
||||
const scopes = parseScopes(process.env.HIVE_MIND_SCOPES);
|
||||
const target = isFullAccess(scopes) ? server : scopeGatedServer(server, scopes);
|
||||
|
||||
// ── Register all tools ──────────────────────────────────────────────
|
||||
|
||||
registerMemoryTools(target);
|
||||
registerKnowledgeTools(target);
|
||||
registerIdentityTools(target);
|
||||
registerAwarenessTools(target);
|
||||
registerWorkspaceTools(target);
|
||||
registerHarvestTools(target);
|
||||
registerCleanupTools(target);
|
||||
registerIngestTools(target);
|
||||
registerWikiTools(target);
|
||||
|
||||
// ── Register all resources ──────────────────────────────────────────
|
||||
|
||||
registerResources(target);
|
||||
|
||||
// ── Main ────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
// Initialize the memory engine (MindDB, embeddings, workspace manager)
|
||||
await initialize();
|
||||
|
||||
// Connect to stdio transport
|
||||
const transport = new StdioServerTransport();
|
||||
await server.connect(transport);
|
||||
|
||||
// Log to stderr (stdout is reserved for MCP protocol)
|
||||
console.error('Hive Mind Memory MCP server running on stdio');
|
||||
console.error(`Data directory: ${process.env.HIVE_MIND_DATA_DIR ?? '~/.hive-mind'}`);
|
||||
if (!isFullAccess(scopes)) {
|
||||
console.error(
|
||||
`Scope gate active: ${[...scopes].sort().join(', ')} — write tools withheld`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Graceful shutdown ───────────────────────────────────────────────
|
||||
|
||||
function handleShutdown(): void {
|
||||
console.error('Shutting down Hive Mind Memory MCP server...');
|
||||
shutdown();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
process.on('SIGINT', handleShutdown);
|
||||
process.on('SIGTERM', handleShutdown);
|
||||
|
||||
// ── Launch ──────────────────────────────────────────────────────────
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('Fatal error starting Hive Mind Memory MCP server:', err);
|
||||
shutdown();
|
||||
process.exit(1);
|
||||
});
|
||||
133
packages/hive-mind-mcp-server/src/integration.test.ts
Normal file
133
packages/hive-mind-mcp-server/src/integration.test.ts
Normal file
@@ -0,0 +1,133 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
|
||||
import { registerMemoryTools } from './tools/memory.js';
|
||||
import { registerKnowledgeTools } from './tools/knowledge.js';
|
||||
import { registerIdentityTools } from './tools/identity.js';
|
||||
import { registerAwarenessTools } from './tools/awareness.js';
|
||||
import { registerWorkspaceTools } from './tools/workspace.js';
|
||||
import { registerHarvestTools } from './tools/harvest.js';
|
||||
import { registerCleanupTools } from './tools/cleanup.js';
|
||||
import { registerIngestTools } from './tools/ingest.js';
|
||||
import { registerWikiTools } from './tools/wiki.js';
|
||||
import { registerResources } from './resources/memory.js';
|
||||
|
||||
/**
|
||||
* A hand-rolled stub that satisfies the subset of the McpServer API
|
||||
* the register functions call:
|
||||
*
|
||||
* server.tool(name, description, schema, handler)
|
||||
* server.resource(name, uri, handler)
|
||||
*
|
||||
* The stub captures `name` / `description` / `uri` so tests can assert
|
||||
* which tools got wired up, without spinning up a real MCP server or
|
||||
* the stdio transport.
|
||||
*/
|
||||
function makeStub(): {
|
||||
server: McpServer;
|
||||
tools: { name: string; description: string }[];
|
||||
resources: { name: string; uri: string }[];
|
||||
} {
|
||||
const tools: { name: string; description: string }[] = [];
|
||||
const resources: { name: string; uri: string }[] = [];
|
||||
const server = {
|
||||
tool: (name: string, description: string, _schema: unknown, _handler: unknown) => {
|
||||
tools.push({ name, description });
|
||||
},
|
||||
resource: (name: string, uri: string, _handler: unknown) => {
|
||||
resources.push({ name, uri });
|
||||
},
|
||||
} as unknown as McpServer;
|
||||
return { server, tools, resources };
|
||||
}
|
||||
|
||||
describe('@waggle/hive-mind-mcp-server registration wiring', () => {
|
||||
it('registerMemoryTools registers save_memory + recall_memory', () => {
|
||||
const { server, tools } = makeStub();
|
||||
registerMemoryTools(server);
|
||||
expect(tools.map((t) => t.name).sort()).toEqual(['recall_memory', 'save_memory']);
|
||||
// Every tool description should be non-trivial (> 30 chars) so the MCP
|
||||
// client actually sees useful guidance.
|
||||
for (const t of tools) expect(t.description.length).toBeGreaterThan(30);
|
||||
});
|
||||
|
||||
it('registerKnowledgeTools registers search_entities + save_entity + create_relation', () => {
|
||||
const { server, tools } = makeStub();
|
||||
registerKnowledgeTools(server);
|
||||
expect(tools.map((t) => t.name).sort()).toEqual(['create_relation', 'save_entity', 'search_entities']);
|
||||
});
|
||||
|
||||
it('registerIdentityTools registers get_identity + set_identity', () => {
|
||||
const { server, tools } = makeStub();
|
||||
registerIdentityTools(server);
|
||||
expect(tools.map((t) => t.name).sort()).toEqual(['get_identity', 'set_identity']);
|
||||
});
|
||||
|
||||
it('registerAwarenessTools registers get/set/clear_awareness', () => {
|
||||
const { server, tools } = makeStub();
|
||||
registerAwarenessTools(server);
|
||||
expect(tools.map((t) => t.name).sort()).toEqual(['clear_awareness', 'get_awareness', 'set_awareness']);
|
||||
});
|
||||
|
||||
it('registerWorkspaceTools registers list_workspaces + create_workspace', () => {
|
||||
const { server, tools } = makeStub();
|
||||
registerWorkspaceTools(server);
|
||||
expect(tools.map((t) => t.name).sort()).toEqual(['create_workspace', 'list_workspaces']);
|
||||
});
|
||||
|
||||
it('registerHarvestTools registers harvest_import + harvest_sources', () => {
|
||||
const { server, tools } = makeStub();
|
||||
registerHarvestTools(server);
|
||||
expect(tools.map((t) => t.name).sort()).toEqual(['harvest_import', 'harvest_sources']);
|
||||
});
|
||||
|
||||
it('registerCleanupTools registers cleanup_frames + cleanup_entities', () => {
|
||||
const { server, tools } = makeStub();
|
||||
registerCleanupTools(server);
|
||||
expect(tools.map((t) => t.name).sort()).toEqual(['cleanup_entities', 'cleanup_frames']);
|
||||
});
|
||||
|
||||
it('registerIngestTools registers ingest_source', () => {
|
||||
const { server, tools } = makeStub();
|
||||
registerIngestTools(server);
|
||||
expect(tools.map((t) => t.name)).toEqual(['ingest_source']);
|
||||
});
|
||||
|
||||
it('registerWikiTools registers compile_wiki + get_page + search_wiki + compile_health', () => {
|
||||
const { server, tools } = makeStub();
|
||||
registerWikiTools(server);
|
||||
expect(tools.map((t) => t.name).sort()).toEqual([
|
||||
'compile_health', 'compile_wiki', 'get_page', 'search_wiki',
|
||||
]);
|
||||
});
|
||||
|
||||
it('registerResources registers the 4 memory:// resource roots', () => {
|
||||
const { server, resources } = makeStub();
|
||||
registerResources(server);
|
||||
const names = resources.map((r) => r.name).sort();
|
||||
expect(names).toEqual(['awareness', 'identity', 'personal-stats', 'workspace']);
|
||||
// All URIs should be under the memory:// scheme so the client namespaces them together.
|
||||
for (const r of resources) expect(r.uri.startsWith('memory://')).toBe(true);
|
||||
});
|
||||
|
||||
it('full registration surface adds up to 21 tools + 4 resources', () => {
|
||||
const { server, tools, resources } = makeStub();
|
||||
registerMemoryTools(server);
|
||||
registerKnowledgeTools(server);
|
||||
registerIdentityTools(server);
|
||||
registerAwarenessTools(server);
|
||||
registerWorkspaceTools(server);
|
||||
registerHarvestTools(server);
|
||||
registerCleanupTools(server);
|
||||
registerIngestTools(server);
|
||||
registerWikiTools(server);
|
||||
registerResources(server);
|
||||
|
||||
expect(tools).toHaveLength(21);
|
||||
expect(resources).toHaveLength(4);
|
||||
|
||||
// Every tool name is unique — duplicates would cause MCP registration errors at runtime.
|
||||
const names = new Set(tools.map((t) => t.name));
|
||||
expect(names.size).toBe(tools.length);
|
||||
});
|
||||
});
|
||||
188
packages/hive-mind-mcp-server/src/resources/memory.ts
Normal file
188
packages/hive-mind-mcp-server/src/resources/memory.ts
Normal file
@@ -0,0 +1,188 @@
|
||||
/**
|
||||
* MCP Resource handlers — expose memory state as readable resources.
|
||||
*
|
||||
* Resources:
|
||||
* memory://personal/stats → frame count, entity count, embedding status
|
||||
* memory://workspace/{id} → workspace info + memory stats
|
||||
* memory://identity → current identity profile
|
||||
* memory://awareness → current awareness items
|
||||
*/
|
||||
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import {
|
||||
getFrameStore,
|
||||
getKnowledgeGraph,
|
||||
getIdentity,
|
||||
getAwareness,
|
||||
getEmbedder,
|
||||
getWorkspaceManager,
|
||||
getWorkspaceMind,
|
||||
} from '../core/setup.js';
|
||||
|
||||
export function registerResources(server: McpServer): void {
|
||||
|
||||
// ── memory://personal/stats ─────────────────────────────────────
|
||||
server.resource(
|
||||
'personal-stats',
|
||||
'memory://personal/stats',
|
||||
async (uri) => {
|
||||
const frameStore = getFrameStore();
|
||||
const kg = getKnowledgeGraph();
|
||||
const embedder = getEmbedder();
|
||||
|
||||
const stats = frameStore.getStats();
|
||||
const entityCount = kg.getEntityCount();
|
||||
const entityTypes = kg.getEntityTypeCounts();
|
||||
const embeddingStatus = embedder.getStatus();
|
||||
|
||||
const data = {
|
||||
uri: uri.href,
|
||||
frames: {
|
||||
total: stats.total,
|
||||
by_type: stats.byType,
|
||||
by_importance: stats.byImportance,
|
||||
},
|
||||
knowledge_graph: {
|
||||
entities: entityCount,
|
||||
entity_types: entityTypes,
|
||||
},
|
||||
embedding: {
|
||||
provider: embeddingStatus.activeProvider,
|
||||
model: embeddingStatus.modelName,
|
||||
dimensions: embeddingStatus.dimensions,
|
||||
available_providers: embeddingStatus.availableProviders,
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
contents: [{
|
||||
uri: uri.href,
|
||||
mimeType: 'application/json',
|
||||
text: JSON.stringify(data, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// ── memory://identity ───────────────────────────────────────────
|
||||
server.resource(
|
||||
'identity',
|
||||
'memory://identity',
|
||||
async (uri) => {
|
||||
const identity = getIdentity();
|
||||
|
||||
let data: Record<string, unknown>;
|
||||
if (!identity.exists()) {
|
||||
data = { configured: false };
|
||||
} else {
|
||||
const id = identity.get();
|
||||
data = {
|
||||
configured: true,
|
||||
name: id.name,
|
||||
role: id.role,
|
||||
department: id.department,
|
||||
personality: id.personality,
|
||||
capabilities: id.capabilities,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
contents: [{
|
||||
uri: uri.href,
|
||||
mimeType: 'application/json',
|
||||
text: JSON.stringify(data, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// ── memory://awareness ──────────────────────────────────────────
|
||||
server.resource(
|
||||
'awareness',
|
||||
'memory://awareness',
|
||||
async (uri) => {
|
||||
const awareness = getAwareness();
|
||||
const items = awareness.getAll();
|
||||
|
||||
const data = items.map(item => ({
|
||||
id: item.id,
|
||||
category: item.category,
|
||||
content: item.content,
|
||||
priority: item.priority,
|
||||
expires_at: item.expires_at,
|
||||
created_at: item.created_at,
|
||||
}));
|
||||
|
||||
return {
|
||||
contents: [{
|
||||
uri: uri.href,
|
||||
mimeType: 'application/json',
|
||||
text: JSON.stringify(data, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// ── memory://workspace/{id} (resource template) ─────────────────
|
||||
server.resource(
|
||||
'workspace',
|
||||
'memory://workspace/{id}',
|
||||
async (uri) => {
|
||||
// Extract workspace ID from URI
|
||||
const match = uri.href.match(/memory:\/\/workspace\/(.+)/);
|
||||
const workspaceId = match?.[1];
|
||||
|
||||
if (!workspaceId) {
|
||||
return {
|
||||
contents: [{
|
||||
uri: uri.href,
|
||||
mimeType: 'application/json',
|
||||
text: JSON.stringify({ error: 'Invalid workspace URI' }),
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
const wm = getWorkspaceManager();
|
||||
const config = wm.get(workspaceId);
|
||||
|
||||
if (!config) {
|
||||
return {
|
||||
contents: [{
|
||||
uri: uri.href,
|
||||
mimeType: 'application/json',
|
||||
text: JSON.stringify({ error: `Workspace not found: ${workspaceId}` }),
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
// Get workspace mind stats
|
||||
let stats = { frames: 0, entities: 0 };
|
||||
try {
|
||||
const mind = getWorkspaceMind(workspaceId);
|
||||
if (mind) {
|
||||
const fs = mind.frameStore.getStats();
|
||||
stats = { frames: fs.total, entities: mind.knowledgeGraph.getEntityCount() };
|
||||
}
|
||||
} catch { /* workspace mind not accessible */ }
|
||||
|
||||
const data = {
|
||||
id: config.id,
|
||||
name: config.name,
|
||||
group: config.group,
|
||||
icon: config.icon ?? null,
|
||||
model: config.model ?? null,
|
||||
created: config.created,
|
||||
frames: stats.frames,
|
||||
entities: stats.entities,
|
||||
};
|
||||
|
||||
return {
|
||||
contents: [{
|
||||
uri: uri.href,
|
||||
mimeType: 'application/json',
|
||||
text: JSON.stringify(data, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
132
packages/hive-mind-mcp-server/src/scope.ts
Normal file
132
packages/hive-mind-mcp-server/src/scope.ts
Normal file
@@ -0,0 +1,132 @@
|
||||
/**
|
||||
* MCP tool scope gate.
|
||||
*
|
||||
* Controls WHICH tools get registered at server start based on an env-driven
|
||||
* scope set, so a read-only token literally cannot mutate the substrate:
|
||||
*
|
||||
* <ENV>=memory:read -> only read tools registered
|
||||
* <ENV>=memory:read,memory:write -> full read + write
|
||||
*
|
||||
* Default (env unset/empty) = full read + write, for backward-compat.
|
||||
* Granting memory:write auto-adds memory:read (write-implies-read).
|
||||
*
|
||||
* No HTTP / token-minting here — registration gating + scope parsing only.
|
||||
* Aligns with the mind-isolation rule: this never crosses minds, it only
|
||||
* narrows the tool surface a client sees.
|
||||
*/
|
||||
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
|
||||
export type McpScope = 'memory:read' | 'memory:write';
|
||||
|
||||
/** Mutating tools — withheld unless the scope set contains `memory:write`. */
|
||||
export const WRITE_TOOLS: ReadonlySet<string> = new Set<string>([
|
||||
'save_memory',
|
||||
'save_entity',
|
||||
'create_relation',
|
||||
'set_identity',
|
||||
'set_awareness',
|
||||
'clear_awareness',
|
||||
'create_workspace',
|
||||
'harvest_import',
|
||||
'cleanup_frames',
|
||||
'cleanup_entities',
|
||||
'ingest_source',
|
||||
'compile_wiki',
|
||||
]);
|
||||
|
||||
/** Read-only tools — registered whenever the scope set contains `memory:read`. */
|
||||
export const READ_TOOLS: ReadonlySet<string> = new Set<string>([
|
||||
'recall_memory',
|
||||
'search_entities',
|
||||
'get_identity',
|
||||
'get_awareness',
|
||||
'list_workspaces',
|
||||
'harvest_sources',
|
||||
'get_page',
|
||||
'search_wiki',
|
||||
'compile_health',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Classify a tool name. Unknown names fall back to 'write' (fail safe): a
|
||||
* read-only token never exposes a tool we have not explicitly vetted as read.
|
||||
*/
|
||||
export function toolScope(name: string): 'read' | 'write' {
|
||||
return READ_TOOLS.has(name) ? 'read' : 'write';
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a comma-separated scope string into a normalized scope set.
|
||||
*
|
||||
* - undefined / empty / whitespace -> full read+write (backward-compat default)
|
||||
* - explicit but no valid token -> falls closed to read-only
|
||||
* - memory:write -> auto-adds memory:read (write-implies-read)
|
||||
*/
|
||||
export function parseScopes(raw: string | undefined): ReadonlySet<McpScope> {
|
||||
if (raw === undefined || raw.trim() === '') {
|
||||
return new Set<McpScope>(['memory:read', 'memory:write']);
|
||||
}
|
||||
|
||||
const set = new Set<McpScope>();
|
||||
for (const token of raw.split(',')) {
|
||||
const t = token.trim().toLowerCase();
|
||||
if (t === 'memory:read' || t === 'memory:write') set.add(t);
|
||||
// Unknown tokens are ignored — never silently grant an unrecognized scope.
|
||||
}
|
||||
|
||||
// Explicit-but-unrecognized value: fall closed to read-only.
|
||||
if (set.size === 0) set.add('memory:read');
|
||||
|
||||
// write-implies-read expansion.
|
||||
if (set.has('memory:write')) set.add('memory:read');
|
||||
|
||||
return set;
|
||||
}
|
||||
|
||||
/** True when every tool (read + write) is permitted — the default path. */
|
||||
export function isFullAccess(scopes: ReadonlySet<McpScope>): boolean {
|
||||
return scopes.has('memory:read') && scopes.has('memory:write');
|
||||
}
|
||||
|
||||
/** Decide whether a single tool may be registered under the given scopes. */
|
||||
export function isToolAllowed(name: string, scopes: ReadonlySet<McpScope>): boolean {
|
||||
return toolScope(name) === 'write'
|
||||
? scopes.has('memory:write')
|
||||
: scopes.has('memory:read');
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap an McpServer so that `.tool(name, ...)` registrations outside the granted
|
||||
* scopes are silently skipped. All other members (`.resource`, `.connect`, …)
|
||||
* pass straight through, bound to the real server so private-field access keeps
|
||||
* working. Immutable: the original server is never mutated.
|
||||
*/
|
||||
export function scopeGatedServer(
|
||||
server: McpServer,
|
||||
scopes: ReadonlySet<McpScope>,
|
||||
): McpServer {
|
||||
return new Proxy(server, {
|
||||
// Resolve every member against the REAL target (receiver = target), so SDK
|
||||
// getters/methods that read private #fields (e.g. `.connect`) never run with
|
||||
// `this` bound to the proxy. Only `.tool` registrations are gated.
|
||||
get(target, prop) {
|
||||
if (prop === 'tool') {
|
||||
const original = Reflect.get(target, prop, target) as (
|
||||
...args: unknown[]
|
||||
) => unknown;
|
||||
return (...args: unknown[]): unknown => {
|
||||
const name = args[0];
|
||||
if (typeof name === 'string' && !isToolAllowed(name, scopes)) {
|
||||
return undefined;
|
||||
}
|
||||
return original.apply(target, args);
|
||||
};
|
||||
}
|
||||
const value = Reflect.get(target, prop, target);
|
||||
return typeof value === 'function'
|
||||
? (value as (...a: unknown[]) => unknown).bind(target)
|
||||
: value;
|
||||
},
|
||||
});
|
||||
}
|
||||
139
packages/hive-mind-mcp-server/src/tools/awareness.ts
Normal file
139
packages/hive-mind-mcp-server/src/tools/awareness.ts
Normal file
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* Awareness tools — get_awareness + set_awareness + clear_awareness.
|
||||
* Wraps AwarenessLayer from @waggle/hive-mind-core.
|
||||
*/
|
||||
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { z } from 'zod';
|
||||
import { getAwareness } from '../core/setup.js';
|
||||
import type { AwarenessCategory } from '@waggle/hive-mind-core';
|
||||
|
||||
export function registerAwarenessTools(server: McpServer): void {
|
||||
|
||||
// ── get_awareness ───────────────────────────────────────────────
|
||||
server.tool(
|
||||
'get_awareness',
|
||||
'Get current awareness items — active tasks, pending actions, context flags. These are short-lived context items that help the AI maintain situational awareness.',
|
||||
{
|
||||
category: z.enum(['task', 'action', 'pending', 'flag']).optional()
|
||||
.describe('Filter by category. Omit to get all'),
|
||||
},
|
||||
async ({ category }) => {
|
||||
const awareness = getAwareness();
|
||||
|
||||
const items = category
|
||||
? awareness.getByCategory(category as AwarenessCategory)
|
||||
: awareness.getAll();
|
||||
|
||||
if (items.length === 0) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: 'No active awareness items.',
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
const formatted = items.map(item => ({
|
||||
id: item.id,
|
||||
category: item.category,
|
||||
content: item.content,
|
||||
priority: item.priority,
|
||||
expires_at: item.expires_at,
|
||||
created_at: item.created_at,
|
||||
}));
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify(formatted, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// ── set_awareness ───────────────────────────────────────────────
|
||||
server.tool(
|
||||
'set_awareness',
|
||||
'Set an awareness item — a short-lived context marker for active tasks, pending actions, or flags. Items can auto-expire.',
|
||||
{
|
||||
category: z.enum(['task', 'action', 'pending', 'flag'])
|
||||
.describe('Item category: task (active work), action (recent action), pending (waiting), flag (context note)'),
|
||||
content: z.string().describe('The awareness content'),
|
||||
priority: z.number().min(0).max(10).optional()
|
||||
.describe('Priority 0-10, higher = more important. Defaults to 0'),
|
||||
ttl_minutes: z.number().min(1).optional()
|
||||
.describe('Time-to-live in minutes. Item auto-expires after this duration'),
|
||||
},
|
||||
async ({ category, content, priority, ttl_minutes }) => {
|
||||
const awareness = getAwareness();
|
||||
|
||||
let expiresAt: string | undefined;
|
||||
if (ttl_minutes) {
|
||||
const expiry = new Date(Date.now() + ttl_minutes * 60_000);
|
||||
expiresAt = expiry.toISOString();
|
||||
}
|
||||
|
||||
const item = awareness.add(
|
||||
category as AwarenessCategory,
|
||||
content,
|
||||
priority ?? 0,
|
||||
expiresAt,
|
||||
);
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
id: item.id,
|
||||
category: item.category,
|
||||
content: item.content,
|
||||
priority: item.priority,
|
||||
expires_at: item.expires_at,
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// ── clear_awareness ─────────────────────────────────────────────
|
||||
server.tool(
|
||||
'clear_awareness',
|
||||
'Remove an awareness item by ID, or clear all items in a category.',
|
||||
{
|
||||
id: z.number().optional().describe('Specific item ID to remove'),
|
||||
category: z.enum(['task', 'action', 'pending', 'flag']).optional()
|
||||
.describe('Clear all items in this category'),
|
||||
},
|
||||
async ({ id, category }) => {
|
||||
const awareness = getAwareness();
|
||||
|
||||
if (id !== undefined) {
|
||||
awareness.remove(id);
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: `Removed awareness item #${id}`,
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
if (category) {
|
||||
awareness.clearCategory(category as AwarenessCategory);
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: `Cleared all "${category}" awareness items`,
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: 'Provide either an id or category to clear.',
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
590
packages/hive-mind-mcp-server/src/tools/cleanup.ts
Normal file
590
packages/hive-mind-mcp-server/src/tools/cleanup.ts
Normal file
@@ -0,0 +1,590 @@
|
||||
/**
|
||||
* Cleanup tools — data maintenance for the memory system.
|
||||
*
|
||||
* cleanup_frames: Wipe test pollution, compact stale frames, reconcile indexes.
|
||||
* cleanup_entities: Delete misclassified KG entities, dedup, retire orphans.
|
||||
*/
|
||||
|
||||
import { spawn } from 'node:child_process';
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { z } from 'zod';
|
||||
import {
|
||||
getPersonalDb,
|
||||
getFrameStore,
|
||||
getKnowledgeGraph,
|
||||
getEmbedder,
|
||||
getSearch,
|
||||
getWorkspaceMind,
|
||||
} from '../core/setup.js';
|
||||
import {
|
||||
reconcileIndexes,
|
||||
normalizeEntityName,
|
||||
collectObservations,
|
||||
detectSupersessionChains,
|
||||
detectEntityGroups,
|
||||
applyConsolidation,
|
||||
type MindDB,
|
||||
type FrameStore,
|
||||
type HybridSearch,
|
||||
type ConsolidationLlm,
|
||||
} from '@waggle/hive-mind-core';
|
||||
|
||||
// Common nouns that get misclassified as person/project entities
|
||||
const NOISE_ENTITY_NAMES = new Set([
|
||||
'begin week', 'end week', 'begin day', 'end day',
|
||||
'test', 'testing', 'tests', 'todo', 'todos', 'fix', 'bug',
|
||||
'error', 'warning', 'success', 'failure', 'result', 'results',
|
||||
'start', 'stop', 'begin', 'end', 'run', 'running',
|
||||
'true', 'false', 'null', 'undefined', 'none',
|
||||
'yes', 'no', 'ok', 'okay',
|
||||
'step 1', 'step 2', 'step 3', 'step 4', 'step 5',
|
||||
'phase 1', 'phase 2', 'phase 3', 'phase 4',
|
||||
'part 1', 'part 2', 'part 3',
|
||||
'item', 'items', 'thing', 'things', 'stuff',
|
||||
'data', 'file', 'files', 'folder', 'path',
|
||||
'input', 'output', 'response', 'request',
|
||||
'user', 'admin', 'system', 'server', 'client',
|
||||
'the', 'a', 'an', 'this', 'that',
|
||||
]);
|
||||
|
||||
function isNoiseEntity(name: string, entityType: string): boolean {
|
||||
const lower = name.toLowerCase().trim();
|
||||
|
||||
// Very short names are usually noise
|
||||
if (lower.length <= 2) return true;
|
||||
|
||||
// Check the noise list
|
||||
if (NOISE_ENTITY_NAMES.has(lower)) return true;
|
||||
|
||||
// Single character or number-only names
|
||||
if (/^\d+$/.test(lower)) return true;
|
||||
|
||||
// Common nouns misclassified as person
|
||||
if (entityType === 'person') {
|
||||
// Names that are clearly not people
|
||||
if (/^(step|phase|part|section|item|task|bug|fix|test)\b/i.test(lower)) return true;
|
||||
// Names that are too generic
|
||||
if (/^(the|a|an|this|that|my|your)\s/i.test(lower)) return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// ── P/B consolidation executor ─────────────────────────────────────
|
||||
// The core supersede.ts module is provider-agnostic (pure) — the LLM transport
|
||||
// lives here at the call site. Default: zero-key `claude -p` subprocess. An
|
||||
// OpenAI-style model id + OPENAI_API_KEY routes to the OpenAI chat API — the
|
||||
// executor the benchmark validated with.
|
||||
|
||||
function spawnClaudeText(prompt: string, timeoutMs = 120_000): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const proc = spawn('claude', ['-p', '--output-format=text'], {
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
shell: process.platform === 'win32',
|
||||
env: { ...process.env, HIVE_MIND_NO_SYNTH: '1' },
|
||||
});
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
let settled = false;
|
||||
const timer = setTimeout(() => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
try { proc.kill('SIGKILL'); } catch { /* noop */ }
|
||||
reject(new Error(`claude -p timed out after ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
proc.stdout.on('data', (d: Buffer) => { stdout += d.toString('utf8'); });
|
||||
proc.stderr.on('data', (d: Buffer) => { stderr += d.toString('utf8'); });
|
||||
proc.on('error', (err) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
reject(new Error(`spawn claude failed: ${err.message}`));
|
||||
});
|
||||
proc.on('close', (code) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
if (code === 0) resolve(stdout.trim());
|
||||
else reject(new Error(`claude -p exited ${code}: ${stderr.slice(0, 300)}`));
|
||||
});
|
||||
proc.stdin.write(prompt);
|
||||
proc.stdin.end();
|
||||
});
|
||||
}
|
||||
|
||||
async function callOpenAIChat(model: string, system: string, user: string, timeoutMs = 120_000): Promise<string> {
|
||||
const key = process.env.OPENAI_API_KEY;
|
||||
if (!key) throw new Error('consolidation consolidate_model requires OPENAI_API_KEY');
|
||||
const ctrl = new AbortController();
|
||||
const timer = setTimeout(() => ctrl.abort(), timeoutMs);
|
||||
try {
|
||||
const res = await fetch('https://api.openai.com/v1/chat/completions', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
messages: [{ role: 'system', content: system }, { role: 'user', content: user }],
|
||||
temperature: 0,
|
||||
response_format: { type: 'json_object' },
|
||||
}),
|
||||
signal: ctrl.signal,
|
||||
});
|
||||
const text = await res.text();
|
||||
if (!res.ok) throw new Error(`openai ${res.status}: ${text.slice(0, 200)}`);
|
||||
return (JSON.parse(text).choices?.[0]?.message?.content ?? '').trim();
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
function buildConsolidationLlm(model?: string): ConsolidationLlm {
|
||||
if (model && /^(gpt-|o[0-9])/.test(model)) {
|
||||
return (system, user) => callOpenAIChat(model, system, user);
|
||||
}
|
||||
return (system, user) => spawnClaudeText(`${system}\n\n${user}`);
|
||||
}
|
||||
|
||||
function bridgeIndexText(frame: { content: string }): string {
|
||||
try {
|
||||
const parsed = JSON.parse(frame.content) as { description?: string; references?: unknown[] };
|
||||
const n = Array.isArray(parsed.references) ? parsed.references.length : 0;
|
||||
return `${parsed.description ?? 'group'}: bridge of ${n} items`;
|
||||
} catch {
|
||||
return frame.content;
|
||||
}
|
||||
}
|
||||
|
||||
interface ConsolidationCounts {
|
||||
chains: number;
|
||||
groups: number;
|
||||
pframes: number;
|
||||
bframes: number;
|
||||
deprecated: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the P/B consolidation pass on a mind: gather I-frame observations,
|
||||
* LLM-detect chains + groups, apply (deprecate stale + emit P/B frames), then
|
||||
* frames anchor to the newest observation's gop.
|
||||
*/
|
||||
async function runConsolidation(
|
||||
db: MindDB,
|
||||
frameStore: FrameStore,
|
||||
search: HybridSearch,
|
||||
model?: string,
|
||||
limit = 400,
|
||||
): Promise<ConsolidationCounts> {
|
||||
const empty: ConsolidationCounts = { chains: 0, groups: 0, pframes: 0, bframes: 0, deprecated: 0 };
|
||||
const observations = collectObservations(db, { limit });
|
||||
if (observations.length < 2) return empty;
|
||||
|
||||
const anchor = db
|
||||
.getDatabase()
|
||||
.prepare(
|
||||
"SELECT gop_id FROM memory_frames WHERE frame_type = 'I' AND importance != 'deprecated' ORDER BY created_at DESC, id DESC LIMIT 1",
|
||||
)
|
||||
.get() as { gop_id: string } | undefined;
|
||||
if (!anchor) return empty;
|
||||
|
||||
const llm = buildConsolidationLlm(model);
|
||||
const [chains, groups] = await Promise.all([
|
||||
detectSupersessionChains(observations, llm),
|
||||
detectEntityGroups(observations, llm),
|
||||
]);
|
||||
const { pframes, bframes, deprecated } = applyConsolidation(frameStore, chains, groups, anchor.gop_id);
|
||||
|
||||
const toIndex = [
|
||||
...pframes.map((f) => ({ id: f.id, content: f.content })),
|
||||
...bframes.map((f) => ({ id: f.id, content: bridgeIndexText(f) })),
|
||||
];
|
||||
if (toIndex.length > 0) {
|
||||
try {
|
||||
await search.indexFramesBatch(toIndex);
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
process.stderr.write(`[consolidate] vec-index failed (frames remain FTS-searchable): ${msg}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
chains: chains.length,
|
||||
groups: groups.length,
|
||||
pframes: pframes.length,
|
||||
bframes: bframes.length,
|
||||
deprecated: deprecated.length,
|
||||
};
|
||||
}
|
||||
|
||||
export function registerCleanupTools(server: McpServer): void {
|
||||
|
||||
// ── cleanup_frames ─────────────────────────────────────────────
|
||||
server.tool(
|
||||
'cleanup_frames',
|
||||
'Maintenance tool: compact stale frames, remove test pollution, and reconcile search indexes. Use with mode="compact" for routine maintenance, or mode="wipe_imports" to remove all imported frames (e.g., E2E test data).',
|
||||
{
|
||||
mode: z.enum(['compact', 'wipe_imports', 'wipe_all', 'reconcile'])
|
||||
.describe('compact: prune old temp/deprecated + merge P-frames. wipe_imports: delete all source=import frames. wipe_all: delete ALL frames (DANGER). reconcile: repair FTS/vector indexes.'),
|
||||
workspace: z.string().optional()
|
||||
.describe('Workspace ID. Omit for personal mind.'),
|
||||
max_temp_age_days: z.number().optional()
|
||||
.describe('For compact mode: delete temporary frames older than N days (default 30)'),
|
||||
max_deprecated_age_days: z.number().optional()
|
||||
.describe('For compact mode: delete deprecated frames older than N days (default 90)'),
|
||||
consolidate: z.boolean().optional()
|
||||
.describe('For compact mode: additionally run P/B consolidation — LLM-detect supersession chains (deprecate stale I-frames + emit a current-value P-frame) and enumerable entity groups (emit a B-frame per group), then vec-index the new frames. Requires an LLM (see consolidate_model).'),
|
||||
consolidate_model: z.string().optional()
|
||||
.describe('LLM for consolidate: an OpenAI-style id (e.g. gpt-4o-mini, needs OPENAI_API_KEY) uses the OpenAI API; otherwise the zero-key `claude -p` subprocess.'),
|
||||
},
|
||||
async ({ mode, workspace, max_temp_age_days, max_deprecated_age_days, consolidate, consolidate_model }) => {
|
||||
const db = workspace
|
||||
? getWorkspaceMind(workspace)?.db ?? null
|
||||
: getPersonalDb();
|
||||
|
||||
if (!db) {
|
||||
return {
|
||||
content: [{ type: 'text' as const, text: `Workspace "${workspace}" not found.` }],
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
|
||||
const frameStore = workspace
|
||||
? getWorkspaceMind(workspace)!.frameStore
|
||||
: getFrameStore();
|
||||
const raw = db.getDatabase();
|
||||
|
||||
if (mode === 'compact') {
|
||||
const result = frameStore.compact(
|
||||
max_temp_age_days ?? 30,
|
||||
max_deprecated_age_days ?? 90,
|
||||
);
|
||||
// Optional P/B consolidation pass, additive to compaction.
|
||||
let consolidation: ConsolidationCounts | undefined;
|
||||
if (consolidate) {
|
||||
const search = workspace
|
||||
? getWorkspaceMind(workspace)!.search
|
||||
: getSearch();
|
||||
consolidation = await runConsolidation(db, frameStore, search, consolidate_model);
|
||||
}
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
action: 'compact',
|
||||
temporary_pruned: result.temporaryPruned,
|
||||
deprecated_pruned: result.deprecatedPruned,
|
||||
pframes_merged: result.pframesMerged,
|
||||
...(consolidation ? { consolidation } : {}),
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
if (mode === 'wipe_imports') {
|
||||
// Delete all frames with source='import'
|
||||
const countRow = raw.prepare(
|
||||
"SELECT COUNT(*) as cnt FROM memory_frames WHERE source = 'import'",
|
||||
).get() as { cnt: number };
|
||||
|
||||
if (countRow.cnt === 0) {
|
||||
return {
|
||||
content: [{ type: 'text' as const, text: 'No imported frames found.' }],
|
||||
};
|
||||
}
|
||||
|
||||
// Get IDs for cascade cleanup
|
||||
const frameIds = raw.prepare(
|
||||
"SELECT id FROM memory_frames WHERE source = 'import'",
|
||||
).all() as { id: number }[];
|
||||
|
||||
const deleteTx = raw.transaction(() => {
|
||||
for (const { id } of frameIds) {
|
||||
// Clean FTS
|
||||
raw.prepare('DELETE FROM memory_frames_fts WHERE rowid = ?').run(id);
|
||||
// Clean vector
|
||||
try { raw.prepare('DELETE FROM memory_frames_vec WHERE rowid = ?').run(id); } catch { /* ok */ }
|
||||
}
|
||||
// Bulk delete frames
|
||||
raw.prepare("DELETE FROM memory_frames WHERE source = 'import'").run();
|
||||
});
|
||||
deleteTx();
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
action: 'wipe_imports',
|
||||
frames_deleted: countRow.cnt,
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
if (mode === 'wipe_all') {
|
||||
const stats = frameStore.getStats();
|
||||
|
||||
const deleteTx = raw.transaction(() => {
|
||||
raw.prepare('DELETE FROM memory_frames_fts').run();
|
||||
try { raw.prepare('DELETE FROM memory_frames_vec').run(); } catch { /* ok */ }
|
||||
raw.prepare('DELETE FROM memory_frames').run();
|
||||
raw.prepare('DELETE FROM sessions').run();
|
||||
});
|
||||
deleteTx();
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
action: 'wipe_all',
|
||||
frames_deleted: stats.total,
|
||||
warning: 'ALL frames and sessions deleted. This cannot be undone.',
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
if (mode === 'reconcile') {
|
||||
const result = await reconcileIndexes(db, getEmbedder());
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
action: 'reconcile',
|
||||
fts_fixed: result.ftsFixed,
|
||||
vec_fixed: result.vecFixed,
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{ type: 'text' as const, text: `Unknown mode: ${mode}` }],
|
||||
isError: true,
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// ── cleanup_entities ───────────────────────────────────────────
|
||||
server.tool(
|
||||
'cleanup_entities',
|
||||
'Maintenance tool: remove noise entities from the knowledge graph, deduplicate by normalized name, and retire orphan entities with no relations.',
|
||||
{
|
||||
mode: z.enum(['audit', 'remove_noise', 'dedup', 'retire_orphans', 'wipe_all'])
|
||||
.describe('audit: report noise + duplicates without deleting. remove_noise: delete misclassified entities. dedup: merge duplicate entities. retire_orphans: soft-delete entities with 0 relations. wipe_all: delete ALL entities and relations.'),
|
||||
workspace: z.string().optional()
|
||||
.describe('Workspace ID. Omit for personal mind.'),
|
||||
},
|
||||
async ({ mode, workspace }) => {
|
||||
const kg = workspace
|
||||
? getWorkspaceMind(workspace)?.knowledgeGraph ?? null
|
||||
: getKnowledgeGraph();
|
||||
const db = workspace
|
||||
? getWorkspaceMind(workspace)?.db ?? null
|
||||
: getPersonalDb();
|
||||
|
||||
if (!kg || !db) {
|
||||
return {
|
||||
content: [{ type: 'text' as const, text: `Workspace "${workspace}" not found.` }],
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
|
||||
const raw = db.getDatabase();
|
||||
|
||||
if (mode === 'audit') {
|
||||
// Count noise entities
|
||||
const allEntities = kg.getEntities(10000);
|
||||
const noiseEntities = allEntities.filter(e => isNoiseEntity(e.name, e.entity_type));
|
||||
|
||||
// Find duplicates
|
||||
const normalizedGroups = new Map<string, typeof allEntities>();
|
||||
for (const entity of allEntities) {
|
||||
const key = `${normalizeEntityName(entity.name)}::${entity.entity_type.toLowerCase()}`;
|
||||
let group = normalizedGroups.get(key);
|
||||
if (!group) {
|
||||
group = [];
|
||||
normalizedGroups.set(key, group);
|
||||
}
|
||||
group.push(entity);
|
||||
}
|
||||
const duplicateGroups = Array.from(normalizedGroups.values()).filter(g => g.length > 1);
|
||||
|
||||
// Count orphans (entities with no relations)
|
||||
const orphanCount = raw.prepare(`
|
||||
SELECT COUNT(*) as cnt FROM knowledge_entities e
|
||||
WHERE e.valid_to IS NULL
|
||||
AND e.id NOT IN (SELECT source_id FROM knowledge_relations WHERE valid_to IS NULL)
|
||||
AND e.id NOT IN (SELECT target_id FROM knowledge_relations WHERE valid_to IS NULL)
|
||||
`).get() as { cnt: number };
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
action: 'audit',
|
||||
total_entities: allEntities.length,
|
||||
noise_entities: noiseEntities.length,
|
||||
noise_sample: noiseEntities.slice(0, 20).map(e => ({ id: e.id, name: e.name, type: e.entity_type })),
|
||||
duplicate_groups: duplicateGroups.length,
|
||||
duplicate_sample: duplicateGroups.slice(0, 10).map(g => g.map(e => ({ id: e.id, name: e.name, type: e.entity_type }))),
|
||||
orphan_entities: orphanCount.cnt,
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
if (mode === 'remove_noise') {
|
||||
const allEntities = kg.getEntities(10000);
|
||||
const noiseEntities = allEntities.filter(e => isNoiseEntity(e.name, e.entity_type));
|
||||
|
||||
let removed = 0;
|
||||
const removeTx = raw.transaction(() => {
|
||||
for (const entity of noiseEntities) {
|
||||
// Retire relations first
|
||||
const rels = [
|
||||
...kg.getRelationsFrom(entity.id),
|
||||
...kg.getRelationsTo(entity.id),
|
||||
];
|
||||
for (const rel of rels) {
|
||||
kg.retireRelation(rel.id);
|
||||
}
|
||||
kg.retireEntity(entity.id);
|
||||
removed++;
|
||||
}
|
||||
});
|
||||
removeTx();
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
action: 'remove_noise',
|
||||
entities_retired: removed,
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
if (mode === 'dedup') {
|
||||
const allEntities = kg.getEntities(10000);
|
||||
const normalizedGroups = new Map<string, typeof allEntities>();
|
||||
for (const entity of allEntities) {
|
||||
const key = `${normalizeEntityName(entity.name)}::${entity.entity_type.toLowerCase()}`;
|
||||
let group = normalizedGroups.get(key);
|
||||
if (!group) {
|
||||
group = [];
|
||||
normalizedGroups.set(key, group);
|
||||
}
|
||||
group.push(entity);
|
||||
}
|
||||
|
||||
let merged = 0;
|
||||
const dedupTx = raw.transaction(() => {
|
||||
for (const group of normalizedGroups.values()) {
|
||||
if (group.length <= 1) continue;
|
||||
|
||||
// Keep the entity with the most relations (or the oldest)
|
||||
const sorted = group.sort((a, b) => {
|
||||
const aRels = kg.getRelationsFrom(a.id).length + kg.getRelationsTo(a.id).length;
|
||||
const bRels = kg.getRelationsFrom(b.id).length + kg.getRelationsTo(b.id).length;
|
||||
return bRels - aRels;
|
||||
});
|
||||
const keep = sorted[0];
|
||||
const retire = sorted.slice(1);
|
||||
|
||||
for (const dup of retire) {
|
||||
// Re-point relations from dup to keep
|
||||
for (const rel of kg.getRelationsFrom(dup.id)) {
|
||||
try {
|
||||
kg.createRelation(keep.id, rel.target_id, rel.relation_type, rel.confidence);
|
||||
} catch { /* may already exist */ }
|
||||
kg.retireRelation(rel.id);
|
||||
}
|
||||
for (const rel of kg.getRelationsTo(dup.id)) {
|
||||
try {
|
||||
kg.createRelation(rel.source_id, keep.id, rel.relation_type, rel.confidence);
|
||||
} catch { /* may already exist */ }
|
||||
kg.retireRelation(rel.id);
|
||||
}
|
||||
|
||||
// Merge properties
|
||||
try {
|
||||
const keepProps = JSON.parse(keep.properties || '{}');
|
||||
const dupProps = JSON.parse(dup.properties || '{}');
|
||||
const mergedProps = { ...dupProps, ...keepProps };
|
||||
kg.updateEntity(keep.id, { properties: mergedProps });
|
||||
} catch { /* ok */ }
|
||||
|
||||
kg.retireEntity(dup.id);
|
||||
merged++;
|
||||
}
|
||||
}
|
||||
});
|
||||
dedupTx();
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
action: 'dedup',
|
||||
entities_merged: merged,
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
if (mode === 'retire_orphans') {
|
||||
const orphans = raw.prepare(`
|
||||
SELECT id FROM knowledge_entities e
|
||||
WHERE e.valid_to IS NULL
|
||||
AND e.id NOT IN (SELECT source_id FROM knowledge_relations WHERE valid_to IS NULL)
|
||||
AND e.id NOT IN (SELECT target_id FROM knowledge_relations WHERE valid_to IS NULL)
|
||||
`).all() as { id: number }[];
|
||||
|
||||
let retired = 0;
|
||||
const retireTx = raw.transaction(() => {
|
||||
for (const { id } of orphans) {
|
||||
kg.retireEntity(id);
|
||||
retired++;
|
||||
}
|
||||
});
|
||||
retireTx();
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
action: 'retire_orphans',
|
||||
entities_retired: retired,
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
if (mode === 'wipe_all') {
|
||||
const entityCount = kg.getEntityCount();
|
||||
|
||||
const wipeTx = raw.transaction(() => {
|
||||
raw.prepare("UPDATE knowledge_relations SET valid_to = datetime('now') WHERE valid_to IS NULL").run();
|
||||
raw.prepare("UPDATE knowledge_entities SET valid_to = datetime('now') WHERE valid_to IS NULL").run();
|
||||
});
|
||||
wipeTx();
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
action: 'wipe_all',
|
||||
entities_retired: entityCount,
|
||||
warning: 'ALL entities and relations soft-deleted. This cannot be undone.',
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{ type: 'text' as const, text: `Unknown mode: ${mode}` }],
|
||||
isError: true,
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
281
packages/hive-mind-mcp-server/src/tools/harvest.ts
Normal file
281
packages/hive-mind-mcp-server/src/tools/harvest.ts
Normal file
@@ -0,0 +1,281 @@
|
||||
/**
|
||||
* Harvest tools — harvest_import + harvest_sources.
|
||||
* Import conversations from ChatGPT, Claude, Gemini, and other AI systems.
|
||||
*/
|
||||
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { z } from 'zod';
|
||||
import fs from 'node:fs';
|
||||
import { resolveRelativeDate, HARVEST_FRAME_CONTENT_CAP, writeRawTurnFrames, RawArchive, SuppressionStore, readArchiveUids, withArchiveUid } from '@waggle/hive-mind-core';
|
||||
import {
|
||||
getFrameStore,
|
||||
getSessions,
|
||||
getSearch,
|
||||
getKnowledgeGraph,
|
||||
getHarvestSourceStore,
|
||||
getPersonalDb,
|
||||
getAdapter,
|
||||
} from '../core/setup.js';
|
||||
|
||||
export function registerHarvestTools(server: McpServer): void {
|
||||
|
||||
// ── harvest_import ──────────────────────────────────────────────
|
||||
server.tool(
|
||||
'harvest_import',
|
||||
'Import conversation history from external AI systems (ChatGPT, Claude, Gemini, etc.). Parses the export data and saves extracted memories to the personal mind.',
|
||||
{
|
||||
source: z.enum([
|
||||
'chatgpt', 'claude', 'claude-code', 'gemini', 'universal',
|
||||
]).describe('Source AI system'),
|
||||
data: z.string().optional()
|
||||
.describe('JSON string of the export data. Provide this OR file_path, not both'),
|
||||
file_path: z.string().optional()
|
||||
.describe('Path to the export file on disk. Provide this OR data, not both'),
|
||||
},
|
||||
async ({ source, data, file_path }) => {
|
||||
// Validate: one of data or file_path must be provided
|
||||
if (!data && !file_path) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: 'Error: provide either "data" (JSON string) or "file_path" (path to export file)',
|
||||
}],
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
|
||||
// Parse input
|
||||
let parsed: unknown;
|
||||
try {
|
||||
if (file_path) {
|
||||
const raw = fs.readFileSync(file_path, 'utf-8');
|
||||
parsed = JSON.parse(raw);
|
||||
} else {
|
||||
parsed = JSON.parse(data!);
|
||||
}
|
||||
} catch (err) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: `Error parsing input: ${err instanceof Error ? err.message : 'invalid JSON'}`,
|
||||
}],
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
|
||||
// Get the appropriate adapter
|
||||
const adapter = getAdapter(source);
|
||||
const items = adapter.parse(parsed);
|
||||
|
||||
if (items.length === 0) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: `No conversations found in ${source} export data.`,
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
// Save each item as an I-Frame in the personal mind
|
||||
const frameStore = getFrameStore();
|
||||
const sessions = getSessions();
|
||||
const search = getSearch();
|
||||
const kg = getKnowledgeGraph();
|
||||
const harvestStore = getHarvestSourceStore();
|
||||
|
||||
// Ensure a persistent harvest session
|
||||
const session = sessions.ensure(
|
||||
`harvest:${source}`,
|
||||
undefined,
|
||||
`Harvest import from ${source}`,
|
||||
);
|
||||
|
||||
let framesCreated = 0;
|
||||
let duplicatesSkipped = 0;
|
||||
let entitiesCreated = 0;
|
||||
let rawTurnsWritten = 0;
|
||||
|
||||
// Record max frame id before the batch. createIFrame dedups by content,
|
||||
// so a "not new" frame returns an older id. id-based detection is
|
||||
// format-agnostic; comparing timestamps here would trip on the mismatch
|
||||
// between JS's ISO format and SQLite's space-separated datetime('now').
|
||||
const rawDb = getPersonalDb().getDatabase();
|
||||
const maxBefore =
|
||||
(rawDb.prepare('SELECT COALESCE(MAX(id), 0) AS m FROM memory_frames').get() as { m: number }).m;
|
||||
|
||||
// #7: verbatim provenance archive — full immutable source per item, linked
|
||||
// from the summary frame via metadata.archiveUid. Append-only; idempotent.
|
||||
const rawArchive = new RawArchive(getPersonalDb());
|
||||
// #7 sticky erasure: skip re-importing an Art.17-erased subject. One `continue`
|
||||
// short-circuits the whole per-item fan-out (archive + summary + raw-turns + KG).
|
||||
const suppression = new SuppressionStore(getPersonalDb());
|
||||
let suppressedSkipped = 0;
|
||||
|
||||
for (const item of items) {
|
||||
if (suppression.isSuppressed(item.source, item.id)) { suppressedSkipped++; continue; }
|
||||
// Build a summary from the conversation
|
||||
const content = item.title
|
||||
? `[${item.source}] ${item.title}: ${item.content.slice(0, HARVEST_FRAME_CONTENT_CAP)}`
|
||||
: `[${item.source}] ${item.content.slice(0, HARVEST_FRAME_CONTENT_CAP)}`;
|
||||
|
||||
// Write-time temporal anchoring. The frame's created_at should reflect WHEN the
|
||||
// event happened, not the ingest wall-clock. Start from the source timestamp; if
|
||||
// the content carries a relative cue ("yesterday", "last week"), resolve it
|
||||
// against that source date to the true event date. Benchmark-validated: this is
|
||||
// the production counterpart of the LoCoMo Phase-4 win (temporal parity vs Memori
|
||||
// — see benchmarks/results/memori-phase22-RESULT.md). createIFrame validates the
|
||||
// ISO string and falls back to datetime('now') if it's unusable.
|
||||
const resolved = resolveRelativeDate(content, item.timestamp);
|
||||
const createdAt = resolved ? `${resolved.iso}T00:00:00Z` : (item.timestamp || undefined);
|
||||
|
||||
// #7: archive the FULL untruncated verbatim source BEFORE the frame's
|
||||
// capped preview is built. Best-effort — a failure must not abort the
|
||||
// item (degraded provenance beats a lost import); never silent.
|
||||
let archiveUid: string | undefined;
|
||||
try {
|
||||
archiveUid = rawArchive.append({
|
||||
source: item.source,
|
||||
sourceRef: item.id,
|
||||
title: item.title,
|
||||
content: item.content,
|
||||
sourceTimestamp: item.timestamp,
|
||||
}).archiveUid;
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[harvest] raw_archive append failed for ${item.source}/${item.id} — frame persists without provenance link:`,
|
||||
err instanceof Error ? err.message : 'unknown',
|
||||
);
|
||||
}
|
||||
|
||||
// createIFrame handles dedup internally — returns existing frame if content matches
|
||||
const frame = frameStore.createIFrame(
|
||||
session.gop_id,
|
||||
content,
|
||||
'normal',
|
||||
'import',
|
||||
createdAt,
|
||||
);
|
||||
|
||||
// #7: stamp provenance metadata. On a fresh frame (default '{}' metadata)
|
||||
// record sourceId + the archive link; on an already-stamped/dedup'd frame,
|
||||
// accumulate the archiveUid into the canonical archiveUids[] without clobbering
|
||||
// existing metadata.
|
||||
// Multi-source accumulation (resolved): two DIFFERENT sources with byte-identical
|
||||
// content dedup to ONE frame, and that frame now links to EVERY source's archive
|
||||
// row via metadata.archiveUids[] (withArchiveUid migrates any legacy scalar and
|
||||
// set-unions). reconstructSource resolves them all; no frame→source link is lost.
|
||||
// (Server harvest route shares this.)
|
||||
if (!frame.metadata || frame.metadata === '{}') {
|
||||
frameStore.setMetadata(frame.id, JSON.stringify({
|
||||
sourceId: item.id,
|
||||
...(archiveUid ? { archiveUids: [archiveUid] } : {}),
|
||||
}));
|
||||
} else if (archiveUid) {
|
||||
try {
|
||||
const meta = JSON.parse(frame.metadata) as Record<string, unknown>;
|
||||
// Only write when the uid set actually grows (avoids needless setMetadata
|
||||
// churn on re-imports). withArchiveUid migrates any legacy scalar → array.
|
||||
if (!readArchiveUids(meta).includes(archiveUid)) {
|
||||
frameStore.setMetadata(frame.id, JSON.stringify(withArchiveUid(meta, archiveUid)));
|
||||
}
|
||||
} catch { /* malformed metadata — leave as-is */ }
|
||||
}
|
||||
|
||||
// Frames created during this batch have id > maxBefore.
|
||||
// Dedup hits return the original frame whose id is older.
|
||||
const isNew = frame.id > maxBefore;
|
||||
|
||||
if (isNew) {
|
||||
framesCreated++;
|
||||
|
||||
// Index for semantic search (non-fatal)
|
||||
try {
|
||||
await search.indexFrame(frame.id, content);
|
||||
} catch { /* vector indexing failure is non-fatal */ }
|
||||
|
||||
// Extract basic entities from metadata if present. Route through
|
||||
// importEntitiesForFrame so each entity is LINKED to its frame — the
|
||||
// provenance anchor GDPR Art.17 erasure's orphan sweep needs (an
|
||||
// unlinked entity name, often PII, would otherwise survive erasure).
|
||||
if (item.metadata?.entities && Array.isArray(item.metadata.entities)) {
|
||||
entitiesCreated += kg.importEntitiesForFrame(
|
||||
frame.id,
|
||||
item.metadata.entities as { name: string; type: string }[],
|
||||
{ source: item.source, importedFrom: item.title },
|
||||
);
|
||||
}
|
||||
} else {
|
||||
duplicatesSkipped++;
|
||||
}
|
||||
|
||||
// W4.6: per-turn verbatim dialogue storage — source material for the
|
||||
// RAWDETAIL recall lane. Items without messages are a no-op; dedup
|
||||
// inside makes re-imports idempotent. Kill switch: WAGGLE_RAWDETAIL=0.
|
||||
if (process.env.WAGGLE_RAWDETAIL !== '0') {
|
||||
rawTurnsWritten += writeRawTurnFrames(frameStore, session.gop_id, item).written;
|
||||
}
|
||||
}
|
||||
|
||||
// Record the sync in harvest source store
|
||||
harvestStore.upsert(
|
||||
source as Parameters<typeof harvestStore.upsert>[0],
|
||||
adapter.displayName,
|
||||
file_path ?? undefined,
|
||||
);
|
||||
harvestStore.recordSync(
|
||||
source as Parameters<typeof harvestStore.recordSync>[0],
|
||||
items.length,
|
||||
framesCreated,
|
||||
);
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
source,
|
||||
items_found: items.length,
|
||||
frames_created: framesCreated,
|
||||
duplicates_skipped: duplicatesSkipped,
|
||||
suppressed_skipped: suppressedSkipped,
|
||||
entities_created: entitiesCreated,
|
||||
raw_turns_written: rawTurnsWritten,
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// ── harvest_sources ─────────────────────────────────────────────
|
||||
server.tool(
|
||||
'harvest_sources',
|
||||
'List all registered harvest sources and their sync status.',
|
||||
{},
|
||||
async () => {
|
||||
const store = getHarvestSourceStore();
|
||||
const sources = store.getAll();
|
||||
|
||||
if (sources.length === 0) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: 'No harvest sources registered yet. Use harvest_import to import conversation data.',
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify(sources.map(s => ({
|
||||
source: s.source,
|
||||
display_name: s.displayName,
|
||||
last_synced: s.lastSyncedAt,
|
||||
items_imported: s.itemsImported,
|
||||
frames_created: s.framesCreated,
|
||||
auto_sync: s.autoSync,
|
||||
})), null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
122
packages/hive-mind-mcp-server/src/tools/identity.ts
Normal file
122
packages/hive-mind-mcp-server/src/tools/identity.ts
Normal file
@@ -0,0 +1,122 @@
|
||||
/**
|
||||
* Identity tools — get_identity + set_identity.
|
||||
* Wraps IdentityLayer from @waggle/hive-mind-core.
|
||||
*/
|
||||
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { z } from 'zod';
|
||||
import { getIdentity } from '../core/setup.js';
|
||||
|
||||
export function registerIdentityTools(server: McpServer): void {
|
||||
|
||||
// ── get_identity ────────────────────────────────────────────────
|
||||
server.tool(
|
||||
'get_identity',
|
||||
'Get the user\'s identity profile — name, role, department, personality, capabilities. Returns the persistent identity stored in the personal mind.',
|
||||
{},
|
||||
async () => {
|
||||
const identity = getIdentity();
|
||||
|
||||
if (!identity.exists()) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
configured: false,
|
||||
message: 'No identity configured yet. Use set_identity to create one.',
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
const id = identity.get();
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
configured: true,
|
||||
name: id.name,
|
||||
role: id.role,
|
||||
department: id.department,
|
||||
personality: id.personality,
|
||||
capabilities: id.capabilities,
|
||||
system_prompt: id.system_prompt,
|
||||
updated_at: id.updated_at,
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// ── set_identity ────────────────────────────────────────────────
|
||||
server.tool(
|
||||
'set_identity',
|
||||
'Create or update the user\'s identity profile. Only provided fields are updated; omitted fields are preserved.',
|
||||
{
|
||||
name: z.string().optional().describe('User\'s name'),
|
||||
role: z.string().optional().describe('Professional role (e.g., "Senior Engineer", "Product Manager")'),
|
||||
department: z.string().optional().describe('Department or team'),
|
||||
personality: z.string().optional().describe('Communication style preferences'),
|
||||
capabilities: z.string().optional().describe('Technical capabilities and expertise areas'),
|
||||
system_prompt: z.string().optional().describe('Custom system prompt additions'),
|
||||
},
|
||||
async ({ name, role, department, personality, capabilities, system_prompt }) => {
|
||||
const identity = getIdentity();
|
||||
|
||||
if (!identity.exists()) {
|
||||
// Create new identity — require at least a name
|
||||
const id = identity.create({
|
||||
name: name ?? 'User',
|
||||
role: role ?? '',
|
||||
department: department ?? '',
|
||||
personality: personality ?? '',
|
||||
capabilities: capabilities ?? '',
|
||||
system_prompt: system_prompt ?? '',
|
||||
});
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
action: 'created',
|
||||
name: id.name,
|
||||
role: id.role,
|
||||
department: id.department,
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
// Update existing identity — only set provided fields
|
||||
const updates: Record<string, string> = {};
|
||||
if (name !== undefined) updates.name = name;
|
||||
if (role !== undefined) updates.role = role;
|
||||
if (department !== undefined) updates.department = department;
|
||||
if (personality !== undefined) updates.personality = personality;
|
||||
if (capabilities !== undefined) updates.capabilities = capabilities;
|
||||
if (system_prompt !== undefined) updates.system_prompt = system_prompt;
|
||||
|
||||
if (Object.keys(updates).length === 0) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: 'No fields provided to update.',
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
const id = identity.update(updates);
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
action: 'updated',
|
||||
name: id.name,
|
||||
role: id.role,
|
||||
department: id.department,
|
||||
updated_at: id.updated_at,
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
207
packages/hive-mind-mcp-server/src/tools/ingest.ts
Normal file
207
packages/hive-mind-mcp-server/src/tools/ingest.ts
Normal file
@@ -0,0 +1,207 @@
|
||||
/**
|
||||
* Ingest tools — import documents, URLs, and files into the memory system.
|
||||
*
|
||||
* ingest_source: Universal ingestion tool that auto-detects content type
|
||||
* and routes through the appropriate adapter.
|
||||
*/
|
||||
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { z } from 'zod';
|
||||
import fs from 'node:fs';
|
||||
import {
|
||||
getFrameStore,
|
||||
getSessions,
|
||||
getSearch,
|
||||
getKnowledgeGraph,
|
||||
getHarvestSourceStore,
|
||||
getPersonalDb,
|
||||
getAdapter,
|
||||
} from '../core/setup.js';
|
||||
import { UrlAdapter } from '@waggle/hive-mind-core';
|
||||
import type { PdfAdapter } from '@waggle/hive-mind-core';
|
||||
import type { UniversalImportItem } from '@waggle/hive-mind-core';
|
||||
|
||||
export function registerIngestTools(server: McpServer): void {
|
||||
|
||||
// ── ingest_source ──────────────────────────────────────────────
|
||||
server.tool(
|
||||
'ingest_source',
|
||||
'Ingest a document, URL, or text into the memory system. Auto-detects content type or use type_hint. Supports: markdown files, plain text, PDF files, web URLs, and raw text content.',
|
||||
{
|
||||
content: z.string()
|
||||
.describe('Content to ingest: a file path, URL, or raw text/markdown content'),
|
||||
type_hint: z.enum(['markdown', 'plaintext', 'pdf', 'url', 'auto']).default('auto')
|
||||
.describe('Content type hint. "auto" detects from content (default)'),
|
||||
importance: z.enum(['critical', 'important', 'normal']).default('normal')
|
||||
.describe('Importance level for stored frames'),
|
||||
tags: z.array(z.string()).optional()
|
||||
.describe('Optional tags to attach as metadata'),
|
||||
workspace: z.string().optional()
|
||||
.describe('Workspace ID. Omit for personal mind'),
|
||||
},
|
||||
async ({ content, type_hint, importance, tags }) => {
|
||||
// Detect content type
|
||||
const detectedType = type_hint === 'auto'
|
||||
? detectContentType(content)
|
||||
: type_hint;
|
||||
|
||||
let items: UniversalImportItem[];
|
||||
|
||||
try {
|
||||
if (detectedType === 'url') {
|
||||
// URL requires async fetch
|
||||
const urlAdapter = new UrlAdapter();
|
||||
items = await urlAdapter.fetchAndParse(content);
|
||||
} else if (detectedType === 'pdf') {
|
||||
// PDF requires async parse
|
||||
const { PdfAdapter: PdfAdapterClass } = await import('@waggle/hive-mind-core');
|
||||
const pdfAdapter = new PdfAdapterClass() as PdfAdapter;
|
||||
items = await pdfAdapter.parseFile(content);
|
||||
} else {
|
||||
// Markdown, plaintext, or raw text — synchronous
|
||||
const adapter = getAdapter(detectedType);
|
||||
items = adapter.parse(content);
|
||||
}
|
||||
} catch (err) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: `Error processing ${detectedType} content: ${err instanceof Error ? err.message : String(err)}`,
|
||||
}],
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (items.length === 0) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: `No content extracted from ${detectedType} input.`,
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
// Store items as frames
|
||||
const frameStore = getFrameStore();
|
||||
const sessions = getSessions();
|
||||
const search = getSearch();
|
||||
const kg = getKnowledgeGraph();
|
||||
const harvestStore = getHarvestSourceStore();
|
||||
|
||||
const sessionId = `ingest:${detectedType}:${new Date().toISOString().slice(0, 10)}`;
|
||||
sessions.ensure(sessionId, undefined, `Ingested ${detectedType} content`);
|
||||
|
||||
let framesCreated = 0;
|
||||
let duplicatesSkipped = 0;
|
||||
let entitiesCreated = 0;
|
||||
|
||||
// Record max frame id before the batch — see harvest.ts for rationale
|
||||
// (id-based dedup detection avoids the ISO-vs-space timestamp format
|
||||
// mismatch between JS Dates and SQLite datetime('now')).
|
||||
const rawDb = getPersonalDb().getDatabase();
|
||||
const maxBefore =
|
||||
(rawDb.prepare('SELECT COALESCE(MAX(id), 0) AS m FROM memory_frames').get() as { m: number }).m;
|
||||
|
||||
for (const item of items) {
|
||||
const frameContent = item.title
|
||||
? `[${detectedType}] ${item.title}: ${item.content.slice(0, 3000)}`
|
||||
: `[${detectedType}] ${item.content.slice(0, 3000)}`;
|
||||
|
||||
const frame = frameStore.createIFrame(
|
||||
sessionId,
|
||||
frameContent,
|
||||
importance,
|
||||
'import',
|
||||
);
|
||||
|
||||
const isNew = frame.id > maxBefore;
|
||||
|
||||
if (isNew) {
|
||||
framesCreated++;
|
||||
|
||||
// Index for semantic search
|
||||
try {
|
||||
await search.indexFrame(frame.id, frameContent);
|
||||
} catch { /* non-fatal */ }
|
||||
|
||||
// Extract entities from metadata
|
||||
const metaEntities = item.metadata?.entities;
|
||||
if (Array.isArray(metaEntities)) {
|
||||
for (const ent of metaEntities as { name: string; type: string }[]) {
|
||||
try {
|
||||
kg.createEntity(ent.type || 'concept', ent.name, {
|
||||
source: detectedType,
|
||||
...(tags && { tags }),
|
||||
});
|
||||
entitiesCreated++;
|
||||
} catch { /* non-fatal */ }
|
||||
}
|
||||
}
|
||||
} else {
|
||||
duplicatesSkipped++;
|
||||
}
|
||||
}
|
||||
|
||||
// Record in harvest source store
|
||||
const sourceKey = detectedType === 'url' ? 'unknown' : detectedType;
|
||||
harvestStore.upsert(
|
||||
sourceKey as Parameters<typeof harvestStore.upsert>[0],
|
||||
items[0]?.title ?? detectedType,
|
||||
content.startsWith('http') ? content : undefined,
|
||||
);
|
||||
harvestStore.recordSync(
|
||||
sourceKey as Parameters<typeof harvestStore.recordSync>[0],
|
||||
items.length,
|
||||
framesCreated,
|
||||
);
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
source_type: detectedType,
|
||||
items_parsed: items.length,
|
||||
frames_created: framesCreated,
|
||||
duplicates_skipped: duplicatesSkipped,
|
||||
entities_created: entitiesCreated,
|
||||
...(tags && { tags }),
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/** Detect content type from the input string. */
|
||||
function detectContentType(input: string): 'markdown' | 'plaintext' | 'pdf' | 'url' {
|
||||
const trimmed = input.trim();
|
||||
|
||||
// URL detection
|
||||
if (trimmed.startsWith('http://') || trimmed.startsWith('https://')) {
|
||||
return 'url';
|
||||
}
|
||||
|
||||
// File path detection
|
||||
if (trimmed.length < 500 && !trimmed.includes('\n')) {
|
||||
const lower = trimmed.toLowerCase();
|
||||
if (lower.endsWith('.pdf')) return 'pdf';
|
||||
if (lower.endsWith('.md') || lower.endsWith('.markdown')) return 'markdown';
|
||||
if (lower.endsWith('.txt')) return 'plaintext';
|
||||
|
||||
// Check if it's an existing file
|
||||
try {
|
||||
if (fs.existsSync(trimmed)) {
|
||||
if (lower.endsWith('.pdf')) return 'pdf';
|
||||
if (lower.endsWith('.md')) return 'markdown';
|
||||
return 'plaintext';
|
||||
}
|
||||
} catch { /* not a path */ }
|
||||
}
|
||||
|
||||
// Content-based detection
|
||||
if (trimmed.startsWith('#') || trimmed.includes('\n## ') || trimmed.includes('\n### ')) {
|
||||
return 'markdown';
|
||||
}
|
||||
|
||||
return 'plaintext';
|
||||
}
|
||||
185
packages/hive-mind-mcp-server/src/tools/knowledge.ts
Normal file
185
packages/hive-mind-mcp-server/src/tools/knowledge.ts
Normal file
@@ -0,0 +1,185 @@
|
||||
/**
|
||||
* Knowledge graph tools — search_entities + save_entity + create_relation.
|
||||
* Wraps KnowledgeGraph from @waggle/hive-mind-core.
|
||||
*/
|
||||
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { z } from 'zod';
|
||||
import { getKnowledgeGraph, getWorkspaceMind } from '../core/setup.js';
|
||||
|
||||
const ENTITY_TYPES = [
|
||||
'person', 'project', 'concept', 'organization',
|
||||
'technology', 'tool', 'location', 'event',
|
||||
] as const;
|
||||
|
||||
export function registerKnowledgeTools(server: McpServer): void {
|
||||
|
||||
// ── search_entities ─────────────────────────────────────────────
|
||||
server.tool(
|
||||
'search_entities',
|
||||
'Search the knowledge graph for entities (people, projects, concepts, organizations, technologies). Returns matching entities with their relations.',
|
||||
{
|
||||
query: z.string().describe('Search query — matches entity names'),
|
||||
type: z.enum(ENTITY_TYPES).optional()
|
||||
.describe('Filter by entity type'),
|
||||
limit: z.number().min(1).max(200).optional()
|
||||
.describe('Maximum results. Defaults to 20'),
|
||||
workspace: z.string().optional()
|
||||
.describe('Workspace ID. Omit for personal knowledge graph'),
|
||||
},
|
||||
async ({ query, type, limit, workspace }) => {
|
||||
const maxResults = limit ?? 20;
|
||||
const target = workspace ? getWorkspaceMind(workspace) : null;
|
||||
const kg = target?.knowledgeGraph ?? getKnowledgeGraph();
|
||||
|
||||
// If type filter provided, search within that type
|
||||
let entities;
|
||||
if (type) {
|
||||
const typed = kg.getEntitiesByType(type, maxResults * 2);
|
||||
entities = typed.filter(e =>
|
||||
e.name.toLowerCase().includes(query.toLowerCase())
|
||||
).slice(0, maxResults);
|
||||
} else {
|
||||
entities = kg.searchEntities(query, maxResults);
|
||||
}
|
||||
|
||||
// Enrich with relations for each entity
|
||||
const enriched = entities.map(e => {
|
||||
const relationsFrom = kg.getRelationsFrom(e.id);
|
||||
const relationsTo = kg.getRelationsTo(e.id);
|
||||
return {
|
||||
id: e.id,
|
||||
type: e.entity_type,
|
||||
name: e.name,
|
||||
properties: safeParseJson(e.properties),
|
||||
valid_from: e.valid_from,
|
||||
valid_to: e.valid_to,
|
||||
relations: {
|
||||
outgoing: relationsFrom.map(r => ({
|
||||
target_id: r.target_id,
|
||||
type: r.relation_type,
|
||||
confidence: r.confidence,
|
||||
})),
|
||||
incoming: relationsTo.map(r => ({
|
||||
source_id: r.source_id,
|
||||
type: r.relation_type,
|
||||
confidence: r.confidence,
|
||||
})),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
if (enriched.length === 0) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: `No entities found matching "${query}"`,
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify(enriched, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// ── save_entity ─────────────────────────────────────────────────
|
||||
server.tool(
|
||||
'save_entity',
|
||||
'Create or update an entity in the knowledge graph. Auto-deduplicates by normalized name.',
|
||||
{
|
||||
type: z.enum(ENTITY_TYPES).describe('Entity type'),
|
||||
name: z.string().describe('Entity name (e.g., "John Smith", "Project Alpha")'),
|
||||
properties: z.record(z.string(), z.unknown()).optional()
|
||||
.describe('Key-value properties for the entity'),
|
||||
workspace: z.string().optional()
|
||||
.describe('Workspace ID. Omit for personal knowledge graph'),
|
||||
},
|
||||
async ({ type, name, properties, workspace }) => {
|
||||
const target = workspace ? getWorkspaceMind(workspace) : null;
|
||||
const kg = target?.knowledgeGraph ?? getKnowledgeGraph();
|
||||
const props = properties ?? {};
|
||||
|
||||
// Check for existing entity with same name and type (dedup)
|
||||
const existing = kg.searchEntities(name, 10)
|
||||
.find(e =>
|
||||
e.entity_type === type &&
|
||||
e.name.toLowerCase() === name.toLowerCase()
|
||||
);
|
||||
|
||||
let entity;
|
||||
if (existing) {
|
||||
// Merge properties into existing entity
|
||||
const existingProps = safeParseJson(existing.properties);
|
||||
const merged = { ...existingProps, ...props };
|
||||
entity = kg.updateEntity(existing.id, { properties: merged });
|
||||
} else {
|
||||
entity = kg.createEntity(type, name, props);
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
id: entity.id,
|
||||
type: entity.entity_type,
|
||||
name: entity.name,
|
||||
properties: safeParseJson(entity.properties),
|
||||
created: !existing,
|
||||
updated: !!existing,
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// ── create_relation ──────────────────────────────────────────────
|
||||
server.tool(
|
||||
'create_relation',
|
||||
'Create a relationship between two entities in the knowledge graph.',
|
||||
{
|
||||
source_id: z.number().describe('Source entity ID'),
|
||||
target_id: z.number().describe('Target entity ID'),
|
||||
relation_type: z.string().describe('Relationship type (e.g., "works_on", "knows", "uses")'),
|
||||
confidence: z.number().min(0).max(1).optional()
|
||||
.describe('Confidence score 0-1. Defaults to 1.0'),
|
||||
properties: z.record(z.string(), z.unknown()).optional()
|
||||
.describe('Additional relation properties'),
|
||||
workspace: z.string().optional()
|
||||
.describe('Workspace ID. Omit for personal knowledge graph'),
|
||||
},
|
||||
async ({ source_id, target_id, relation_type, confidence, properties, workspace }) => {
|
||||
const target = workspace ? getWorkspaceMind(workspace) : null;
|
||||
const kg = target?.knowledgeGraph ?? getKnowledgeGraph();
|
||||
|
||||
const relation = kg.createRelation(
|
||||
source_id,
|
||||
target_id,
|
||||
relation_type,
|
||||
confidence ?? 1.0,
|
||||
properties ?? {},
|
||||
);
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
id: relation.id,
|
||||
source_id: relation.source_id,
|
||||
target_id: relation.target_id,
|
||||
type: relation.relation_type,
|
||||
confidence: relation.confidence,
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function safeParseJson(raw: string): Record<string, unknown> {
|
||||
try { return JSON.parse(raw); } catch { return {}; }
|
||||
}
|
||||
189
packages/hive-mind-mcp-server/src/tools/memory.ts
Normal file
189
packages/hive-mind-mcp-server/src/tools/memory.ts
Normal file
@@ -0,0 +1,189 @@
|
||||
/**
|
||||
* Memory tools — save_memory + recall_memory.
|
||||
* The bread and butter: create I-Frames and hybrid-search recall.
|
||||
*/
|
||||
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { z } from 'zod';
|
||||
import {
|
||||
getFrameStore,
|
||||
getSearch,
|
||||
getSessions,
|
||||
getWorkspaceMind,
|
||||
getWorkspaceManager,
|
||||
} from '../core/setup.js';
|
||||
import type { Importance, FrameSource } from '@waggle/hive-mind-core';
|
||||
|
||||
export function registerMemoryTools(server: McpServer): void {
|
||||
|
||||
// ── save_memory ─────────────────────────────────────────────────
|
||||
server.tool(
|
||||
'save_memory',
|
||||
'Save a memory (fact, decision, preference, context) that persists across conversations. Auto-indexes for semantic search.',
|
||||
{
|
||||
content: z.string().describe('The memory content to save'),
|
||||
importance: z.enum(['critical', 'important', 'normal', 'temporary']).optional()
|
||||
.describe('Memory importance level. Defaults to "normal"'),
|
||||
source: z.enum(['user_stated', 'tool_verified', 'agent_inferred', 'system']).optional()
|
||||
.describe('How this memory was obtained. Defaults to "agent_inferred"'),
|
||||
workspace: z.string().optional()
|
||||
.describe('Workspace ID to save into. Omit for personal memory'),
|
||||
},
|
||||
async ({ content, importance, source, workspace }) => {
|
||||
const imp = (importance ?? 'normal') as Importance;
|
||||
const src = (source ?? 'agent_inferred') as FrameSource;
|
||||
|
||||
// Resolve target mind
|
||||
const target = workspace ? getWorkspaceMind(workspace) : null;
|
||||
const frameStore = target?.frameStore ?? getFrameStore();
|
||||
const sessions = target?.sessions ?? getSessions();
|
||||
const search = target?.search ?? getSearch();
|
||||
|
||||
// Group frames into daily sessions (mcp:YYYY-MM-DD)
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
const session = sessions.ensure(`mcp:${today}`, undefined, `MCP session ${today}`);
|
||||
|
||||
// Create the I-Frame (dedup is built into FrameStore)
|
||||
const frame = frameStore.createIFrame(session.gop_id, content, imp, src);
|
||||
|
||||
// Index in vector store for semantic search
|
||||
try {
|
||||
await search.indexFrame(frame.id, content);
|
||||
} catch {
|
||||
// Vector indexing failure is non-fatal — FTS still works
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
id: frame.id,
|
||||
content: frame.content,
|
||||
importance: frame.importance,
|
||||
source: frame.source,
|
||||
created_at: frame.created_at,
|
||||
workspace: workspace ?? 'personal',
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// ── recall_memory ───────────────────────────────────────────────
|
||||
server.tool(
|
||||
'recall_memory',
|
||||
'Search memories using semantic hybrid search (keyword + vector). Returns ranked results from personal and/or workspace memories.',
|
||||
{
|
||||
query: z.string().describe('Natural language search query'),
|
||||
limit: z.number().min(1).max(100).optional()
|
||||
.describe('Maximum results to return. Defaults to 10'),
|
||||
workspace: z.string().optional()
|
||||
.describe('Workspace ID to search. Omit to search personal memory'),
|
||||
scope: z.enum(['current', 'personal', 'all']).optional()
|
||||
.describe('"current" = active workspace only, "personal" = personal mind only, "all" = search everything. Defaults to "personal"'),
|
||||
profile: z.enum(['balanced', 'recent', 'important', 'connected']).optional()
|
||||
.describe('Scoring profile for ranking results. Defaults to "balanced"'),
|
||||
},
|
||||
async ({ query, limit, workspace, scope, profile }) => {
|
||||
const maxResults = limit ?? 10;
|
||||
const scoringProfile = profile ?? 'balanced';
|
||||
const searchScope = scope ?? 'personal';
|
||||
|
||||
interface ResultItem {
|
||||
id: number;
|
||||
content: string;
|
||||
importance: string;
|
||||
source: string;
|
||||
score: number;
|
||||
created_at: string;
|
||||
from: string;
|
||||
}
|
||||
const results: ResultItem[] = [];
|
||||
|
||||
const searchOpts = {
|
||||
limit: maxResults,
|
||||
profile: scoringProfile as 'balanced' | 'recent' | 'important' | 'connected',
|
||||
};
|
||||
|
||||
// Search personal mind
|
||||
if (searchScope === 'personal' || searchScope === 'all') {
|
||||
const search = getSearch();
|
||||
const personalResults = await search.search(query, searchOpts);
|
||||
for (const r of personalResults) {
|
||||
results.push({
|
||||
id: r.frame.id,
|
||||
content: r.frame.content,
|
||||
importance: r.frame.importance,
|
||||
source: r.frame.source,
|
||||
score: Math.round(r.finalScore * 1000) / 1000,
|
||||
created_at: r.frame.created_at,
|
||||
from: 'personal',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Search specific workspace
|
||||
if (searchScope === 'current' && workspace) {
|
||||
const wsMind = getWorkspaceMind(workspace);
|
||||
if (wsMind) {
|
||||
const wsResults = await wsMind.search.search(query, searchOpts);
|
||||
for (const r of wsResults) {
|
||||
results.push({
|
||||
id: r.frame.id,
|
||||
content: r.frame.content,
|
||||
importance: r.frame.importance,
|
||||
source: r.frame.source,
|
||||
score: Math.round(r.finalScore * 1000) / 1000,
|
||||
created_at: r.frame.created_at,
|
||||
from: `workspace:${workspace}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Search ALL workspaces when scope is 'all'
|
||||
if (searchScope === 'all') {
|
||||
const wm = getWorkspaceManager();
|
||||
const allWorkspaces = wm.list();
|
||||
for (const ws of allWorkspaces) {
|
||||
const wsMind = getWorkspaceMind(ws.id);
|
||||
if (!wsMind) continue;
|
||||
try {
|
||||
const wsResults = await wsMind.search.search(query, searchOpts);
|
||||
for (const r of wsResults) {
|
||||
results.push({
|
||||
id: r.frame.id,
|
||||
content: r.frame.content,
|
||||
importance: r.frame.importance,
|
||||
source: r.frame.source,
|
||||
score: Math.round(r.finalScore * 1000) / 1000,
|
||||
created_at: r.frame.created_at,
|
||||
from: `workspace:${ws.id}`,
|
||||
});
|
||||
}
|
||||
} catch { /* workspace search failure is non-fatal */ }
|
||||
}
|
||||
}
|
||||
|
||||
// Sort all results by score descending and trim
|
||||
results.sort((a, b) => b.score - a.score);
|
||||
const trimmed = results.slice(0, maxResults);
|
||||
|
||||
if (trimmed.length === 0) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: `No memories found for query: "${query}"`,
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify(trimmed, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
232
packages/hive-mind-mcp-server/src/tools/wiki.ts
Normal file
232
packages/hive-mind-mcp-server/src/tools/wiki.ts
Normal file
@@ -0,0 +1,232 @@
|
||||
/**
|
||||
* Wiki tools — compile, search, and browse the personal wiki.
|
||||
*
|
||||
* compile_wiki: Trigger incremental or full compilation
|
||||
* get_page: Read a compiled wiki page by slug
|
||||
* search_wiki: Search compiled pages
|
||||
* compile_health: Run health check on wiki data quality
|
||||
*/
|
||||
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { z } from 'zod';
|
||||
import {
|
||||
getPersonalDb,
|
||||
getFrameStore,
|
||||
getSearch,
|
||||
getKnowledgeGraph,
|
||||
} from '../core/setup.js';
|
||||
import {
|
||||
WikiCompiler,
|
||||
CompilationState,
|
||||
resolveSynthesizer as resolveWikiSynthesizer,
|
||||
} from '@waggle/hive-mind-wiki-compiler';
|
||||
import type { ResolvedSynthesizer } from '@waggle/hive-mind-wiki-compiler';
|
||||
|
||||
// Cached synthesizer — resolved once on first compile_wiki call
|
||||
let _synthesizer: ResolvedSynthesizer | null = null;
|
||||
|
||||
async function getSynthesizer(): Promise<ResolvedSynthesizer> {
|
||||
if (!_synthesizer) {
|
||||
_synthesizer = await resolveWikiSynthesizer();
|
||||
console.error(`[hive-mind-memory] Wiki synthesizer: ${_synthesizer.provider} (${_synthesizer.model})`);
|
||||
}
|
||||
return _synthesizer;
|
||||
}
|
||||
|
||||
async function getCompiler(): Promise<{ compiler: WikiCompiler; state: CompilationState; provider: string }> {
|
||||
const db = getPersonalDb();
|
||||
const state = new CompilationState(db);
|
||||
const synth = await getSynthesizer();
|
||||
const compiler = new WikiCompiler(
|
||||
getKnowledgeGraph(),
|
||||
getFrameStore(),
|
||||
getSearch(),
|
||||
state,
|
||||
{ synthesize: synth.synthesize },
|
||||
);
|
||||
return { compiler, state, provider: synth.provider };
|
||||
}
|
||||
|
||||
export function registerWikiTools(server: McpServer): void {
|
||||
|
||||
// ── compile_wiki ───────────────────────────────────────────────
|
||||
server.tool(
|
||||
'compile_wiki',
|
||||
'Compile the personal wiki from memory frames and knowledge graph. Uses incremental compilation by default (only processes new frames). Returns compilation statistics.',
|
||||
{
|
||||
mode: z.enum(['incremental', 'full']).default('incremental')
|
||||
.describe('incremental: only recompile affected pages. full: rebuild everything.'),
|
||||
concepts: z.array(z.string()).optional()
|
||||
.describe('Optional list of concept names to compile pages for. Auto-detected if omitted.'),
|
||||
},
|
||||
async ({ mode, concepts }) => {
|
||||
const { compiler, provider } = await getCompiler();
|
||||
|
||||
try {
|
||||
const result = await compiler.compile({
|
||||
incremental: mode === 'incremental',
|
||||
concepts: concepts ?? undefined,
|
||||
});
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
mode,
|
||||
llm_provider: provider,
|
||||
pages_created: result.pagesCreated,
|
||||
pages_updated: result.pagesUpdated,
|
||||
pages_unchanged: result.pagesUnchanged,
|
||||
entity_pages: result.entityPages,
|
||||
concept_pages: result.conceptPages,
|
||||
synthesis_pages: result.synthesisPages,
|
||||
health_issues: result.healthIssues,
|
||||
watermark: result.watermark,
|
||||
duration_ms: result.durationMs,
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: `Compilation error: ${err instanceof Error ? err.message : String(err)}`,
|
||||
}],
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// ── get_page ───────────────────────────────────────────────────
|
||||
server.tool(
|
||||
'get_page',
|
||||
'Read a compiled wiki page by its slug (e.g., "project-alpha", "index", "synthesis-memory").',
|
||||
{
|
||||
slug: z.string().describe('Page slug (URL-safe name). Use "index" for the wiki index.'),
|
||||
},
|
||||
async ({ slug }) => {
|
||||
const { state } = await getCompiler();
|
||||
const page = state.getPage(slug);
|
||||
|
||||
if (!page) {
|
||||
// Try fuzzy match
|
||||
const allPages = state.getAllPages();
|
||||
const matches = allPages.filter(p =>
|
||||
p.slug.includes(slug) || p.name.toLowerCase().includes(slug.toLowerCase()),
|
||||
);
|
||||
|
||||
if (matches.length > 0) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: `Page "${slug}" not found. Did you mean:\n${matches.map(m => ` - ${m.slug} (${m.name})`).join('\n')}`,
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: `Page "${slug}" not found. Run compile_wiki first, or use search_wiki to find pages.`,
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
slug: page.slug,
|
||||
name: page.name,
|
||||
type: page.pageType,
|
||||
sources: page.sourceCount,
|
||||
compiled_at: page.compiledAt,
|
||||
content_hash: page.contentHash,
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// ── search_wiki ────────────────────────────────────────────────
|
||||
server.tool(
|
||||
'search_wiki',
|
||||
'Search compiled wiki pages by name or type. Returns matching page metadata.',
|
||||
{
|
||||
query: z.string().optional()
|
||||
.describe('Search query to match against page names'),
|
||||
type: z.enum(['entity', 'concept', 'synthesis', 'index', 'health']).optional()
|
||||
.describe('Filter by page type'),
|
||||
},
|
||||
async ({ query, type }) => {
|
||||
const { state } = await getCompiler();
|
||||
|
||||
let pages = type
|
||||
? state.getPagesByType(type)
|
||||
: state.getAllPages();
|
||||
|
||||
if (query) {
|
||||
const lower = query.toLowerCase();
|
||||
pages = pages.filter(p =>
|
||||
p.name.toLowerCase().includes(lower) ||
|
||||
p.slug.includes(lower),
|
||||
);
|
||||
}
|
||||
|
||||
if (pages.length === 0) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: query
|
||||
? `No wiki pages matching "${query}". Run compile_wiki to generate pages.`
|
||||
: 'No wiki pages compiled yet. Run compile_wiki first.',
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify(pages.map(p => ({
|
||||
slug: p.slug,
|
||||
name: p.name,
|
||||
type: p.pageType,
|
||||
sources: p.sourceCount,
|
||||
compiled_at: p.compiledAt,
|
||||
})), null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// ── compile_health ─────────────────────────────────────────────
|
||||
server.tool(
|
||||
'compile_health',
|
||||
'Run a health check on the wiki. Reports contradictions, gaps, orphan entities, weak confidence pages, and data quality score.',
|
||||
{},
|
||||
async () => {
|
||||
const { compiler } = await getCompiler();
|
||||
const report = compiler.compileHealth();
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
data_quality_score: report.dataQualityScore,
|
||||
total_entities: report.totalEntities,
|
||||
total_frames: report.totalFrames,
|
||||
total_pages: report.totalPages,
|
||||
issues: report.issues.map(i => ({
|
||||
type: i.type,
|
||||
severity: i.severity,
|
||||
description: i.description,
|
||||
...(i.suggestion && { suggestion: i.suggestion }),
|
||||
})),
|
||||
compiled_at: report.compiledAt,
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
102
packages/hive-mind-mcp-server/src/tools/workspace.ts
Normal file
102
packages/hive-mind-mcp-server/src/tools/workspace.ts
Normal file
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* Workspace tools — list_workspaces + create_workspace.
|
||||
* Wraps WorkspaceManager from @waggle/hive-mind-core.
|
||||
*/
|
||||
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { z } from 'zod';
|
||||
import { getWorkspaceManager, getWorkspaceMind, getFrameStore, getKnowledgeGraph } from '../core/setup.js';
|
||||
|
||||
export function registerWorkspaceTools(server: McpServer): void {
|
||||
|
||||
// ── list_workspaces ─────────────────────────────────────────────
|
||||
server.tool(
|
||||
'list_workspaces',
|
||||
'List all workspaces with their configuration and memory stats.',
|
||||
{},
|
||||
async () => {
|
||||
const wm = getWorkspaceManager();
|
||||
const workspaces = wm.list();
|
||||
|
||||
// Get personal mind stats
|
||||
const personalFrameStore = getFrameStore();
|
||||
const personalKg = getKnowledgeGraph();
|
||||
const personalStats = personalFrameStore.getStats();
|
||||
const personalEntityCount = personalKg.getEntityCount();
|
||||
|
||||
const result = {
|
||||
personal: {
|
||||
frames: personalStats.total,
|
||||
by_type: personalStats.byType,
|
||||
by_importance: personalStats.byImportance,
|
||||
entities: personalEntityCount,
|
||||
},
|
||||
workspaces: workspaces.map(ws => {
|
||||
// Try to get workspace mind stats (non-fatal if unavailable)
|
||||
let wsStats = { frames: 0, entities: 0 };
|
||||
try {
|
||||
const mind = getWorkspaceMind(ws.id);
|
||||
if (mind) {
|
||||
const fs = mind.frameStore.getStats();
|
||||
const ec = mind.knowledgeGraph.getEntityCount();
|
||||
wsStats = { frames: fs.total, entities: ec };
|
||||
}
|
||||
} catch { /* workspace mind not accessible */ }
|
||||
|
||||
return {
|
||||
id: ws.id,
|
||||
name: ws.name,
|
||||
group: ws.group,
|
||||
icon: ws.icon ?? null,
|
||||
model: ws.model ?? null,
|
||||
created: ws.created,
|
||||
...wsStats,
|
||||
};
|
||||
}),
|
||||
default_workspace: wm.getDefault(),
|
||||
};
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify(result, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// ── create_workspace ────────────────────────────────────────────
|
||||
server.tool(
|
||||
'create_workspace',
|
||||
'Create a new workspace with its own isolated memory mind.',
|
||||
{
|
||||
name: z.string().describe('Workspace name'),
|
||||
group: z.string().optional().describe('Group/category. Defaults to "general"'),
|
||||
icon: z.string().optional().describe('Optional short icon (emoji)'),
|
||||
model: z.string().optional().describe('Optional default model id for this workspace'),
|
||||
},
|
||||
async ({ name, group, icon, model }) => {
|
||||
const wm = getWorkspaceManager();
|
||||
const ws = wm.create({
|
||||
name,
|
||||
group: group ?? 'general',
|
||||
...(icon !== undefined && { icon }),
|
||||
...(model !== undefined && { model }),
|
||||
});
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text' as const,
|
||||
text: JSON.stringify({
|
||||
id: ws.id,
|
||||
name: ws.name,
|
||||
group: ws.group,
|
||||
icon: ws.icon ?? null,
|
||||
model: ws.model ?? null,
|
||||
created: ws.created,
|
||||
}, null, 2),
|
||||
}],
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user