oleg.m/waggle-os
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

moving

Browse Source
This commit is contained in:
Oleg Maslov
2026-09-02 10:14:22 +02:00
parent 0c3e2ead3b
commit b20b138fe4
771 changed files with 161561 additions and 9027 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/hive-mind-mcp-server/package.json
View File

@@ -12,6 +12,10 @@
".": {
"types": "./dist/index.d.ts",
"import": "./dist/index.js"
},
"./scope": {
"types": "./dist/scope.d.ts",
"import": "./dist/scope.js"
}
},
"scripts": {

3
packages/hive-mind-mcp-server/src/core/setup.ts
View File

@@ -164,6 +164,7 @@ export async function initialize(): Promise<void> {
_mindCache = new MultiMindCache({
maxOpen: 20,
getMindPath: (workspaceId: string) => _workspaceManager.getMindPath(workspaceId),
allowedRoot: path.join(_dataDir, 'workspaces'),
});
_initialized = true;
@@ -227,7 +228,7 @@ export function getWorkspaceMind(workspaceId: string): WorkspaceMindHandle | nul
export function shutdown(): void {
_workspaceMindLayerCache.clear();
_mindCache.closeAll();
_mindCache?.closeAll();
try { _personalDb?.close(); } catch { /* already closed */ }
_initialized = false;
}

127
packages/hive-mind-mcp-server/src/integration.test.ts
View File

@@ -1,4 +1,5 @@
import { describe, expect, it } from 'vitest';
import { win32 as pathWin32 } from 'node:path';
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import { registerMemoryTools } from './tools/memory.js';
@@ -7,7 +8,17 @@ import { registerIdentityTools } from './tools/identity.js';
import { registerAwarenessTools } from './tools/awareness.js';
import { registerWorkspaceTools } from './tools/workspace.js';
import { registerHarvestTools } from './tools/harvest.js';
import { registerCleanupTools } from './tools/cleanup.js';
import {
buildClaudeLaunch,
registerCleanupTools,
resolveConsolidationGop,
} from './tools/cleanup.js';
import {
collectObservations,
FrameStore,
MindDB,
SessionStore,
} from '@waggle/hive-mind-core';
import { registerIngestTools } from './tools/ingest.js';
import { registerWikiTools } from './tools/wiki.js';
import { registerResources } from './resources/memory.js';
@@ -42,6 +53,40 @@ function makeStub(): {
}
describe('@waggle/hive-mind-mcp-server registration wiring', () => {
it('builds a shell-free Windows Claude launch without ambient secrets', () => {
const npmRoot = 'C:\\Users\\test\\AppData\\Roaming\\npm';
const shim = pathWin32.join(npmRoot, 'claude.cmd');
const cli = pathWin32.join(npmRoot, 'node_modules', '@anthropic-ai', 'claude-code', 'cli.js');
const files = new Set([shim.toLowerCase(), cli.toLowerCase()]);
const launch = buildClaudeLaunch(['-p', '--output-format=text'], {
platform: 'win32',
env: {
Path: npmRoot,
USERPROFILE: 'C:\\Users\\test',
APPDATA: 'C:\\Users\\test\\AppData\\Roaming',
CLAUDE_CONFIG_DIR: 'C:\\Users\\test\\.claude-profile',
ANTHROPIC_API_KEY: 'must-not-reach-claude',
OPENAI_API_KEY: 'must-not-reach-claude',
WAGGLE_FUTURE_PROVIDER_SECRET: 'must-also-be-denied',
},
isFile: (candidate) => files.has(pathWin32.normalize(candidate).toLowerCase()),
});
expect(launch.command).toBe(process.execPath);
expect(launch.args).toEqual([cli, '-p', '--output-format=text']);
expect(launch.options.shell).toBe(false);
expect(launch.options.env).toMatchObject({
Path: npmRoot,
USERPROFILE: 'C:\\Users\\test',
APPDATA: 'C:\\Users\\test\\AppData\\Roaming',
CLAUDE_CONFIG_DIR: 'C:\\Users\\test\\.claude-profile',
HIVE_MIND_NO_SYNTH: '1',
});
expect(launch.options.env.ANTHROPIC_API_KEY).toBeUndefined();
expect(launch.options.env.OPENAI_API_KEY).toBeUndefined();
expect(launch.options.env.WAGGLE_FUTURE_PROVIDER_SECRET).toBeUndefined();
});
it('registerMemoryTools registers save_memory + recall_memory', () => {
const { server, tools } = makeStub();
registerMemoryTools(server);
@@ -131,3 +176,83 @@ describe('@waggle/hive-mind-mcp-server registration wiring', () => {
expect(names.size).toBe(tools.length);
});
});
describe('@waggle/hive-mind-mcp-server consolidation GOP anchor', () => {
function fixture(): { db: MindDB; frames: FrameStore; sessions: SessionStore } {
const db = new MindDB(':memory:');
return { db, frames: new FrameStore(db), sessions: new SessionStore(db) };
}
function setCreatedAt(db: MindDB, id: number, createdAt: string): void {
db.getDatabase().prepare('UPDATE memory_frames SET created_at = ? WHERE id = ?')
.run(createdAt, id);
}
it('ignores a newer excluded-source frame', () => {
const { db, frames, sessions } = fixture();
try {
const target = sessions.create();
const decoy = sessions.create();
const first = frames.createIFrame(target.gop_id, 'eligible first', 'normal', 'agent_inferred');
const second = frames.createIFrame(target.gop_id, 'eligible second', 'normal', 'agent_inferred');
const excluded = frames.createIFrame(decoy.gop_id, 'excluded future', 'normal', 'user_stated');
setCreatedAt(db, first.id, '2026-01-01 00:00:00');
setCreatedAt(db, second.id, '2026-01-02 00:00:00');
setCreatedAt(db, excluded.id, '2026-12-01 00:00:00');
const observations = collectObservations(db, { limit: 400 });
expect(resolveConsolidationGop(frames, observations)).toBe(target.gop_id);
} finally {
db.close();
}
});
it('uses canonical offset chronology instead of textual order', () => {
const { db, frames, sessions } = fixture();
try {
const olderSession = sessions.create();
const newerSession = sessions.create();
const older = frames.createIFrame(olderSession.gop_id, 'older instant', 'normal', 'agent_inferred');
const newer = frames.createIFrame(newerSession.gop_id, 'newer instant', 'normal', 'agent_inferred');
setCreatedAt(db, older.id, '2026-01-01 01:00:00+0200');
setCreatedAt(db, newer.id, '2026-01-01 00:30:00+0100');
const observations = collectObservations(db, { limit: 400 });
expect(resolveConsolidationGop(frames, observations)).toBe(newerSession.gop_id);
} finally {
db.close();
}
});
it('breaks equal-instant ties with the higher frame id', () => {
const { db, frames, sessions } = fixture();
try {
const lowerSession = sessions.create();
const higherSession = sessions.create();
const lower = frames.createIFrame(lowerSession.gop_id, 'equal lower id', 'normal', 'agent_inferred');
const higher = frames.createIFrame(higherSession.gop_id, 'equal higher id', 'normal', 'agent_inferred');
setCreatedAt(db, lower.id, '2026-01-01 13:00:00+0100');
setCreatedAt(db, higher.id, '2026-01-01 12:00:00Z');
const observations = collectObservations(db, { limit: 400 });
expect(resolveConsolidationGop(frames, observations)).toBe(higherSession.gop_id);
} finally {
db.close();
}
});
it('returns no anchor when there are no eligible observations', () => {
const { db, frames, sessions } = fixture();
try {
const session = sessions.create();
frames.createIFrame(session.gop_id, 'excluded one', 'normal', 'user_stated');
frames.createIFrame(session.gop_id, 'excluded two', 'normal', 'user_stated');
const observations = collectObservations(db, { limit: 400 });
expect(observations).toEqual([]);
expect(resolveConsolidationGop(frames, observations)).toBeNull();
} finally {
db.close();
}
});
});

12
packages/hive-mind-mcp-server/src/tools/awareness.ts
View File

@@ -6,7 +6,7 @@
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import { z } from 'zod';
import { getAwareness } from '../core/setup.js';
import type { AwarenessCategory } from '@waggle/hive-mind-core';
import { evaluateExternalMemoryIngress, type AwarenessCategory } from '@waggle/hive-mind-core';
export function registerAwarenessTools(server: McpServer): void {
@@ -66,6 +66,16 @@ export function registerAwarenessTools(server: McpServer): void {
.describe('Time-to-live in minutes. Item auto-expires after this duration'),
},
async ({ category, content, priority, ttl_minutes }) => {
if (evaluateExternalMemoryIngress({ content }).action === 'block') {
return {
content: [{
type: 'text' as const,
text: 'Error: Awareness content could not be saved.',
}],
isError: true,
};
}
const awareness = getAwareness();
let expiresAt: string | undefined;

123
packages/hive-mind-mcp-server/src/tools/cleanup.ts
View File

@@ -6,6 +6,8 @@
*/
import { spawn } from 'node:child_process';
import { statSync } from 'node:fs';
import { win32 as pathWin32 } from 'node:path';
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import { z } from 'zod';
import {
@@ -76,13 +78,103 @@ function isNoiseEntity(name: string, entityType: string): boolean {
// OpenAI-style model id + OPENAI_API_KEY routes to the OpenAI chat API — the
// executor the benchmark validated with.
const CLAUDE_ENV_ALLOWLIST = new Set([
'PATH', 'PATHEXT', 'SYSTEMROOT', 'WINDIR', 'SYSTEMDRIVE', 'COMSPEC',
'HOME', 'USERPROFILE', 'HOMEDRIVE', 'HOMEPATH', 'USER', 'USERNAME',
'LOGNAME', 'SHELL', 'APPDATA', 'LOCALAPPDATA', 'PROGRAMDATA',
'PROGRAMFILES', 'PROGRAMFILES(X86)', 'PROGRAMW6432',
'TEMP', 'TMP', 'TMPDIR', 'LANG', 'LANGUAGE', 'LC_ALL', 'LC_CTYPE',
'TERM', 'COLORTERM', 'TZ', 'XDG_CONFIG_HOME', 'XDG_DATA_HOME',
'XDG_CACHE_HOME', 'XDG_STATE_HOME', 'CLAUDE_CONFIG_DIR',
'NODE_EXTRA_CA_CERTS', 'SSL_CERT_FILE', 'SSL_CERT_DIR',
]);
export interface ClaudeLaunchDeps {
platform?: NodeJS.Platform;
env?: NodeJS.ProcessEnv;
isFile?: (candidate: string) => boolean;
}
export interface ClaudeLaunch {
command: string;
args: string[];
options: {
stdio: ['pipe', 'pipe', 'pipe'];
shell: false;
windowsHide: true;
env: NodeJS.ProcessEnv;
};
}
function regularFile(candidate: string): boolean {
try { return statSync(candidate).isFile(); } catch { return false; }
}
function envValue(env: NodeJS.ProcessEnv, name: string): string | undefined {
const found = Object.entries(env).find(([key]) => key.toUpperCase() === name);
return found?.[1];
}
function claudeProcessEnv(base: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
const env: NodeJS.ProcessEnv = {};
for (const [key, value] of Object.entries(base)) {
if (value !== undefined && CLAUDE_ENV_ALLOWLIST.has(key.toUpperCase())) env[key] = value;
}
env.HIVE_MIND_NO_SYNTH = '1';
return env;
}
export function buildClaudeLaunch(
args: string[],
deps: ClaudeLaunchDeps = {},
): ClaudeLaunch {
const platform = deps.platform ?? process.platform;
const sourceEnv = deps.env ?? process.env;
const env = claudeProcessEnv(sourceEnv);
const isFile = deps.isFile ?? regularFile;
let command = 'claude';
let launchArgs = [...args];
if (platform === 'win32') {
const pathEntries = (envValue(env, 'PATH') ?? '')
.split(';')
.map((entry) => entry.trim().replace(/^"|"$/g, ''))
.filter(Boolean);
let resolved = false;
for (const directory of pathEntries) {
const executable = pathWin32.join(directory, 'claude.exe');
if (isFile(executable)) {
command = executable;
resolved = true;
break;
}
const shim = pathWin32.join(directory, 'claude.cmd');
if (!isFile(shim)) continue;
const cliCandidates = [
pathWin32.join(directory, 'node_modules', '@anthropic-ai', 'claude-code', 'cli.js'),
pathWin32.resolve(directory, '..', '@anthropic-ai', 'claude-code', 'cli.js'),
];
const cli = cliCandidates.find(isFile);
if (!cli) continue;
command = process.execPath;
launchArgs = [cli, ...args];
resolved = true;
break;
}
if (!resolved) throw new Error('Claude CLI not found on the sanitized Windows PATH');
}
return {
command,
args: launchArgs,
options: { stdio: ['pipe', 'pipe', 'pipe'], shell: false, windowsHide: true, env },
};
}
function spawnClaudeText(prompt: string, timeoutMs = 120_000): Promise<string> {
return new Promise((resolve, reject) => {
const proc = spawn('claude', ['-p', '--output-format=text'], {
stdio: ['pipe', 'pipe', 'pipe'],
shell: process.platform === 'win32',
env: { ...process.env, HIVE_MIND_NO_SYNTH: '1' },
});
const launch = buildClaudeLaunch(['-p', '--output-format=text']);
const proc = spawn(launch.command, launch.args, launch.options);
let stdout = '';
let stderr = '';
let settled = false;
@@ -162,6 +254,16 @@ interface ConsolidationCounts {
deprecated: number;
}
export function resolveConsolidationGop(
frameStore: FrameStore,
observations: readonly { id: number }[],
): string | null {
const newestObservation = observations[observations.length - 1];
if (!newestObservation) return null;
const anchor = frameStore.getById(newestObservation.id);
return anchor?.gop_id ?? null;
}
/**
* Run the P/B consolidation pass on a mind: gather I-frame observations,
* LLM-detect chains + groups, apply (deprecate stale + emit P/B frames), then
@@ -178,20 +280,15 @@ async function runConsolidation(
const observations = collectObservations(db, { limit });
if (observations.length < 2) return empty;
const anchor = db
.getDatabase()
.prepare(
"SELECT gop_id FROM memory_frames WHERE frame_type = 'I' AND importance != 'deprecated' ORDER BY created_at DESC, id DESC LIMIT 1",
)
.get() as { gop_id: string } | undefined;
if (!anchor) return empty;
const anchorGop = resolveConsolidationGop(frameStore, observations);
if (!anchorGop) return empty;
const llm = buildConsolidationLlm(model);
const [chains, groups] = await Promise.all([
detectSupersessionChains(observations, llm),
detectEntityGroups(observations, llm),
]);
const { pframes, bframes, deprecated } = applyConsolidation(frameStore, chains, groups, anchor.gop_id);
const { pframes, bframes, deprecated } = applyConsolidation(frameStore, chains, groups, anchorGop);
const toIndex = [
...pframes.map((f) => ({ id: f.id, content: f.content })),

127
packages/hive-mind-mcp-server/src/tools/harvest.ts
View File

@@ -6,7 +6,18 @@
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import { z } from 'zod';
import fs from 'node:fs';
import { resolveRelativeDate, HARVEST_FRAME_CONTENT_CAP, writeRawTurnFrames, RawArchive, SuppressionStore, readArchiveUids, withArchiveUid } from '@waggle/hive-mind-core';
import {
evaluateExternalMemoryIngress,
projectExternalMemoryContent,
resolveRelativeDate,
HARVEST_FRAME_CONTENT_CAP,
MAX_TURNS_PER_ITEM,
writeRawTurnFrames,
RawArchive,
SuppressionStore,
readArchiveUids,
withArchiveUid,
} from '@waggle/hive-mind-core';
import {
getFrameStore,
getSessions,
@@ -16,6 +27,7 @@ import {
getPersonalDb,
getAdapter,
} from '../core/setup.js';
import { resolveImportFilePath } from './ingest.js';
export function registerHarvestTools(server: McpServer): void {
@@ -30,15 +42,16 @@ export function registerHarvestTools(server: McpServer): void {
data: z.string().optional()
.describe('JSON string of the export data. Provide this OR file_path, not both'),
file_path: z.string().optional()
.describe('Path to the export file on disk. Provide this OR data, not both'),
.describe('Relative path beneath HIVE_MIND_MCP_IMPORT_ROOT. Provide this OR data, not both'),
},
async ({ source, data, file_path }) => {
// Validate: one of data or file_path must be provided
if (!data && !file_path) {
// Keep raw JSON and local path inputs separate. Local files are resolved
// only beneath the explicit MCP import root.
if ((data === undefined) === (file_path === undefined)) {
return {
content: [{
type: 'text' as const,
text: 'Error: provide either "data" (JSON string) or "file_path" (path to export file)',
text: 'Error: provide either "data" (JSON string) or "file_path" (relative import path), not both',
}],
isError: true,
};
@@ -47,8 +60,9 @@ export function registerHarvestTools(server: McpServer): void {
// Parse input
let parsed: unknown;
try {
if (file_path) {
const raw = fs.readFileSync(file_path, 'utf-8');
if (file_path !== undefined) {
const safePath = resolveImportFilePath(file_path);
const raw = fs.readFileSync(safePath, 'utf-8');
parsed = JSON.parse(raw);
} else {
parsed = JSON.parse(data!);
@@ -76,6 +90,99 @@ export function registerHarvestTools(server: McpServer): void {
};
}
const preparedItems = items.map((item) => {
const storedContent = item.content.slice(0, HARVEST_FRAME_CONTENT_CAP);
const content = item.title
? `[${item.source}] ${item.title}: ${storedContent}`
: `[${item.source}] ${storedContent}`;
const ingressContent = projectExternalMemoryContent({
content: item.content,
messages: item.messages,
parseMethod: item.metadata?.parseMethod,
maxChars: HARVEST_FRAME_CONTENT_CAP,
});
const ingressFrameContent = item.title
? `[${item.source}] ${item.title}: ${ingressContent}`
: `[${item.source}] ${ingressContent}`;
const archiveIngressContent = projectExternalMemoryContent({
content: item.content,
messages: item.messages,
parseMethod: item.metadata?.parseMethod,
});
const entityProjections: Array<{ type: string; name: string; recalled: string }> = [];
if (Array.isArray(item.metadata?.entities)) {
for (const entity of item.metadata.entities) {
if (!entity || typeof entity !== 'object') continue;
const { name, type } = entity as Record<string, unknown>;
if (typeof name !== 'string') continue;
const storedType = typeof type === 'string' && type ? type : 'concept';
entityProjections.push({
type: storedType,
name,
recalled: `${storedType}: ${name}`,
});
}
}
const rawTurnProjections: Array<{ content: string; timestamp?: string }> = [];
if (process.env.WAGGLE_RAWDETAIL !== '0' && Array.isArray(item.messages)) {
for (const message of item.messages) {
if (message.role !== 'user' && message.role !== 'assistant') continue;
const rawTurnContent = (message.text ?? '').trim();
if (!rawTurnContent) continue;
if (rawTurnProjections.length >= MAX_TURNS_PER_ITEM) break;
rawTurnProjections.push({
content: rawTurnContent.slice(0, HARVEST_FRAME_CONTENT_CAP),
timestamp: message.timestamp,
});
}
}
return {
item,
content,
ingressFrameContent,
archiveIngressContent,
entityProjections,
rawTurnProjections,
};
});
const hasUnsafeContent = (file_path !== undefined
&& evaluateExternalMemoryIngress({ content: file_path }).action !== 'allow')
|| preparedItems.some(({
item,
ingressFrameContent,
archiveIngressContent,
entityProjections,
rawTurnProjections,
}) => {
if (evaluateExternalMemoryIngress({ content: ingressFrameContent }).action !== 'allow'
|| evaluateExternalMemoryIngress({
title: item.title,
content: archiveIngressContent,
}).action !== 'allow'
|| [item.source, item.id, item.timestamp].some((value) =>
evaluateExternalMemoryIngress({ content: value }).action !== 'allow')) {
return true;
}
if (entityProjections.some(({ type, name, recalled }) =>
evaluateExternalMemoryIngress({ content: recalled }).action !== 'allow'
|| evaluateExternalMemoryIngress({ title: type, content: name }).action !== 'allow')) {
return true;
}
return rawTurnProjections.some(({ content: rawTurnContent, timestamp }) =>
evaluateExternalMemoryIngress({ content: rawTurnContent }).action !== 'allow'
|| (timestamp !== undefined
&& evaluateExternalMemoryIngress({ content: timestamp }).action !== 'allow'));
});
if (hasUnsafeContent) {
return {
content: [{
type: 'text' as const,
text: 'Error: imported content was blocked by the memory safety policy.',
}],
isError: true,
};
}
// Save each item as an I-Frame in the personal mind
const frameStore = getFrameStore();
const sessions = getSessions();
@@ -111,12 +218,8 @@ export function registerHarvestTools(server: McpServer): void {
const suppression = new SuppressionStore(getPersonalDb());
let suppressedSkipped = 0;
for (const item of items) {
for (const { item, content } of preparedItems) {
if (suppression.isSuppressed(item.source, item.id)) { suppressedSkipped++; continue; }
// Build a summary from the conversation
const content = item.title
? `[${item.source}] ${item.title}: ${item.content.slice(0, HARVEST_FRAME_CONTENT_CAP)}`
: `[${item.source}] ${item.content.slice(0, HARVEST_FRAME_CONTENT_CAP)}`;
// Write-time temporal anchoring. The frame's created_at should reflect WHEN the
// event happened, not the ingest wall-clock. Start from the source timestamp; if

19
packages/hive-mind-mcp-server/src/tools/identity.ts
View File

@@ -5,6 +5,7 @@
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import { z } from 'zod';
import { evaluateExternalMemoryIngress } from '@waggle/hive-mind-core';
import { getIdentity } from '../core/setup.js';
export function registerIdentityTools(server: McpServer): void {
@@ -61,6 +62,24 @@ export function registerIdentityTools(server: McpServer): void {
system_prompt: z.string().optional().describe('Custom system prompt additions'),
},
async ({ name, role, department, personality, capabilities, system_prompt }) => {
const projectedContext = [
name !== undefined ? `Name: ${name}` : undefined,
role !== undefined ? `Role: ${role}` : undefined,
department !== undefined ? `Department: ${department}` : undefined,
personality !== undefined ? `Personality: ${personality}` : undefined,
capabilities !== undefined ? `Capabilities: ${capabilities}` : undefined,
system_prompt !== undefined ? `System Prompt: ${system_prompt}` : undefined,
].filter((value): value is string => value !== undefined).join('\n');
if (evaluateExternalMemoryIngress({ content: projectedContext }).action === 'block') {
return {
content: [{
type: 'text' as const,
text: 'Error: Identity content could not be saved.',
}],
isError: true,
};
}
const identity = getIdentity();
if (!identity.exists()) {

267
packages/hive-mind-mcp-server/src/tools/ingest.ts
View File

@@ -8,6 +8,7 @@
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import { z } from 'zod';
import fs from 'node:fs';
import path from 'node:path';
import {
getFrameStore,
getSessions,
@@ -17,19 +18,134 @@ import {
getPersonalDb,
getAdapter,
} from '../core/setup.js';
import { UrlAdapter } from '@waggle/hive-mind-core';
import {
evaluateExternalMemoryIngress,
projectExternalMemoryContent,
UrlAdapter,
} from '@waggle/hive-mind-core';
import type { PdfAdapter } from '@waggle/hive-mind-core';
import type { UniversalImportItem } from '@waggle/hive-mind-core';
const IMPORT_ROOT_ENV = 'HIVE_MIND_MCP_IMPORT_ROOT';
const SENSITIVE_DIRS = new Set([
'.ssh', '.aws', '.gnupg', '.gpg', '.docker', '.kube', '.azure', '.terraform', '.terraform.d',
]);
const SENSITIVE_FILES = new Set([
'id_rsa', 'id_dsa', 'id_ecdsa', 'id_ed25519', 'authorized_keys', 'known_hosts',
'.netrc', '.pgpass', '.npmrc', '.pypirc', '.git-credentials',
'credentials', 'credentials.json', 'service-account.json',
'terraform.tfstate', 'terraform.tfstate.backup',
]);
const SENSITIVE_EXTENSIONS = new Set(['.pem']);
const BACKUP_SUFFIX_RE = /\.(bak|old|backup|orig|copy|save|swp)$/i;
const SAFE_ENV_TEMPLATES = new Set([
'.env.example', '.env.sample', '.env.template', '.env.dist', '.env.defaults',
]);
function normalizeSegment(segment: string): string {
return segment.toLowerCase().replace(/::.*$/, '').replace(/[. ]+$/, '');
}
function isSensitiveBase(base: string): boolean {
if (SENSITIVE_FILES.has(base)) return true;
const dot = base.lastIndexOf('.');
if (dot > 0 && SENSITIVE_EXTENSIONS.has(base.slice(dot))) return true;
if (base === '.env' || base.startsWith('.env.')) return !SAFE_ENV_TEMPLATES.has(base);
return false;
}
function isSensitivePath(candidate: string): boolean {
const segments = candidate.replace(/\\/g, '/').split('/').map(normalizeSegment).filter(Boolean);
if (segments.some((segment) => SENSITIVE_DIRS.has(segment))) return true;
const base = segments.at(-1);
if (!base) return false;
if (isSensitiveBase(base)) return true;
if (BACKUP_SUFFIX_RE.test(base) && isSensitiveBase(base.replace(BACKUP_SUFFIX_RE, ''))) return true;
return false;
}
function isOutside(root: string, candidate: string): boolean {
const relative = path.relative(root, candidate);
return relative === '..'
|| relative.startsWith(`..${path.sep}`)
|| path.isAbsolute(relative);
}
function isCallerAbsolute(candidate: string): boolean {
return path.isAbsolute(candidate)
|| path.win32.isAbsolute(candidate)
|| path.posix.isAbsolute(candidate)
|| /^[A-Za-z]:/.test(candidate);
}
/** Resolve one caller-supplied relative file under the explicitly configured import root. */
export function resolveImportFilePath(relativePath: string): string {
const configuredRoot = process.env[IMPORT_ROOT_ENV]?.trim();
if (!configuredRoot) {
throw new Error(`Local file imports are disabled. Set ${IMPORT_ROOT_ENV} to an absolute directory.`);
}
if (!path.isAbsolute(configuredRoot)) {
throw new Error(`${IMPORT_ROOT_ENV} must be an absolute directory.`);
}
if (!relativePath.trim() || relativePath.includes('\0')) {
throw new Error('Import file_path must be a non-empty relative path.');
}
if (isCallerAbsolute(relativePath)) {
throw new Error('Absolute import file paths are denied; provide a path relative to the configured import root.');
}
const segments = relativePath.replace(/\\/g, '/').split('/');
if (segments.some((segment) => segment === '..' || segment === '.' || segment === '' || segment.includes(':'))) {
throw new Error(`Import path traversal denied: ${relativePath}`);
}
if (isSensitivePath(relativePath)) {
throw new Error(`Access to sensitive import file denied: ${relativePath}`);
}
let realRoot: string;
try {
realRoot = fs.realpathSync.native(configuredRoot);
} catch {
throw new Error(`Configured import root does not exist: ${configuredRoot}`);
}
if (!fs.statSync(realRoot).isDirectory()) {
throw new Error(`Configured import root is not a directory: ${configuredRoot}`);
}
const lexicalTarget = path.resolve(realRoot, ...segments);
if (isOutside(realRoot, lexicalTarget)) {
throw new Error(`Import path resolves outside the configured root: ${relativePath}`);
}
let realTarget: string;
try {
realTarget = fs.realpathSync.native(lexicalTarget);
} catch {
throw new Error(`Import file does not exist: ${relativePath}`);
}
if (isOutside(realRoot, realTarget)) {
throw new Error(`Import path resolves outside the configured root through a symlink: ${relativePath}`);
}
if (isSensitivePath(path.relative(realRoot, realTarget))) {
throw new Error(`Access to sensitive import file denied: ${relativePath}`);
}
if (!fs.statSync(realTarget).isFile()) {
throw new Error(`Import path is not a regular file: ${relativePath}`);
}
return realTarget;
}
export function registerIngestTools(server: McpServer): void {
// ── ingest_source ──────────────────────────────────────────────
server.tool(
'ingest_source',
'Ingest a document, URL, or text into the memory system. Auto-detects content type or use type_hint. Supports: markdown files, plain text, PDF files, web URLs, and raw text content.',
'Ingest raw text, a web URL, or a file beneath the configured MCP import root. Raw content is never interpreted as a local path.',
{
content: z.string()
.describe('Content to ingest: a file path, URL, or raw text/markdown content'),
content: z.string().optional()
.describe('Raw text/markdown content or a web URL. Provide this OR file_path, not both'),
file_path: z.string().optional()
.describe(`Relative path beneath ${IMPORT_ROOT_ENV}. Provide this OR content, not both`),
type_hint: z.enum(['markdown', 'plaintext', 'pdf', 'url', 'auto']).default('auto')
.describe('Content type hint. "auto" detects from content (default)'),
importance: z.enum(['critical', 'important', 'normal']).default('normal')
@@ -39,28 +155,66 @@ export function registerIngestTools(server: McpServer): void {
workspace: z.string().optional()
.describe('Workspace ID. Omit for personal mind'),
},
async ({ content, type_hint, importance, tags }) => {
// Detect content type
async ({ content, file_path, type_hint, importance, tags }) => {
if ((content === undefined) === (file_path === undefined)) {
return {
content: [{
type: 'text' as const,
text: 'Error: provide either "content" or "file_path", not both',
}],
isError: true,
};
}
let input: string;
try {
input = file_path === undefined ? content! : resolveImportFilePath(file_path);
} catch (err) {
return {
content: [{
type: 'text' as const,
text: `Error processing input: ${err instanceof Error ? err.message : String(err)}`,
}],
isError: true,
};
}
const detectedType = type_hint === 'auto'
? detectContentType(content)
? (file_path === undefined ? detectContentType(input) : detectFileContentType(file_path))
: type_hint;
if (file_path !== undefined && detectedType === 'url') {
return {
content: [{ type: 'text' as const, text: 'Error processing input: a local file_path cannot use the url type hint.' }],
isError: true,
};
}
if (file_path === undefined && detectedType === 'pdf') {
return {
content: [{ type: 'text' as const, text: 'Error processing input: PDF imports require file_path beneath the configured import root.' }],
isError: true,
};
}
let items: UniversalImportItem[];
try {
if (detectedType === 'url') {
// URL requires async fetch
const urlAdapter = new UrlAdapter();
items = await urlAdapter.fetchAndParse(content);
items = await urlAdapter.fetchAndParse(input);
} else if (detectedType === 'pdf') {
// PDF requires async parse
const { PdfAdapter: PdfAdapterClass } = await import('@waggle/hive-mind-core');
const pdfAdapter = new PdfAdapterClass() as PdfAdapter;
items = await pdfAdapter.parseFile(content);
items = await pdfAdapter.parseFile(input);
} else {
// Markdown, plaintext, or raw text — synchronous
const adapter = getAdapter(detectedType);
items = adapter.parse(content);
const adapterInput = file_path === undefined && input.length < 500 && !input.includes('\n')
? `${input}\n`
: input;
items = adapter.parse(adapterInput);
}
} catch (err) {
return {
@@ -81,6 +235,60 @@ export function registerIngestTools(server: McpServer): void {
};
}
const preparedItems = items.map((item) => {
const storedContent = item.content.slice(0, 3000);
const frameContent = item.title
? `[${detectedType}] ${item.title}: ${storedContent}`
: `[${detectedType}] ${storedContent}`;
const ingressContent = projectExternalMemoryContent({
content: item.content,
messages: item.messages,
parseMethod: item.metadata?.parseMethod,
maxChars: 3000,
});
const ingressFrameContent = item.title
? `[${detectedType}] ${item.title}: ${ingressContent}`
: `[${detectedType}] ${ingressContent}`;
const entityProjections: Array<{ type: string; name: string; recalled: string }> = [];
if (Array.isArray(item.metadata?.entities)) {
for (const entity of item.metadata.entities) {
if (!entity || typeof entity !== 'object') continue;
const { name, type } = entity as Record<string, unknown>;
if (typeof name !== 'string') continue;
const storedType = typeof type === 'string' && type ? type : 'concept';
entityProjections.push({
type: storedType,
name,
recalled: `${storedType}: ${name}`,
});
}
}
return { item, frameContent, ingressContent, ingressFrameContent, entityProjections };
});
const sourcePath = input.startsWith('http') ? input : undefined;
const hasUnsafeContent = (sourcePath !== undefined
&& evaluateExternalMemoryIngress({ content: sourcePath }).action === 'block')
|| preparedItems.some(({
item, ingressContent, ingressFrameContent, entityProjections,
}) => {
if (evaluateExternalMemoryIngress({ content: ingressFrameContent }).action === 'block'
|| evaluateExternalMemoryIngress({ title: item.title, content: ingressContent }).action === 'block') {
return true;
}
return entityProjections.some(({ type, name, recalled }) =>
evaluateExternalMemoryIngress({ content: recalled }).action === 'block'
|| evaluateExternalMemoryIngress({ title: type, content: name }).action === 'block');
});
if (hasUnsafeContent) {
return {
content: [{
type: 'text' as const,
text: 'Error: imported content was blocked by the memory safety policy.',
}],
isError: true,
};
}
// Store items as frames
const frameStore = getFrameStore();
const sessions = getSessions();
@@ -102,11 +310,7 @@ export function registerIngestTools(server: McpServer): void {
const maxBefore =
(rawDb.prepare('SELECT COALESCE(MAX(id), 0) AS m FROM memory_frames').get() as { m: number }).m;
for (const item of items) {
const frameContent = item.title
? `[${detectedType}] ${item.title}: ${item.content.slice(0, 3000)}`
: `[${detectedType}] ${item.content.slice(0, 3000)}`;
for (const { frameContent, entityProjections } of preparedItems) {
const frame = frameStore.createIFrame(
sessionId,
frameContent,
@@ -125,11 +329,10 @@ export function registerIngestTools(server: McpServer): void {
} catch { /* non-fatal */ }
// Extract entities from metadata
const metaEntities = item.metadata?.entities;
if (Array.isArray(metaEntities)) {
for (const ent of metaEntities as { name: string; type: string }[]) {
if (entityProjections.length > 0) {
for (const entity of entityProjections) {
try {
kg.createEntity(ent.type || 'concept', ent.name, {
kg.createEntity(entity.type, entity.name, {
source: detectedType,
...(tags && { tags }),
});
@@ -147,7 +350,7 @@ export function registerIngestTools(server: McpServer): void {
harvestStore.upsert(
sourceKey as Parameters<typeof harvestStore.upsert>[0],
items[0]?.title ?? detectedType,
content.startsWith('http') ? content : undefined,
sourcePath,
);
harvestStore.recordSync(
sourceKey as Parameters<typeof harvestStore.recordSync>[0],
@@ -181,23 +384,6 @@ function detectContentType(input: string): 'markdown' | 'plaintext' | 'pdf' | 'u
return 'url';
}
// File path detection
if (trimmed.length < 500 && !trimmed.includes('\n')) {
const lower = trimmed.toLowerCase();
if (lower.endsWith('.pdf')) return 'pdf';
if (lower.endsWith('.md') || lower.endsWith('.markdown')) return 'markdown';
if (lower.endsWith('.txt')) return 'plaintext';
// Check if it's an existing file
try {
if (fs.existsSync(trimmed)) {
if (lower.endsWith('.pdf')) return 'pdf';
if (lower.endsWith('.md')) return 'markdown';
return 'plaintext';
}
} catch { /* not a path */ }
}
// Content-based detection
if (trimmed.startsWith('#') || trimmed.includes('\n## ') || trimmed.includes('\n### ')) {
return 'markdown';
@@ -205,3 +391,10 @@ function detectContentType(input: string): 'markdown' | 'plaintext' | 'pdf' | 'u
return 'plaintext';
}
function detectFileContentType(filePath: string): 'markdown' | 'plaintext' | 'pdf' {
const lower = filePath.toLowerCase();
if (lower.endsWith('.pdf')) return 'pdf';
if (lower.endsWith('.md') || lower.endsWith('.markdown')) return 'markdown';
return 'plaintext';
}

27
packages/hive-mind-mcp-server/src/tools/memory.ts
View File

@@ -12,7 +12,11 @@ import {
getWorkspaceMind,
getWorkspaceManager,
} from '../core/setup.js';
import type { Importance, FrameSource } from '@waggle/hive-mind-core';
import {
evaluateExternalMemoryIngress,
type Importance,
type FrameSource,
} from '@waggle/hive-mind-core';
export function registerMemoryTools(server: McpServer): void {
@@ -30,11 +34,28 @@ export function registerMemoryTools(server: McpServer): void {
.describe('Workspace ID to save into. Omit for personal memory'),
},
async ({ content, importance, source, workspace }) => {
if (evaluateExternalMemoryIngress({ content }).action === 'block') {
return {
content: [{
type: 'text' as const,
text: 'Error: Memory content could not be saved.',
}],
isError: true,
};
}
const imp = (importance ?? 'normal') as Importance;
const src = (source ?? 'agent_inferred') as FrameSource;
// Resolve target mind
const target = workspace ? getWorkspaceMind(workspace) : null;
const workspaceRequested = workspace !== undefined;
const target = workspaceRequested ? getWorkspaceMind(workspace) : null;
if (workspaceRequested && !target) {
return {
content: [{ type: 'text' as const, text: 'Error: Requested workspace is unavailable.' }],
isError: true,
};
}
const frameStore = target?.frameStore ?? getFrameStore();
const sessions = target?.sessions ?? getSessions();
const search = target?.search ?? getSearch();
@@ -62,7 +83,7 @@ export function registerMemoryTools(server: McpServer): void {
importance: frame.importance,
source: frame.source,
created_at: frame.created_at,
workspace: workspace ?? 'personal',
workspace: workspaceRequested ? workspace : 'personal',
}, null, 2),
}],
};

69
packages/hive-mind-mcp-server/tests/runtime.test.ts
View File

@@ -5,6 +5,7 @@ import path from 'node:path';
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
import { describe, expect, it } from 'vitest';
import { shutdown } from '../src/core/setup.js';
const ROOT = path.resolve(import.meta.dirname, '..', '..', '..');
const SERVER_ENTRY = path.join(ROOT, 'packages', 'hive-mind-mcp-server', 'dist', 'index.js');
@@ -20,14 +21,16 @@ interface AsyncRunResult {
stderr: string;
}
function run(command: string, args: string[]): Promise<AsyncRunResult> {
return runInCwd(command, args, ROOT);
}
function runInCwd(command: string, args: string[], cwd: string): Promise<AsyncRunResult> {
function runInCwd(
command: string,
args: string[],
cwd: string,
env: NodeJS.ProcessEnv = process.env,
): Promise<AsyncRunResult> {
return new Promise((resolve, reject) => {
const child = spawn(command, args, {
cwd,
env,
shell: process.platform === 'win32' && command.endsWith('.cmd'),
});
let stdout = '';
@@ -41,6 +44,37 @@ function runInCwd(command: string, args: string[], cwd: string): Promise<AsyncRu
});
}
function runNpm(args: string[], cwd: string = ROOT): Promise<AsyncRunResult> {
const bundledNpmCli = path.join(
path.dirname(process.execPath),
'node_modules',
'waggle-node-runtime',
'node_modules',
'npm',
'bin',
'npm-cli.js',
);
const npmCli = [bundledNpmCli, process.env.npm_execpath]
.find((candidate): candidate is string => (
typeof candidate === 'string' && fs.existsSync(candidate)
));
const npmEnv = { ...process.env };
const inheritedPath = Object.entries(npmEnv)
.find(([key]) => key.toLowerCase() === 'path')?.[1] ?? '';
for (const key of Object.keys(npmEnv)) {
if (key.toLowerCase() === 'path') delete npmEnv[key];
}
npmEnv.PATH = [
...(npmCli ? [path.dirname(npmCli)] : []),
path.dirname(process.execPath),
inheritedPath,
].filter(Boolean).join(path.delimiter);
return npmCli
? runInCwd(process.execPath, [npmCli, ...args], cwd, npmEnv)
: runInCwd(bin('npm'), args, cwd, npmEnv);
}
async function withTimeout<T>(work: Promise<T>, ms: number): Promise<T> {
let timeout: NodeJS.Timeout | undefined;
const timer = new Promise<never>((_, reject) => {
@@ -68,14 +102,19 @@ const HIVE_MIND_MCP_PACKAGE_CLOSURE = [
] as const;
describe('@waggle/hive-mind-mcp-server built runtime', () => {
it('shuts down safely before initialization and remains idempotent', () => {
expect(() => shutdown()).not.toThrow();
expect(() => shutdown()).not.toThrow();
});
it('saves and recalls memory through the built write-scope MCP server', async () => {
const coreBuild = await run(bin('npm'), ['run', 'build', '--workspace', '@waggle/hive-mind-core']);
const coreBuild = await runNpm(['run', 'build', '--workspace', '@waggle/hive-mind-core']);
expect(coreBuild.status).toBe(0);
const wikiBuild = await run(bin('npm'), ['run', 'build', '--workspace', '@waggle/hive-mind-wiki-compiler']);
const wikiBuild = await runNpm(['run', 'build', '--workspace', '@waggle/hive-mind-wiki-compiler']);
expect(wikiBuild.status).toBe(0);
const mcpBuild = await run(bin('npm'), ['run', 'build', '--workspace', '@waggle/hive-mind-mcp-server']);
const mcpBuild = await runNpm(['run', 'build', '--workspace', '@waggle/hive-mind-mcp-server']);
expect(mcpBuild.status).toBe(0);
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hive-mind-mcp-write-'));
@@ -122,7 +161,7 @@ describe('@waggle/hive-mind-mcp-server built runtime', () => {
await client.close().catch(() => {});
fs.rmSync(dataDir, { recursive: true, force: true });
}
}, 30_000);
}, 60_000);
it('installs the local package closure and lists tools from the installed server', async () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hive-mind-mcp-installed-'));
@@ -137,11 +176,10 @@ describe('@waggle/hive-mind-mcp-server built runtime', () => {
try {
const dependencies: Record<string, string> = {};
for (const workspace of HIVE_MIND_MCP_PACKAGE_CLOSURE) {
const build = await run(bin('npm'), ['run', 'build', '--workspace', workspace]);
const build = await runNpm(['run', 'build', '--workspace', workspace]);
expect(build.status).toBe(0);
const pack = await run(
bin('npm'),
const pack = await runNpm(
['pack', '--workspace', workspace, '--pack-destination', packsDir, '--json'],
);
expect(pack.status).toBe(0);
@@ -156,8 +194,7 @@ describe('@waggle/hive-mind-mcp-server built runtime', () => {
JSON.stringify({ private: true, type: 'module', dependencies }, null, 2),
);
const install = await runInCwd(
bin('npm'),
const install = await runNpm(
['install', '--no-audit', '--no-fund', '--prefer-offline'],
projectDir,
);
@@ -182,7 +219,7 @@ describe('@waggle/hive-mind-mcp-server built runtime', () => {
} as Record<string, string>,
});
await withTimeout(client.connect(transport), 10_000);
await withTimeout(client.connect(transport), 30_000);
const tools = await withTimeout(client.listTools(), 10_000);
const names = tools.tools.map((tool) => tool.name);
@@ -193,5 +230,5 @@ describe('@waggle/hive-mind-mcp-server built runtime', () => {
await client.close().catch(() => {});
fs.rmSync(tempDir, { recursive: true, force: true });
}
}, 120_000);
}, 180_000);
});