This commit is contained in:
Oleg Maslov
2026-09-02 10:14:22 +02:00
parent 0c3e2ead3b
commit b20b138fe4
771 changed files with 161561 additions and 9027 deletions

View File

@@ -20,8 +20,8 @@
*/
import type { LLMCallFn } from './pipeline.js';
import { scanForInjection } from '../injection-scanner.js';
import { createCoreLogger } from '../logger.js';
import { evaluateExternalMemoryIngress } from '../memory-ingress-guard.js';
import { isNoiseName, normalizeEntityName } from '../mind/entity-normalizer.js';
import type { KnowledgeGraph } from '../mind/knowledge.js';
@@ -138,6 +138,7 @@ function unwrapFencedBlock(text: string): string {
*/
function parseJsonlOutput(raw: string, validFrameIds: ReadonlySet<number>): KgEntity[] {
const entities: KgEntity[] = [];
const seenEntityFrames = new Set<string>();
const cleaned = unwrapFencedBlock(raw);
for (const line of cleaned.split('\n')) {
@@ -146,13 +147,16 @@ function parseJsonlOutput(raw: string, validFrameIds: ReadonlySet<number>): KgEn
let parsed: Record<string, unknown>;
try {
parsed = JSON.parse(trimmed) as Record<string, unknown>;
const value: unknown = JSON.parse(trimmed);
if (!value || typeof value !== 'object' || Array.isArray(value)) continue;
parsed = value as Record<string, unknown>;
} catch {
continue;
}
const frameId = Number(parsed.frame_id);
if (!Number.isFinite(frameId) || !validFrameIds.has(frameId)) continue;
const frameId = parsed.frame_id;
if (typeof frameId !== 'number' || !Number.isSafeInteger(frameId) ||
!validFrameIds.has(frameId)) continue;
const name = typeof parsed.name === 'string' ? parsed.name.trim() : '';
if (name.length < 2) continue;
@@ -163,12 +167,17 @@ function parseJsonlOutput(raw: string, validFrameIds: ReadonlySet<number>): KgEn
const rawType = typeof parsed.type === 'string' ? parsed.type.toLowerCase().trim() : '';
if (!(KG_ENTITY_TYPES as readonly string[]).includes(rawType)) continue;
const scan = scanForInjection(name, 'tool_output');
if (!scan.safe) {
log.warn('dropping extracted entity name with injection payload', { flags: scan.flags.join(',') });
const ingress = evaluateExternalMemoryIngress({ content: name });
if (ingress.action !== 'allow') {
log.warn('dropping extracted entity name with injection payload', {
flags: ingress.scan.flags.join(','),
});
continue;
}
const entityFrameKey = JSON.stringify([frameId, normalizeEntityName(name)]);
if (seenEntityFrames.has(entityFrameKey)) continue;
seenEntityFrames.add(entityFrameKey);
entities.push({ frameId, name, type: rawType as KgEntityType });
}
@@ -214,7 +223,14 @@ export interface WriteKgEntitiesResult {
function safeParseProps(raw: string | undefined | null): Record<string, unknown> {
if (!raw) return {};
try { return JSON.parse(raw) as Record<string, unknown>; } catch { return {}; }
try {
const value: unknown = JSON.parse(raw);
return value && typeof value === 'object' && !Array.isArray(value)
? value as Record<string, unknown>
: {};
} catch {
return {};
}
}
/**
@@ -231,37 +247,54 @@ export function writeKgEntities(
kg: KnowledgeGraph,
extraction: KgEntityExtraction,
): WriteKgEntitiesResult {
const result: WriteKgEntitiesResult = { created: 0, updated: 0 };
return kg.runInTransaction(() => {
const result: WriteKgEntitiesResult = { created: 0, updated: 0 };
for (const entity of extraction.entities) {
// Defense at the write seam (mirrors the cognify CLI): callers other than
// extractKgEntities may not have noise-filtered.
if (isNoiseName(entity.name)) continue;
if (normalizeEntityName(entity.name).length < 3) continue;
for (const entity of extraction.entities) {
if (!entity || typeof entity !== 'object') continue;
if (!Number.isSafeInteger(entity.frameId) || entity.frameId <= 0) continue;
const name = typeof entity.name === 'string' ? entity.name.trim() : '';
if (isNoiseName(name)) continue;
if (normalizeEntityName(name).length < 3) continue;
if (!(KG_ENTITY_TYPES as readonly string[]).includes(entity.type)) continue;
if (evaluateExternalMemoryIngress({ content: name }).action !== 'allow') continue;
const existing = kg.findEntityByName(entity.name);
if (existing) {
const existingProps = safeParseProps(existing.properties);
const seenCount = Number(existingProps.seen_count ?? 1) + 1;
kg.updateEntity(existing.id, {
properties: { ...existingProps, seen_count: seenCount },
});
kg.linkEntityToFrame(existing.id, entity.frameId);
result.updated++;
} else {
try {
const created = kg.createEntity(entity.type, entity.name, { seen_count: 1, source: 'cognify-llm' });
kg.linkEntityToFrame(created.id, entity.frameId);
result.created++;
} catch (e: unknown) {
// Ontology validation may reject — skip this entity, never abort the pass.
log.warn('createEntity rejected extracted entity', {
name: entity.name,
error: e instanceof Error ? e.message : String(e),
const existing = kg.findEntityByName(name);
if (existing) {
if (!kg.linkEntityToFrameStrict(existing.id, entity.frameId)) continue;
const existingProps = safeParseProps(existing.properties);
const previousSeenCount = Number(existingProps.seen_count ?? 1);
const seenCount = (Number.isFinite(previousSeenCount) && previousSeenCount >= 0
? previousSeenCount
: 1) + 1;
kg.updateEntity(existing.id, {
properties: { ...existingProps, seen_count: seenCount },
});
result.updated++;
} else {
let created: { id: number };
try {
created = kg.createEntity(entity.type, name, {
seen_count: 1,
source: 'cognify-llm',
});
} catch (error) {
if (error instanceof Error && error.message.startsWith('Validation failed:')) {
log.warn('createEntity rejected extracted entity', {
name,
error: error.message,
});
continue;
}
throw error;
}
if (!kg.linkEntityToFrameStrict(created.id, entity.frameId)) {
throw new Error(`KG writer failed to link entity ${created.id} to frame ${entity.frameId}`);
}
result.created++;
}
}
}
return result;
return result;
});
}

View File

@@ -16,7 +16,10 @@ import type {
} from './types.js';
import { CLASSIFY_PROMPT, EXTRACT_PROMPT, SYNTHESIZE_PROMPT } from './prompts.js';
import { dedup } from './dedup.js';
import { scanForInjection } from '../injection-scanner.js';
import {
evaluateExternalMemoryIngress,
projectExternalMemoryContent,
} from '../memory-ingress-guard.js';
import { createCoreLogger } from '../logger.js';
const log = createCoreLogger('harvest-pipeline');
@@ -102,26 +105,34 @@ export class HarvestPipeline {
const errors: string[] = [];
log.info('harvest pipeline starting', { source, itemCount: items.length, batchSize: this.batchSize, concurrency: this.concurrency });
// Pass 0: Injection scan — drop any item whose title or content carries a
// prompt-injection payload (role_override / prompt_extraction / instruction_injection).
// Pass 0: Injection scan — drop any item whose untrusted title or message
// text carries a prompt-injection payload. Structured conversation adapters
// synthesize item.content with trusted `user:` / `assistant:` labels; scan
// their original message text instead so those labels are not mistaken for
// attacker-supplied authority markers. Unstructured items still scan their
// complete content. The exact-serialization check prevents a partial
// messages projection from hiding extra attacker-controlled content.
// Harvest ingests UNTRUSTED external exports (ChatGPT/Claude/Gemini JSON dumps,
// Perplexity shares, URL fetches). A hostile file must not flow through to the
// LLM passes or into memory frames.
const originalCount = items.length;
items = items.filter((item) => {
// Scan title + first 4KB of content — enough to catch payloads hidden in either field.
// Using 'tool_output' context since imports are external data, weighted like tool output.
const probe = `${item.title ?? ''}\n${(item.content ?? '').slice(0, 4000)}`;
const scan = scanForInjection(probe, 'tool_output');
if (!scan.safe) {
const reason = scan.flags.join(',');
const untrustedContent = projectExternalMemoryContent({
content: item.content ?? '',
messages: item.messages,
parseMethod: item.metadata?.parseMethod,
});
const decision = evaluateExternalMemoryIngress({
title: item.title,
content: untrustedContent,
});
if (decision.action === 'block') {
log.warn('dropping harvest item with injection payload', {
itemId: item.id,
title: item.title?.slice(0, 80),
flags: scan.flags,
score: scan.score,
itemId: String(item.id).slice(0, 80),
flags: decision.scan.flags,
score: decision.scan.score,
});
errors.push(`Blocked item "${item.title?.slice(0, 40) ?? item.id}" — injection detected (${reason})`);
errors.push('Blocked imported item due to unsafe content.');
return false;
}
return true;

View File

@@ -33,7 +33,7 @@
import type { FrameStore } from '../mind/frames.js';
import type { UniversalImportItem } from './types.js';
import { HARVEST_FRAME_CONTENT_CAP } from './types.js';
import { scanForInjection } from '../injection-scanner.js';
import { evaluateExternalMemoryIngress } from '../memory-ingress-guard.js';
import { createCoreLogger } from '../logger.js';
const log = createCoreLogger('raw-turns');
@@ -44,6 +44,7 @@ export const MIND_RAWTURN_PREFIX = '[mind-rawturn';
/** Hard per-conversation cap — backstop against pathological exports.
* LoCoMo conversations run ~600 turns; 2000 leaves generous headroom. */
export const MAX_TURNS_PER_ITEM = 2000;
const MAX_INJECTION_DROP_LOGS = 8;
/** Env kill switch (checked by CALLERS, mirrored here for the recall lane). */
export const RAWDETAIL_KILL_SWITCH = 'WAGGLE_RAWDETAIL';
@@ -127,31 +128,38 @@ export function writeRawTurnFrames(
const itemTs = isIsoTimestamp(item.timestamp) ? item.timestamp : undefined;
let turn = 0;
let inspected = 0;
let suppressedInjectionLogs = 0;
for (const msg of messages) {
if (msg.role !== 'user' && msg.role !== 'assistant') continue;
if (inspected >= MAX_TURNS_PER_ITEM) {
result.capped = true;
break;
}
inspected++;
const text = (msg.text ?? '').trim();
if (text.length === 0) {
result.skippedEmpty++;
continue;
}
if (turn >= MAX_TURNS_PER_ITEM) {
result.capped = true;
break;
}
// Scan first 4KB — same probe budget as the harvest pipeline's Pass 0.
const scan = scanForInjection(text.slice(0, 4000), 'tool_output');
if (!scan.safe) {
const storedText = text.slice(0, HARVEST_FRAME_CONTENT_CAP);
const decision = evaluateExternalMemoryIngress({ content: storedText });
if (decision.action === 'block') {
result.injectionDropped++;
log.warn('dropping raw turn with injection payload', {
conv: convKey, turn, flags: scan.flags.join(','),
});
if (result.injectionDropped <= MAX_INJECTION_DROP_LOGS) {
log.warn('dropping raw turn with injection payload', {
conv: convKey, turn, flags: decision.scan.flags,
});
} else {
suppressedInjectionLogs++;
}
continue;
}
const speaker = sanitizeToken(msg.role, 24);
const createdAt = isIsoTimestamp(msg.timestamp) ? msg.timestamp : itemTs;
frames.createIFrame(
gopId,
`${rawTurnHeader(convKey, turn, speaker)}\n${text.slice(0, HARVEST_FRAME_CONTENT_CAP)}`,
`${rawTurnHeader(convKey, turn, speaker)}\n${storedText}`,
'normal',
'import',
createdAt,
@@ -160,9 +168,14 @@ export function writeRawTurnFrames(
turn++;
}
if (suppressedInjectionLogs > 0) {
log.warn('additional raw-turn injection warnings suppressed', {
conv: convKey, suppressed: suppressedInjectionLogs,
});
}
if (result.capped) {
log.warn('raw-turn storage capped — conversation exceeds MAX_TURNS_PER_ITEM', {
conv: convKey, stored: result.written, totalMessages: messages.length,
conv: convKey, inspected, stored: result.written, totalMessages: messages.length,
});
}
return result;

View File

@@ -7,8 +7,8 @@
* internal services on the cloud/TEAMS deploy (sidecar binds 0.0.0.0) and can
* exfiltrate instance-metadata IAM credentials.
*
* This module is a self-contained, dependency-free (node builtins only) copy of
* the guard spec shared with `packages/agent/src/url-egress-guard.ts`. It lives
* This module is a self-contained copy of the guard spec shared with
* `packages/agent/src/url-egress-guard.ts`. It lives
* here — rather than importing from @waggle/agent — because hive-mind-core is
* OSS-mirrored and must not depend on Waggle-proprietary packages. Keep the two
* implementations in sync; they share one spec.
@@ -19,7 +19,8 @@
*/
import { lookup as dnsLookup } from 'node:dns/promises';
import { isIP } from 'node:net';
import { isIP, type LookupFunction } from 'node:net';
import { Agent } from 'undici';
export type AddressClass =
| 'public'
@@ -85,6 +86,7 @@ function classifyIpv4(ip: string): AddressClass {
if (a >= 240) return 'reserved'; // 240.0.0.0/4 + 255.255.255.255
if (a === 192 && b === 0 && c === 0) return 'reserved'; // 192.0.0.0/24
if (a === 192 && b === 0 && c === 2) return 'reserved'; // TEST-NET-1
if (a === 192 && b === 88 && c === 99) return 'reserved'; // Deprecated 6to4 relay anycast
if (a === 198 && (b === 18 || b === 19)) return 'reserved'; // 198.18.0.0/15
if (a === 198 && b === 51 && c === 100) return 'reserved'; // TEST-NET-2
if (a === 203 && b === 0 && c === 113) return 'reserved'; // TEST-NET-3
@@ -142,11 +144,20 @@ function classifyIpv6(ip: string): AddressClass {
}
if ((h[0] & 0xffc0) === 0xfe80) return 'link-local'; // fe80::/10
if ((h[0] & 0xffc0) === 0xfec0) return 'reserved'; // fec0::/10 deprecated site-local
if ((h[0] & 0xfe00) === 0xfc00) return 'unique-local'; // fc00::/7 (ULA)
if ((h[0] & 0xff00) === 0xff00) return 'multicast'; // ff00::/8
if (h[0] === 0x2001 && h[1] === 0x0db8) return 'reserved'; // 2001:db8::/32 docs
if (h[0] === 0x0064 && h[1] === 0xff9b) return 'reserved'; // 64:ff9b::/96 NAT64
if (
h[0] === 0x0064 && h[1] === 0xff9b
&& ((h[2] === 0 && h[3] === 0 && h[4] === 0 && h[5] === 0) || h[2] === 1)
) return 'reserved'; // 64:ff9b::/96 and 64:ff9b:1::/48 translation prefixes
if (h[0] === 0x0100 && h[1] === 0 && h[2] === 0 && h[3] === 0) return 'reserved'; // 100::/64 discard
if (h[0] === 0x0100 && h[1] === 0 && h[2] === 0 && h[3] === 1) return 'reserved'; // 100:0:0:1::/64 dummy
if (h[0] === 0x2001 && h[1] === 2 && h[2] === 0) return 'reserved'; // 2001:2::/48 benchmark
if (h[0] === 0x2002) return 'reserved'; // 2002::/16 deprecated 6to4
if (h[0] === 0x3fff && (h[1] & 0xf000) === 0) return 'reserved'; // 3fff::/20 docs
if (h[0] === 0x5f00) return 'reserved'; // 5f00::/16 SRv6 SIDs
return 'public';
}
@@ -169,6 +180,120 @@ async function defaultLookup(hostname: string): Promise<ResolvedAddress[]> {
return results.map((r) => ({ address: r.address, family: r.family }));
}
async function resolveHostname(
hostname: string,
rawUrl: string,
lookupFn: LookupFn,
): Promise<ResolvedAddress[]> {
try {
const addresses = await lookupFn(hostname);
if (!addresses || addresses.length === 0) {
throw new EgressBlockedError(
`DNS resolution returned no addresses for "${hostname}"`,
rawUrl,
);
}
return addresses;
} catch (err) {
if (err instanceof EgressBlockedError) throw err;
const detail = err instanceof Error ? err.message : String(err);
throw new EgressBlockedError(
`DNS resolution failed for "${hostname}": ${detail}`,
rawUrl,
);
}
}
function validateResolvedAddresses(
addresses: ResolvedAddress[],
hostname: string,
rawUrl: string,
allowLocal: boolean,
): void {
for (const { address } of addresses) {
const cls = classifyAddress(address);
if (!isAllowed(cls, allowLocal)) {
throw new EgressBlockedError(
`Blocked egress to ${cls} address ${address} (host "${hostname}")`,
rawUrl,
cls,
);
}
}
}
function createGuardedLookup(
allowLocal: boolean,
lookupFn: LookupFn,
): LookupFunction {
return (hostname, options, callback) => {
void resolveHostname(hostname, hostname, lookupFn)
.then((addresses) => {
validateResolvedAddresses(addresses, hostname, hostname, allowLocal);
const requestedFamily = options.family === 4 || options.family === 'IPv4'
? 4
: options.family === 6 || options.family === 'IPv6'
? 6
: 0;
const candidates = requestedFamily === 0
? addresses
: addresses.filter(({ family }) => family === requestedFamily);
if (candidates.length === 0) {
throw new EgressBlockedError(
`DNS resolution returned no IPv${requestedFamily} addresses for "${hostname}"`,
hostname,
);
}
if (options.all) {
callback(null, candidates);
} else {
const selected = candidates[0];
callback(null, selected.address, selected.family);
}
})
.catch((err: unknown) => {
callback(err as NodeJS.ErrnoException, '');
});
};
}
function createGuardedAgent(allowLocal: boolean, lookupFn: LookupFn): Agent {
return new Agent({
autoSelectFamily: true,
connect: { lookup: createGuardedLookup(allowLocal, lookupFn) },
});
}
const defaultGuardedAgents = new Map<boolean, Agent>();
function getDefaultGuardedAgent(allowLocal: boolean): Agent {
const existing = defaultGuardedAgents.get(allowLocal);
if (existing) return existing;
const agent = createGuardedAgent(allowLocal, defaultLookup);
defaultGuardedAgents.set(allowLocal, agent);
return agent;
}
function findEgressBlockedError(
error: unknown,
seen = new Set<unknown>(),
): EgressBlockedError | null {
if (error instanceof EgressBlockedError) return error;
if (typeof error !== 'object' || error === null || seen.has(error)) return null;
seen.add(error);
if (error instanceof AggregateError) {
for (const nested of error.errors) {
const blocked = findEgressBlockedError(nested, seen);
if (blocked) return blocked;
}
}
return findEgressBlockedError((error as { cause?: unknown }).cause, seen);
}
/**
* Validate that `rawUrl` is an http(s) URL whose host resolves only to public
* addresses. Throws {@link EgressBlockedError} otherwise. Returns parsed URL.
@@ -191,6 +316,10 @@ export async function assertUrlAllowed(
);
}
if (parsed.username || parsed.password) {
throw new EgressBlockedError('Blocked URL credentials', rawUrl);
}
// url.hostname keeps the surrounding brackets on an IPv6 literal ("[::1]"),
// which isIP() does not recognize — strip them so the literal is classified
// directly (loopback/private/link-local/…) instead of falling through to a DNS
@@ -204,34 +333,11 @@ export async function assertUrlAllowed(
addresses = [{ address: hostname, family: literalFamily }];
} else {
const lookupFn = options.lookup ?? defaultLookup;
try {
addresses = await lookupFn(hostname);
} catch (err) {
const detail = err instanceof Error ? err.message : String(err);
throw new EgressBlockedError(
`DNS resolution failed for "${hostname}": ${detail}`,
rawUrl,
);
}
if (!addresses || addresses.length === 0) {
throw new EgressBlockedError(
`DNS resolution returned no addresses for "${hostname}"`,
rawUrl,
);
}
addresses = await resolveHostname(hostname, rawUrl, lookupFn);
}
const allowLocal = options.allowLocal ?? false;
for (const { address } of addresses) {
const cls = classifyAddress(address);
if (!isAllowed(cls, allowLocal)) {
throw new EgressBlockedError(
`Blocked egress to ${cls} address ${address} (host "${hostname}")`,
rawUrl,
cls,
);
}
}
validateResolvedAddresses(addresses, hostname, rawUrl, allowLocal);
return parsed;
}
@@ -239,29 +345,119 @@ export async function assertUrlAllowed(
export interface SafeFetchOptions extends EgressGuardOptions {
/** Maximum redirect hops to follow (default 5). */
maxRedirects?: number;
/** Injectable fetch (tests). Defaults to globalThis.fetch. */
fetchImpl?: typeof globalThis.fetch;
}
type FetchWithDispatcher = (
input: string | URL | Request,
init: RequestInit & { dispatcher: Agent },
) => Promise<Response>;
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
const CROSS_ORIGIN_SECRET_HEADERS = [
'authorization',
'proxy-authorization',
'cookie',
'cookie2',
'x-api-key',
'api-key',
] as const;
const REQUEST_BODY_HEADERS = [
'content-encoding',
'content-language',
'content-length',
'content-location',
'content-type',
] as const;
function isNonReplayableBody(body: BodyInit): boolean {
const candidate = body as unknown as {
getReader?: unknown;
pipe?: unknown;
[Symbol.asyncIterator]?: unknown;
};
return typeof candidate.getReader === 'function'
|| typeof candidate.pipe === 'function'
|| typeof candidate[Symbol.asyncIterator] === 'function';
}
function redirectRequestInit(
init: RequestInit,
status: number,
fromUrl: URL,
toUrl: URL,
): RequestInit {
const next = { ...init };
const method = (next.method ?? 'GET').toUpperCase();
const rewriteToGet = ((status === 301 || status === 302) && method === 'POST')
|| (status === 303 && method !== 'GET' && method !== 'HEAD');
const headersToDelete = new Set<string>(['host']);
if (rewriteToGet) {
next.method = 'GET';
delete next.body;
for (const name of REQUEST_BODY_HEADERS) headersToDelete.add(name);
} else if (next.body !== undefined && next.body !== null && isNonReplayableBody(next.body)) {
throw new TypeError('Cannot replay a streamed request body across a redirect');
}
if (fromUrl.origin !== toUrl.origin) {
for (const name of CROSS_ORIGIN_SECRET_HEADERS) headersToDelete.add(name);
}
const headers = new Headers(next.headers);
for (const name of headersToDelete) headers.delete(name);
next.headers = headers;
return next;
}
/**
* SSRF-safe fetch. Validates before the request and re-validates every redirect
* hop (`redirect: 'manual'`). Caller-supplied `redirect` in `init` is ignored.
* hop (`redirect: 'manual'`). Native fetch is mandatory; proxy transports need
* an equivalent pinned connector rather than a global dispatcher override.
* Caller-supplied `redirect` in `init` is ignored.
*/
export async function safeFetch(
rawUrl: string,
init: RequestInit = {},
options: SafeFetchOptions = {},
): Promise<Response> {
if ('fetchImpl' in options) {
throw new TypeError('safeFetch fetchImpl injection is not supported; socket pinning requires native fetch');
}
const maxRedirects = options.maxRedirects ?? 5;
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
let currentUrl = rawUrl;
let currentInit = { ...init };
for (let hop = 0; hop <= maxRedirects; hop++) {
await assertUrlAllowed(currentUrl, options);
const allowLocal = options.allowLocal ?? false;
const temporaryAgent = options.lookup !== undefined;
const dispatcher = temporaryAgent
? createGuardedAgent(allowLocal, options.lookup!)
: getDefaultGuardedAgent(allowLocal);
const response = await fetchImpl(currentUrl, { ...init, redirect: 'manual' });
let response: Response;
try {
response = await (globalThis.fetch as unknown as FetchWithDispatcher)(currentUrl, {
...currentInit,
redirect: 'manual',
dispatcher,
});
} catch (err) {
if (temporaryAgent) {
await dispatcher.close().catch(() => undefined);
}
const blocked = findEgressBlockedError(err);
if (blocked) {
throw new EgressBlockedError(blocked.message, currentUrl, blocked.addressClass);
}
throw err;
}
const isRedirect = response.status >= 300 && response.status < 400;
if (temporaryAgent) {
void dispatcher.close().catch(() => undefined);
}
const isRedirect = REDIRECT_STATUSES.has(response.status);
const location = isRedirect ? response.headers.get('location') : null;
if (!location) {
return response;
@@ -273,16 +469,22 @@ export async function safeFetch(
/* best-effort; ignore */
}
let nextUrl: string;
let nextUrl: URL;
try {
nextUrl = new URL(location, currentUrl).toString();
nextUrl = new URL(location, currentUrl);
} catch {
throw new EgressBlockedError(
`Invalid redirect target "${location}"`,
currentUrl,
);
}
currentUrl = nextUrl;
currentInit = redirectRequestInit(
currentInit,
response.status,
new URL(currentUrl),
nextUrl,
);
currentUrl = nextUrl.toString();
}
throw new EgressBlockedError(

View File

@@ -0,0 +1,258 @@
/**
* Minimal synchronous persistence path for short-lived IDE hooks.
*
* Deliberately imports only the SQLite-backed mind primitives. Hook latency
* must not depend on loading the full core barrel, probing an embedding
* provider, or starting an MCP server. FTS is committed synchronously;
* vector enrichment remains repairable through reconcileVecIndex().
*/
import {
existsSync,
lstatSync,
mkdirSync,
readFileSync,
realpathSync,
} from 'node:fs';
import { homedir } from 'node:os';
import { join, resolve, sep } from 'node:path';
import { evaluateExternalMemoryIngress } from './memory-ingress-guard.js';
import { MindDB } from './mind/db.js';
import {
FrameStore,
type FrameSource,
type Importance,
type MemoryFrame,
} from './mind/frames.js';
import { SessionStore } from './mind/sessions.js';
export interface SaveHookFrameOptions {
dataDir?: string;
workspace?: string;
content: string;
importance: Exclude<Importance, 'deprecated'>;
source: Exclude<FrameSource, 'import'>;
}
export interface SaveHookFrameResult {
id: string;
success: true;
workspace: string;
}
export interface RecallHookFramesOptions {
dataDir?: string;
workspace?: string;
limit?: number;
}
export interface HookMemoryHit {
id: number;
content: string;
importance: string;
source: string;
score: number;
created_at: string;
from: string;
}
const WORKSPACE_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{0,199}$/;
const IMPORTANCE_SCORE: Record<Exclude<Importance, 'deprecated'>, number> = {
critical: 1,
important: 0.85,
normal: 0.65,
temporary: 0.45,
};
const ALLOWED_IMPORTANCE = new Set(Object.keys(IMPORTANCE_SCORE));
const ALLOWED_SOURCE = new Set(['user_stated', 'tool_verified', 'agent_inferred', 'system']);
function resolveDataDir(override?: string): string {
if (override !== undefined && override.trim() === '') {
throw new Error('Hook data directory must not be blank');
}
const envDir = process.env.HIVE_MIND_DATA_DIR;
const configured = override
?? (envDir?.trim() ? envDir : join(homedir(), '.hive-mind'));
const expanded = configured.startsWith('~')
? join(homedir(), configured.slice(1))
: configured;
return resolve(expanded);
}
function normalizedPath(value: string): string {
return process.platform === 'win32' ? value.toLowerCase() : value;
}
function isContained(root: string, candidate: string): boolean {
const normalizedRoot = normalizedPath(root);
const normalizedCandidate = normalizedPath(candidate);
return normalizedCandidate === normalizedRoot
|| normalizedCandidate.startsWith(`${normalizedRoot}${sep}`);
}
function rejectLink(path: string, label: string): void {
if (lstatSync(path).isSymbolicLink()) throw new Error(`${label} must not be a link`);
}
function requireRegularFile(path: string, label: string): void {
const stat = lstatSync(path);
if (stat.isSymbolicLink()) throw new Error(`${label} must not be a link`);
if (!stat.isFile()) throw new Error(`${label} must be a regular file`);
if (stat.nlink !== 1) throw new Error(`${label} must not be a hard link`);
}
function hasRegularFileEntry(path: string, label: string): boolean {
const stat = lstatSync(path, { throwIfNoEntry: false });
if (stat === undefined) return false;
if (stat.isSymbolicLink()) throw new Error(`${label} must not be a link`);
if (!stat.isFile()) throw new Error(`${label} must be a regular file`);
if (stat.nlink !== 1) throw new Error(`${label} must not be a hard link`);
return true;
}
function requireSafeSqliteEntries(path: string, label: string): boolean {
const hasDatabase = hasRegularFileEntry(path, label);
for (const suffix of ['-wal', '-shm', '-journal']) {
hasRegularFileEntry(`${path}${suffix}`, `${label}${suffix}`);
}
return hasDatabase;
}
function resolveMind(options: { dataDir?: string; workspace?: string }): {
path: string;
workspace: string;
} {
const dataDir = resolveDataDir(options.dataDir);
if (options.workspace === undefined) {
mkdirSync(dataDir, { recursive: true });
const canonicalDataDir = realpathSync(dataDir);
const personalMind = join(canonicalDataDir, 'personal.mind');
if (requireSafeSqliteEntries(personalMind, 'Personal mind')) {
const canonicalMind = realpathSync(personalMind);
if (!isContained(canonicalDataDir, canonicalMind)) {
throw new Error('Personal mind escapes data directory');
}
return { path: canonicalMind, workspace: 'personal' };
}
return { path: personalMind, workspace: 'personal' };
}
const id = options.workspace;
if (!WORKSPACE_ID.test(id)) throw new Error(`Invalid workspace id: ${id}`);
const workspacesRoot = resolve(dataDir, 'workspaces');
const workspaceDir = resolve(workspacesRoot, id);
if (!workspaceDir.startsWith(`${workspacesRoot}${sep}`)) {
throw new Error(`Workspace path escapes data directory: ${id}`);
}
if (!existsSync(workspacesRoot) || !lstatSync(workspacesRoot).isDirectory()) {
throw new Error(`Workspace not found: ${id}`);
}
rejectLink(workspacesRoot, 'Workspaces directory');
if (!existsSync(workspaceDir) || !lstatSync(workspaceDir).isDirectory()) {
throw new Error(`Workspace not found: ${id}`);
}
rejectLink(workspaceDir, 'Workspace directory');
const canonicalDataDir = realpathSync(dataDir);
const canonicalRoot = realpathSync(workspacesRoot);
const canonicalWorkspace = realpathSync(workspaceDir);
if (!isContained(canonicalDataDir, canonicalRoot)
|| !isContained(canonicalRoot, canonicalWorkspace)) {
throw new Error(`Workspace path escapes data directory: ${id}`);
}
const configPath = join(canonicalWorkspace, 'workspace.json');
if (!existsSync(configPath)) {
throw new Error(`Workspace not found: ${id}`);
}
requireRegularFile(configPath, 'Workspace config');
const canonicalConfig = realpathSync(configPath);
if (!isContained(canonicalWorkspace, canonicalConfig)) {
throw new Error(`Workspace config escapes data directory: ${id}`);
}
let configuredId: unknown;
try {
configuredId = (JSON.parse(readFileSync(canonicalConfig, 'utf8')) as { id?: unknown }).id;
} catch {
throw new Error(`Workspace config is invalid: ${id}`);
}
if (configuredId !== id) throw new Error(`Workspace config id mismatch: ${id}`);
const mindPath = join(canonicalWorkspace, 'workspace.mind');
if (requireSafeSqliteEntries(mindPath, 'Workspace mind')) {
const canonicalMind = realpathSync(mindPath);
if (!isContained(canonicalWorkspace, canonicalMind)) {
throw new Error(`Workspace mind escapes data directory: ${id}`);
}
return { path: canonicalMind, workspace: id };
}
return { path: mindPath, workspace: id };
}
function assertSaveInput(options: SaveHookFrameOptions): void {
if (typeof options.content !== 'string') throw new Error('Hook frame content must be a string');
if (evaluateExternalMemoryIngress({ content: options.content }).action !== 'allow') {
throw new Error('Hook frame content was rejected because it is unsafe.');
}
if (!ALLOWED_IMPORTANCE.has(options.importance)) {
throw new Error(`Invalid hook frame importance: ${String(options.importance)}`);
}
if (!ALLOWED_SOURCE.has(options.source)) {
throw new Error(`Invalid hook frame source: ${String(options.source)}`);
}
}
export function saveHookFrame(options: SaveHookFrameOptions): SaveHookFrameResult {
assertSaveInput(options);
const target = resolveMind(options);
const db = new MindDB(target.path);
try {
const raw = db.getDatabase();
const save = raw.transaction(() => {
const sessions = new SessionStore(db);
const frames = new FrameStore(db);
const today = new Date().toISOString().slice(0, 10);
const session = sessions.ensure(`mcp:${today}`, undefined, `MCP session ${today}`);
return frames.createIFrame(
session.gop_id,
options.content,
options.importance,
options.source,
);
});
const frame = db.runWithBusyRetry(save);
return { id: String(frame.id), success: true, workspace: target.workspace };
} finally {
db.close();
}
}
export function recallHookFrames(options: RecallHookFramesOptions = {}): HookMemoryHit[] {
const target = resolveMind(options);
const requestedLimit = options.limit;
const limit = typeof requestedLimit === 'number' && Number.isFinite(requestedLimit)
? Math.min(100, Math.max(1, Math.floor(requestedLimit)))
: 20;
const db = new MindDB(target.path);
try {
const rows = db.getDatabase().prepare(`
SELECT * FROM memory_frames
WHERE importance != 'deprecated'
ORDER BY CASE importance
WHEN 'critical' THEN 4
WHEN 'important' THEN 3
WHEN 'normal' THEN 2
ELSE 1
END DESC, id DESC
LIMIT ?
`).all(limit) as MemoryFrame[];
return rows.map((frame, index) => ({
id: frame.id,
content: frame.content,
importance: frame.importance,
source: frame.source,
score: Math.max(0, (IMPORTANCE_SCORE[frame.importance as Exclude<Importance, 'deprecated'>] ?? 0) - index / 1000),
created_at: frame.created_at,
from: target.workspace === 'personal' ? 'personal' : `workspace:${target.workspace}`,
}));
} finally {
db.close();
}
}

View File

@@ -1,7 +1,8 @@
// @waggle/hive-mind-core — substrate package barrel.
//
// Distribution: Apache 2.0 OSS via `git subtree split` from waggle-os monorepo
// to marolinik/hive-mind. Apps/web + Waggle agent harness stay proprietary in monorepo.
// Distribution: Apache 2.0 OSS via a maintainer-curated forward-port from the
// waggle-os monorepo to marolinik/hive-mind. Raw subtree-split branches are
// inspection inputs only and must not be pushed as the public mirror.
//
// Contents: mind/ (substrate), harvest/ (ingestion pipeline), prompt-injection
// scanner, structured logger.
@@ -9,6 +10,13 @@
// ── Logger + injection scanner (utilities used by substrate + Waggle agent) ──
export { createCoreLogger, type CoreLogger } from './logger.js';
export { scanForInjection, type ScanResult } from './injection-scanner.js';
export {
evaluateExternalMemoryIngress,
projectExternalMemoryContent,
type ExternalMemoryIngressDecision,
type ExternalMemoryIngressInput,
type ExternalMemoryProjectionInput,
} from './memory-ingress-guard.js';
// ── mind/ — memory substrate (FrameStore, KnowledgeGraph, embedders, search, scoring) ──
export {
@@ -97,6 +105,7 @@ export {
applyConsolidation,
collectObservations,
getCurrentValues,
MAX_CONSOLIDATION_OBSERVATIONS,
} from './mind/supersede.js';
export type {
ConsolidationLlm,

View File

@@ -0,0 +1,751 @@
import { scanForInjection, type ScanResult } from './injection-scanner.js';
export interface ExternalMemoryIngressInput {
title?: string;
content: string;
}
export interface ExternalMemoryProjectionInput {
content: string;
messages?: unknown;
parseMethod?: unknown;
maxChars?: number;
}
export type ExternalMemoryIngressDecision =
| { action: 'allow'; scan: ScanResult }
| { action: 'block'; reason: 'prompt_injection'; scan: ScanResult };
// RawArchive supports one million characters per item. Keep the same explicit
// public-ingress budget, checked before concatenation, scanning, or normalization.
const MAX_EXTERNAL_MEMORY_INGRESS_CHARS = 1_000_000;
const NAMED_HTML_ENTITIES: Readonly<Record<string, string>> = Object.freeze({
af: '\u2061',
amp: '&',
applyfunction: '\u2061',
apos: "'",
colon: ':',
emsp: ' ',
ensp: ' ',
gt: '>',
hairsp: ' ',
ic: '\u2063',
invisiblecomma: '\u2063',
invisibletimes: '\u2062',
it: '\u2062',
lrm: '\u200e',
lt: '<',
negativemediumspace: '\u200b',
negativethickspace: '\u200b',
negativethinspace: '\u200b',
negativeverythinspace: '\u200b',
newline: '\n',
nbsp: ' ',
nobreak: '\u2060',
quot: '"',
rlm: '\u200f',
shy: '\u00ad',
tab: '\t',
thinsp: ' ',
zwj: '',
zwnj: '',
zwsp: '',
zerowidthspace: '\u200b',
});
type CanonicalMemoryMessage = {
role: 'user' | 'assistant' | 'system';
text: string;
};
/**
* Return only the attacker-controlled text represented by a stored adapter
* projection. Role prefixes may be omitted only when plain canonical messages
* exactly reproduce the full content and did not come from universal raw text.
*/
export function projectExternalMemoryContent(input: ExternalMemoryProjectionInput): string {
let cappedContent = '';
try {
const content = typeof input.content === 'string' ? input.content : '';
const maxChars = input.maxChars;
cappedContent = maxChars === undefined
|| !Number.isSafeInteger(maxChars)
|| maxChars < 0
? content
: content.slice(0, maxChars);
if (input.parseMethod === 'universal-text'
|| !Array.isArray(input.messages)
|| input.messages.length === 0) {
return cappedContent;
}
const messages: CanonicalMemoryMessage[] = [];
for (const candidate of input.messages) {
if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) {
return cappedContent;
}
const prototype = Object.getPrototypeOf(candidate);
if (prototype !== Object.prototype && prototype !== null) return cappedContent;
const roleDescriptor = Object.getOwnPropertyDescriptor(candidate, 'role');
const textDescriptor = Object.getOwnPropertyDescriptor(candidate, 'text');
if (!roleDescriptor || !('value' in roleDescriptor)
|| !textDescriptor || !('value' in textDescriptor)) {
return cappedContent;
}
const role = roleDescriptor.value as unknown;
const text = textDescriptor.value as unknown;
if ((role !== 'user' && role !== 'assistant' && role !== 'system')
|| typeof text !== 'string') {
return cappedContent;
}
messages.push({ role, text });
}
const serialized = messages
.map(message => `${message.role}: ${message.text}`)
.join('\n\n');
if (serialized !== content) return cappedContent;
const parts: string[] = [];
let cursor = 0;
let offset = 0;
for (const [index, message] of messages.entries()) {
if (index > 0) offset += 2;
const prefixStart = offset;
const prefixEnd = prefixStart + `${message.role}: `.length;
if (prefixStart >= cappedContent.length) break;
if (message.role === 'system') return cappedContent;
parts.push(cappedContent.slice(cursor, prefixStart));
cursor = Math.min(prefixEnd, cappedContent.length);
offset = prefixEnd + message.text.length;
}
parts.push(cappedContent.slice(cursor));
return parts.join('');
} catch {
return cappedContent;
}
}
function decodeHtmlEntities(value: string): string {
if (!value.includes('&')) return value;
return value
.replace(/&(?:amp;){2,}/gi, '&')
.replace(/&#(?:x([0-9a-f]{1,6})|([0-9]{1,7}));?/gi, (match, hex: string, decimal: string) => {
const codePoint = Number.parseInt(hex ?? decimal, hex ? 16 : 10);
if (!Number.isInteger(codePoint)
|| codePoint <= 0
|| codePoint > 0x10ffff
|| (codePoint >= 0xd800 && codePoint <= 0xdfff)) {
return match;
}
return String.fromCodePoint(codePoint);
})
.replace(/&([a-z][a-z0-9]+);/gi, (match, name: string) =>
NAMED_HTML_ENTITIES[name.toLowerCase()] ?? match)
.replace(
/&(amp|apos|colon|emsp|ensp|gt|hairsp|lt|newline|nbsp|quot|tab|thinsp|zwj|zwnj|zwsp)(?=[^a-z0-9;]|$)/gi,
(_match, name: string) => NAMED_HTML_ENTITIES[name.toLowerCase()],
);
}
function decodePercentEncoding(value: string): string {
if (!/[+%]/.test(value)) return value;
const withSpaces = value.replace(/\+/g, ' ');
return withSpaces.replace(/(?:%[0-9a-f]{2})+/gi, (run) => {
try {
return decodeURIComponent(run);
} catch {
const bytes = Uint8Array.from(
run.match(/[0-9a-f]{2}/gi) ?? [],
hex => Number.parseInt(hex, 16),
);
return new TextDecoder('utf-8', { fatal: false }).decode(bytes);
}
});
}
function decodeUnicodeEscapes(value: string): string {
if (!/\\u/i.test(value)) return value;
return value.replace(
/\\u(?:\{([0-9a-f]{1,6})\}|([0-9a-f]{4}))/gi,
(match, braced: string | undefined, fixed: string | undefined) => {
const codePoint = Number.parseInt(braced ?? fixed ?? '', 16);
if (!Number.isInteger(codePoint) || codePoint > 0x10ffff) return match;
if (braced !== undefined) {
if (codePoint >= 0xd800 && codePoint <= 0xdfff) return match;
return String.fromCodePoint(codePoint);
}
return String.fromCharCode(codePoint);
},
);
}
function decodeHexEscapes(value: string): string {
if (!/\\x/i.test(value)) return value;
return value.replace(/\\x([0-9a-f]{2})/gi, (_match, hex: string) =>
String.fromCharCode(Number.parseInt(hex, 16)));
}
const MIXED_SCRIPT_CONFUSABLES: Readonly<Record<string, string>> = Object.freeze({
'\u0391': 'A',
'\u0392': 'B',
'\u0395': 'E',
'\u0396': 'Z',
'\u0397': 'H',
'\u0399': 'I',
'\u039a': 'K',
'\u039c': 'M',
'\u039d': 'N',
'\u039f': 'O',
'\u03a1': 'P',
'\u03a4': 'T',
'\u03a5': 'Y',
'\u03a7': 'X',
'\u03b1': 'a',
'\u03b5': 'e',
'\u03b9': 'i',
'\u03bf': 'o',
'\u03c1': 'p',
'\u03c7': 'x',
'\u03f2': 'c',
'\u03f9': 'C',
'\u0405': 'S',
'\u0406': 'I',
'\u0408': 'J',
'\u0410': 'A',
'\u0412': 'B',
'\u0415': 'E',
'\u041a': 'K',
'\u041c': 'M',
'\u041d': 'H',
'\u041e': 'O',
'\u0420': 'P',
'\u0421': 'C',
'\u0422': 'T',
'\u0425': 'X',
'\u0430': 'a',
'\u0435': 'e',
'\u043e': 'o',
'\u0440': 'p',
'\u0441': 'c',
'\u0443': 'y',
'\u0445': 'x',
'\u0455': 's',
'\u0456': 'i',
'\u0458': 'j',
});
const MIXED_SCRIPT_CONFUSABLE_PATTERN = /[\u0391\u0392\u0395\u0396\u0397\u0399\u039a\u039c\u039d\u039f\u03a1\u03a4\u03a5\u03a7\u03b1\u03b5\u03b9\u03bf\u03c1\u03c7\u03f2\u03f9\u0405\u0406\u0408\u0410\u0412\u0415\u041a\u041c\u041d\u041e\u0420\u0421\u0422\u0425\u0430\u0435\u043e\u0440\u0441\u0443\u0445\u0455\u0456\u0458]/;
const MIXED_SCRIPT_CONFUSABLE_REPLACE_PATTERN = new RegExp(
MIXED_SCRIPT_CONFUSABLE_PATTERN.source,
'g',
);
function projectMixedScriptConfusables(value: string): string | undefined {
if (!MIXED_SCRIPT_CONFUSABLE_PATTERN.test(value)) return undefined;
let changed = false;
const projected = value.replace(/[\p{L}\p{M}]+/gu, (token) => {
if (!/[A-Za-z]/.test(token) || !MIXED_SCRIPT_CONFUSABLE_PATTERN.test(token)) return token;
changed = true;
return token.replace(
MIXED_SCRIPT_CONFUSABLE_REPLACE_PATTERN,
char => MIXED_SCRIPT_CONFUSABLES[char] ?? char,
);
});
return changed ? projected : undefined;
}
const BASE64_CANDIDATE_PATTERN = /(?:^|[^A-Za-z0-9+/_=-])([A-Za-z0-9+/_-]{24,}={0,2})(?=$|[^A-Za-z0-9+/_=-])/g;
const MAX_BASE64_CANDIDATES = 16;
const MAX_BASE64_CANDIDATE_CHARS = 262_144;
const MAX_BASE64_TOTAL_CHARS = 524_288;
const MAX_BASE64_DEPTH = 4;
type Base64Candidate = {
value: string;
start: number;
end: number;
};
function collectBase64Candidates(source: string, allowImplicitWrapped = false): {
candidates: Base64Candidate[];
complete: boolean;
} {
const wrappedBlocks: Base64Candidate[] = [];
const trimmed = source.trim();
const wrappedChunks = trimmed.split(/[ \t\r\n]+/);
const wrapWidth = wrappedChunks[0]?.length ?? 0;
if (allowImplicitWrapped
&& wrappedChunks.length > 1
&& wrapWidth >= 4
&& wrapWidth <= 76
&& wrapWidth % 4 === 0
&& wrappedChunks.every(chunk => /^[A-Za-z0-9+/_-]+={0,2}$/.test(chunk))
&& wrappedChunks.slice(0, -1).every(chunk => chunk.length === wrapWidth && !chunk.includes('='))
&& wrappedChunks.at(-1)!.length <= wrapWidth) {
const candidate = wrappedChunks.join('');
if (candidate.length >= 24) {
const start = source.length - source.trimStart().length;
wrappedBlocks.push({ value: candidate, start, end: start + trimmed.length });
}
}
const directivePattern = /(?:\bdecode\b[^\r\n:]{0,160}\bbase64\b|\bbase64\b[^\r\n:]{0,160}\bdecode\b)[^\r\n:]{0,160}:/gi;
for (const directive of source.matchAll(directivePattern)) {
const tailStart = (directive.index ?? 0) + directive[0].length;
const tail = source.slice(tailStart);
const wrapped = tail.match(/^[ \t\r\n]*([A-Za-z0-9+/_=-]+(?:[ \t\r\n]+[A-Za-z0-9+/_=-]+)*)/);
if (!wrapped) continue;
const captured = wrapped[1];
const start = tailStart + wrapped[0].indexOf(captured);
let candidate = '';
let end = start;
for (const chunk of captured.matchAll(/[A-Za-z0-9+/_=-]+/g)) {
candidate += chunk[0];
end = start + (chunk.index ?? 0) + chunk[0].length;
if (chunk[0].includes('=')) break;
}
if (candidate.length >= 24) wrappedBlocks.push({ value: candidate, start, end });
}
const candidates: Base64Candidate[] = [];
const contiguousPattern = new RegExp(BASE64_CANDIDATE_PATTERN.source, 'g');
for (const match of source.matchAll(contiguousPattern)) {
const value = match[1];
const start = (match.index ?? 0) + match[0].length - value.length;
const end = start + value.length;
if (wrappedBlocks.some(block => start >= block.start && end <= block.end)) continue;
candidates.push({ value, start, end });
}
candidates.push(...wrappedBlocks);
candidates.sort((left, right) => left.start - right.start || left.end - right.end);
return { candidates, complete: true };
}
function decodeBase64Text(candidate: string): string | undefined {
let normalized = candidate.replace(/-/g, '+').replace(/_/g, '/');
const remainder = normalized.length % 4;
if (remainder === 1) return undefined;
if (remainder > 0) normalized += '='.repeat(4 - remainder);
try {
const binary = atob(normalized);
const bytes = Uint8Array.from(binary, char => char.charCodeAt(0));
const decoded = new TextDecoder('utf-8').decode(bytes);
if (!decoded) return undefined;
let printable = 0;
let total = 0;
for (const char of decoded) {
total++;
const codePoint = char.codePointAt(0) ?? 0;
const isPrintable = char === '\n' || char === '\r' || char === '\t'
|| (codePoint >= 0x20 && codePoint !== 0x7f);
if (codePoint !== 0xfffd && isPrintable) {
printable++;
}
}
return total > 0 && printable / total >= 0.85 ? decoded : undefined;
} catch {
return undefined;
}
}
function addProjection(projections: Set<string>, value: string): void {
if (projections.has(value)) return;
projections.add(value);
const confusable = projectMixedScriptConfusables(value);
if (confusable !== undefined) projections.add(confusable);
}
const FORMAT_CHARACTER_PATTERN = /\p{Cf}/u;
const DEFAULT_IGNORABLE_PATTERN = /\p{Default_Ignorable_Code_Point}/u;
function isHiddenSeparator(char: string): boolean {
const codePoint = char.codePointAt(0) ?? 0;
if (codePoint <= 0x9f) {
return codePoint <= 0x08
|| codePoint === 0x0b
|| codePoint === 0x0c
|| (codePoint >= 0x0e && codePoint <= 0x1f)
|| codePoint >= 0x7f;
}
if (codePoint >= 0xd800 && codePoint <= 0xdfff) return true;
return FORMAT_CHARACTER_PATTERN.test(char) || DEFAULT_IGNORABLE_PATTERN.test(char);
}
function replaceHiddenSeparators(value: string, replacement: string): string {
let containsHidden = false;
for (const char of value) {
if (isHiddenSeparator(char)) {
containsHidden = true;
break;
}
}
if (!containsHidden) return value;
let projected = '';
for (const char of value) projected += isHiddenSeparator(char) ? replacement : char;
return projected;
}
function projectDelimitedWords(value: string): string | undefined {
const projected = value.replace(
/(\p{L})([\p{P}\p{S}\p{White_Space}]+)(?=\p{L})/gu,
(match, letter: string, separators: string) =>
/[\p{P}\p{S}]/u.test(separators) ? `${letter} ` : match,
);
return projected === value ? undefined : projected;
}
type HtmlTagBoundary =
| { kind: 'close'; index: number }
| { kind: 'nested'; index: number }
| { kind: 'eof'; index: number };
function findHtmlTagBoundary(value: string, start: number): HtmlTagBoundary {
let quote: '"' | "'" | undefined;
for (let index = start; index < value.length; index++) {
const char = value[index];
if (char === '<') return { kind: 'nested', index };
if (quote) {
if (char === quote) quote = undefined;
} else if (char === '"' || char === "'") {
quote = char;
} else if (char === '>') {
return { kind: 'close', index };
}
}
return { kind: 'eof', index: value.length };
}
function normalizeHtmlToken(value: string): string {
return value.replace(/[:_-]+/g, ' ');
}
function extractHtmlAttributeTokens(value: string, start: number, end: number): string {
const tokens: string[] = [];
let index = start;
while (index < end) {
while (index < end && /[\s/]/.test(value[index])) index++;
const nameStart = index;
while (index < end && !/[\s=/>]/.test(value[index])) index++;
if (index === nameStart) {
index++;
continue;
}
const name = normalizeHtmlToken(value.slice(nameStart, index));
while (index < end && /\s/.test(value[index])) index++;
if (value[index] !== '=') {
if (name) tokens.push(name);
continue;
}
index++;
while (index < end && /\s/.test(value[index])) index++;
const quote = value[index] === '"' || value[index] === "'"
? value[index]
: undefined;
if (quote) index++;
const valueStart = index;
if (quote) {
while (index < end && value[index] !== quote) index++;
} else {
while (index < end && !/[\s>]/.test(value[index])) index++;
}
if (index > valueStart) tokens.push(normalizeHtmlToken(value.slice(valueStart, index)));
if (quote && index < end) index++;
}
return tokens.join(' ');
}
function stripHtmlMarkup(value: string): {
rendered: string;
tagNames: string;
attributes: string;
lexical: string;
} {
if (!value.includes('<')) {
return { rendered: value, tagNames: value, attributes: value, lexical: value };
}
const rendered: string[] = [];
const tagNames: string[] = [];
const attributes: string[] = [];
const lexical: string[] = [];
for (let index = 0; index < value.length;) {
if (value.startsWith('<!--', index)) {
const commentEnd = value.indexOf('-->', index + 4);
if (commentEnd === -1) {
const visibleTail = value.slice(index + 4);
rendered.push(visibleTail);
tagNames.push(visibleTail);
attributes.push(visibleTail);
lexical.push(visibleTail);
break;
}
const commentText = value.slice(index + 4, commentEnd);
if (commentText) {
tagNames.push(' ', commentText, ' ');
attributes.push(' ', commentText, ' ');
lexical.push(' ', commentText, ' ');
}
index = commentEnd + 3;
continue;
}
if (value.startsWith('-->', index)) {
index += 3;
continue;
}
if (value[index] === '<' && /[!/A-Za-z?]/.test(value[index + 1] ?? '')) {
let tagNameEnd = index + 1;
if (value[tagNameEnd] === '/') tagNameEnd++;
const tagNameStart = tagNameEnd;
while (/[A-Za-z0-9:!_-]/.test(value[tagNameEnd] ?? '')) tagNameEnd++;
const tagName = normalizeHtmlToken(value.slice(tagNameStart, tagNameEnd));
const boundary = findHtmlTagBoundary(value, tagNameEnd);
if (boundary.kind === 'close') {
const attributeTokens = extractHtmlAttributeTokens(
value,
tagNameEnd,
boundary.index,
);
if (tagName) tagNames.push(' ', tagName, ' ');
if (attributeTokens) attributes.push(' ', attributeTokens, ' ');
if (tagName || attributeTokens) {
lexical.push(' ', tagName, ' ', attributeTokens, ' ');
}
index = boundary.index + 1;
continue;
}
const visibleTail = value.slice(tagNameEnd, boundary.index);
rendered.push(visibleTail);
if (tagName) tagNames.push(' ', tagName, ' ');
tagNames.push(visibleTail);
attributes.push(visibleTail);
if (tagName) lexical.push(' ', tagName, ' ');
lexical.push(visibleTail);
if (boundary.kind === 'eof') break;
index = boundary.index;
continue;
}
rendered.push(value[index]);
tagNames.push(value[index]);
attributes.push(value[index]);
lexical.push(value[index]);
index++;
}
return {
rendered: rendered.join(''),
tagNames: tagNames.join(''),
attributes: attributes.join(''),
lexical: lexical.join(''),
};
}
function normalizedIngressProjections(value: string): {
projections: string[];
complete: boolean;
};
function normalizedIngressProjections(value: string, includeBase64: boolean): {
projections: string[];
complete: boolean;
};
function normalizedIngressProjections(value: string, includeBase64 = true): {
projections: string[];
complete: boolean;
} {
const maxPasses = 64;
const maxWork = MAX_EXTERNAL_MEMORY_INGRESS_CHARS + 1;
const originalProjection = value.normalize('NFKC');
const normalizationStages = new Set([originalProjection]);
let decodedProjection = originalProjection;
let complete = false;
let work = 0;
for (let pass = 0; pass < maxPasses; pass++) {
work += decodedProjection.length;
if (work > maxWork) break;
const decodedText = decodeUnicodeEscapes(decodeHexEscapes(
decodeHtmlEntities(decodePercentEncoding(decodedProjection)),
));
const decoded = decodedText === decodedProjection
? decodedProjection
: decodedText.normalize('NFKC');
if (decoded === decodedProjection) {
complete = true;
break;
}
normalizationStages.add(decoded);
decodedProjection = decoded;
}
if (!complete) return { projections: [decodedProjection], complete: false };
const htmlProjections = stripHtmlMarkup(decodedProjection);
const projections = new Set<string>();
for (const projection of normalizationStages) addProjection(projections, projection);
for (const projection of new Set([
htmlProjections.rendered,
htmlProjections.tagNames,
htmlProjections.attributes,
htmlProjections.lexical,
])) {
let unformatted = projection;
if (unformatted.includes('[')) {
unformatted = unformatted
.replace(/!\[([^\]]*)\]\([^)]*\)/g, '$1')
.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
.replace(/\[([^\]]+)\]\[[^\]]*\]/g, '$1');
}
if (/[*_~`]/.test(unformatted)) unformatted = unformatted.replace(/[*_~`]/g, '');
const compact = replaceHiddenSeparators(unformatted, '');
addProjection(projections, compact);
if (compact !== unformatted) {
addProjection(projections, replaceHiddenSeparators(unformatted, ' '));
}
}
if (!includeBase64) return { projections: [...projections], complete: true };
const candidateLineages = new Map<number, Map<string, number[]>>();
let nextCandidateLineage = 1;
const resolveCandidateLineage = (
parentLineage: number,
candidate: string,
ordinal: number,
): { lineage: number; created: boolean } => {
let candidates = candidateLineages.get(parentLineage);
if (!candidates) {
candidates = new Map();
candidateLineages.set(parentLineage, candidates);
}
let lineages = candidates.get(candidate);
if (!lineages) {
lineages = [];
candidates.set(candidate, lineages);
}
const existing = lineages[ordinal];
if (existing !== undefined) return { lineage: existing, created: false };
const lineage = nextCandidateLineage++;
lineages[ordinal] = lineage;
return { lineage, created: true };
};
let decodedCandidateCount = 0;
let decodedCandidateChars = 0;
let base64Sources = [...projections].map(value => ({ value, lineage: 0 }));
let reachedDepthLimit = true;
for (let depth = 0; depth < MAX_BASE64_DEPTH; depth++) {
const decodedValues: Array<{ value: string; lineage: number }> = [];
for (const source of base64Sources) {
const collected = collectBase64Candidates(source.value, depth > 0);
if (!collected.complete) return { projections: [...projections], complete: false };
const occurrenceCounts = new Map<string, number>();
for (const occurrence of collected.candidates) {
const candidate = occurrence.value;
const ordinal = occurrenceCounts.get(candidate) ?? 0;
occurrenceCounts.set(candidate, ordinal + 1);
const occurrenceLineage = resolveCandidateLineage(
source.lineage,
candidate,
ordinal,
);
if (!occurrenceLineage.created) continue;
if (candidate.length > MAX_BASE64_CANDIDATE_CHARS) {
return { projections: [...projections], complete: false };
}
const decoded = decodeBase64Text(candidate);
if (decoded === undefined) continue;
decodedCandidateCount++;
decodedCandidateChars += candidate.length;
if (decodedCandidateCount > MAX_BASE64_CANDIDATES
|| decodedCandidateChars > MAX_BASE64_TOTAL_CHARS) {
return { projections: [...projections], complete: false };
}
decodedValues.push({ value: decoded, lineage: occurrenceLineage.lineage });
}
}
if (decodedValues.length === 0) {
reachedDepthLimit = false;
break;
}
const nextSources: Array<{ value: string; lineage: number }> = [];
for (const decoded of decodedValues) {
const nested = normalizedIngressProjections(decoded.value, false);
if (!nested.complete) return { projections: [...projections], complete: false };
for (const projection of nested.projections) {
addProjection(projections, projection);
nextSources.push({ value: projection, lineage: decoded.lineage });
}
}
base64Sources = nextSources;
}
if (reachedDepthLimit && base64Sources.length > 0) {
for (const source of base64Sources) {
const collected = collectBase64Candidates(source.value, true);
if (!collected.complete) return { projections: [...projections], complete: false };
const occurrenceCounts = new Map<string, number>();
for (const occurrence of collected.candidates) {
const candidate = occurrence.value;
const ordinal = occurrenceCounts.get(candidate) ?? 0;
occurrenceCounts.set(candidate, ordinal + 1);
const occurrenceSeen = candidateLineages
.get(source.lineage)
?.get(candidate)?.[ordinal] !== undefined;
if (!occurrenceSeen
&& decodeBase64Text(candidate) !== undefined) {
return { projections: [...projections], complete: false };
}
}
}
}
return { projections: [...projections], complete: true };
}
/** Evaluate untrusted content before it can enter persistent memory. */
export function evaluateExternalMemoryIngress(
input: ExternalMemoryIngressInput,
): ExternalMemoryIngressDecision {
const title = input.title ?? '';
if (title.length > MAX_EXTERNAL_MEMORY_INGRESS_CHARS
|| input.content.length > MAX_EXTERNAL_MEMORY_INGRESS_CHARS - title.length) {
return {
action: 'block',
reason: 'prompt_injection',
scan: { safe: false, score: 0.6, flags: ['normalization_limit'] },
};
}
const projection = `${title}\n${input.content}`;
let scan = scanForInjection(projection, 'tool_output');
if (scan.safe) {
const normalizedIngress = normalizedIngressProjections(projection);
for (const normalized of normalizedIngress.projections) {
if (normalized !== projection) {
const normalizedScan = scanForInjection(normalized, 'tool_output');
if (!normalizedScan.safe) {
scan = normalizedScan;
break;
}
}
const delimited = projectDelimitedWords(normalized);
if (delimited !== undefined) {
const delimitedScan = scanForInjection(delimited, 'tool_output');
if (!delimitedScan.safe) {
scan = delimitedScan;
break;
}
}
}
if (scan.safe && !normalizedIngress.complete) {
scan = { safe: false, score: 0.6, flags: ['normalization_limit'] };
}
}
return scan.safe
? { action: 'allow', scan }
: { action: 'block', reason: 'prompt_injection', scan };
}

View File

@@ -109,6 +109,7 @@ export interface StartTraceInput {
export interface FinalizeTraceInput {
outcome: TraceOutcome;
output: string;
model?: string | null;
reasoning?: TraceReasoningStep[];
toolCalls?: TraceToolCall[];
artifacts?: string[];
@@ -158,6 +159,30 @@ const EXECUTION_TRACES_DDL: string[] = [
`CREATE INDEX IF NOT EXISTS idx_traces_persona ON execution_traces (persona_id, outcome)`,
`CREATE INDEX IF NOT EXISTS idx_traces_outcome ON execution_traces (outcome, created_at DESC)`,
`CREATE INDEX IF NOT EXISTS idx_traces_workspace ON execution_traces (workspace_id, created_at DESC)`,
`CREATE TABLE IF NOT EXISTS execution_trace_spend (
id INTEGER PRIMARY KEY AUTOINCREMENT,
trace_id INTEGER NOT NULL REFERENCES execution_traces(id) ON DELETE CASCADE,
cost_usd REAL NOT NULL CHECK (cost_usd > 0),
settled_at TEXT NOT NULL
)`,
`CREATE INDEX IF NOT EXISTS idx_trace_spend_settled ON execution_trace_spend (settled_at, trace_id)`,
`CREATE TABLE IF NOT EXISTS execution_trace_spend_reservations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
trace_id INTEGER NOT NULL REFERENCES execution_traces(id) ON DELETE CASCADE,
estimated_cost_usd REAL NOT NULL CHECK (estimated_cost_usd > 0),
actual_cost_usd REAL CHECK (actual_cost_usd IS NULL OR actual_cost_usd >= 0),
state TEXT NOT NULL DEFAULT 'pending'
CHECK (state IN ('pending', 'settled', 'released')),
reserved_at TEXT NOT NULL,
resolved_at TEXT,
CHECK (
(state = 'pending' AND actual_cost_usd IS NULL AND resolved_at IS NULL) OR
(state = 'settled' AND actual_cost_usd IS NOT NULL AND resolved_at IS NOT NULL) OR
(state = 'released' AND actual_cost_usd IS NULL AND resolved_at IS NOT NULL)
)
)`,
`CREATE INDEX IF NOT EXISTS idx_trace_spend_reservations_time
ON execution_trace_spend_reservations (reserved_at, trace_id, state)`,
];
/** Exported DDL concatenated — kept for anyone who needs the full table SQL. */
@@ -175,10 +200,6 @@ export class ExecutionTraceStore {
private ensureTable(): void {
try {
const raw = this.db.getDatabase();
const exists = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='execution_traces'",
).get();
if (exists) return;
for (const stmt of EXECUTION_TRACES_DDL) {
raw.prepare(stmt).run();
}
@@ -249,6 +270,141 @@ export class ExecutionTraceStore {
.run(JSON.stringify(payload), id);
}
/** Persist one settled model charge while a long-running trace is pending. */
recordCost(id: number, costUsd: number, timestamp = new Date().toISOString()): void {
if (!Number.isFinite(costUsd) || costUsd <= 0) {
throw new RangeError('Trace cost entry must be a positive finite number');
}
const settledMs = Date.parse(timestamp);
if (!Number.isFinite(settledMs)) {
throw new RangeError('Trace cost timestamp must be a valid ISO date');
}
const settledAt = new Date(settledMs).toISOString();
const raw = this.db.getDatabase();
raw.transaction(() => {
const updated = raw.prepare(`
UPDATE execution_traces SET cost_usd = cost_usd + ? WHERE id = ?
`).run(costUsd, id);
if (updated.changes === 0) return;
raw.prepare(`
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
VALUES (?, ?, ?)
`).run(id, costUsd, settledAt);
})();
}
/**
* Persist a conservative model-spend reservation before provider dispatch.
* Pending reservations intentionally have no settled-spend row, so restart
* recovery counts the estimate through the reservation ledger.
*/
reserveCost(
id: number,
estimatedCostUsd: number,
timestamp = new Date().toISOString(),
): number {
if (!Number.isFinite(estimatedCostUsd) || estimatedCostUsd <= 0) {
throw new RangeError('Trace cost reservation must be positive and finite');
}
const reservedMs = Date.parse(timestamp);
if (!Number.isFinite(reservedMs)) {
throw new RangeError('Trace cost reservation timestamp must be valid ISO date');
}
const result = this.db.getDatabase().prepare(`
INSERT INTO execution_trace_spend_reservations
(trace_id, estimated_cost_usd, state, reserved_at)
SELECT id, ?, 'pending', ?
FROM execution_traces
WHERE id = ? AND outcome = 'pending'
`).run(estimatedCostUsd, new Date(reservedMs).toISOString(), id);
if (result.changes !== 1) {
throw new Error(`Pending execution trace ${id} does not exist`);
}
return Number(result.lastInsertRowid);
}
/** Replace a pending estimate with one authoritative settled charge. */
settleReservedCost(
reservationId: number,
actualCostUsd: number,
timestamp = new Date().toISOString(),
): boolean {
if (!Number.isFinite(actualCostUsd) || actualCostUsd < 0) {
throw new RangeError('Settled trace cost must be non-negative and finite');
}
const settledMs = Date.parse(timestamp);
if (!Number.isFinite(settledMs)) {
throw new RangeError('Trace cost timestamp must be valid ISO date');
}
const settledAt = new Date(settledMs).toISOString();
const raw = this.db.getDatabase();
return raw.transaction(() => {
const current = raw.prepare(`
SELECT trace_id AS traceId, reserved_at AS reservedAt,
state, actual_cost_usd AS actualCostUsd
FROM execution_trace_spend_reservations
WHERE id = ?
`).get(reservationId) as {
traceId: number;
reservedAt: string;
state: 'pending' | 'settled' | 'released';
actualCostUsd: number | null;
} | undefined;
if (!current) throw new Error(`Cost reservation ${reservationId} does not exist`);
if (current.state === 'settled' && current.actualCostUsd === actualCostUsd) return false;
if (current.state !== 'pending') {
throw new Error(`Cost reservation ${reservationId} is already ${current.state}`);
}
const updated = raw.prepare(`
UPDATE execution_trace_spend_reservations
SET state = 'settled', actual_cost_usd = ?, resolved_at = ?
WHERE id = ? AND state = 'pending'
`).run(actualCostUsd, settledAt, reservationId);
if (updated.changes !== 1) return false;
if (actualCostUsd > 0) {
raw.prepare(`
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
VALUES (?, ?, ?)
`).run(current.traceId, actualCostUsd, current.reservedAt);
raw.prepare(`
UPDATE execution_traces SET cost_usd = cost_usd + ? WHERE id = ?
`).run(actualCostUsd, current.traceId);
}
return true;
})();
}
/** Release a definitely pre-inference reservation without recording spend. */
releaseReservedCost(reservationId: number, timestamp = new Date().toISOString()): boolean {
const resolvedMs = Date.parse(timestamp);
if (!Number.isFinite(resolvedMs)) {
throw new RangeError('Trace cost timestamp must be valid ISO date');
}
const raw = this.db.getDatabase();
return raw.transaction(() => {
const current = raw.prepare(`
SELECT id, state
FROM execution_trace_spend_reservations
WHERE id = ?
`).get(reservationId) as {
id: number;
state: 'pending' | 'settled' | 'released';
} | undefined;
if (!current) throw new Error(`Cost reservation ${reservationId} does not exist`);
if (current.state === 'released') return false;
if (current.state !== 'pending') {
throw new Error(`Cost reservation ${reservationId} is already ${current.state}`);
}
const result = raw.prepare(`
UPDATE execution_trace_spend_reservations
SET state = 'released', actual_cost_usd = NULL, resolved_at = ?
WHERE id = ? AND state = 'pending'
`).run(new Date(resolvedMs).toISOString(), reservationId);
if (result.changes !== 1) return false;
return true;
})();
}
/** Finalize a trace — set outcome, merge payload, record cost + duration. */
finalize(id: number, input: FinalizeTraceInput): ExecutionTrace | undefined {
const current = this.get(id);
@@ -270,22 +426,42 @@ export class ExecutionTraceStore {
const createdMs = Date.parse(current.created_at + 'Z');
const now = Date.now();
const durationMs = Number.isFinite(createdMs) ? Math.max(0, now - createdMs) : 0;
const finalModel = input.model === undefined ? current.model : input.model;
this.db.getDatabase().prepare(`
UPDATE execution_traces
SET outcome = ?,
trace_json = ?,
cost_usd = ?,
duration_ms = ?,
finalized_at = datetime('now')
WHERE id = ?
`).run(
input.outcome,
JSON.stringify(merged),
input.costUsd ?? current.cost_usd,
durationMs,
id,
);
const raw = this.db.getDatabase();
raw.transaction(() => {
const finalCost = Math.max(current.cost_usd, input.costUsd ?? current.cost_usd);
if (input.costUsd !== undefined && finalCost > 0) {
const ledger = raw.prepare(`
SELECT COUNT(*) AS entries, COALESCE(SUM(cost_usd), 0) AS total
FROM execution_trace_spend WHERE trace_id = ?
`).get(id) as { entries: number; total: number | null };
const missingCost = Math.max(0, finalCost - Number(ledger.total ?? 0));
if (ledger.entries > 0 && missingCost > 0) {
raw.prepare(`
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
VALUES (?, ?, ?)
`).run(id, missingCost, new Date().toISOString());
}
}
raw.prepare(`
UPDATE execution_traces
SET outcome = ?,
model = ?,
trace_json = ?,
cost_usd = ?,
duration_ms = ?,
finalized_at = datetime('now')
WHERE id = ?
`).run(
input.outcome,
finalModel,
JSON.stringify(merged),
finalCost,
durationMs,
id,
);
})();
return this.get(id);
}
@@ -320,6 +496,42 @@ export class ExecutionTraceStore {
return row ? toParsed(row) : undefined;
}
/** Highest trace id present when a consumer starts its process-local ledger. */
getLatestId(): number {
const row = this.db.getDatabase().prepare(
'SELECT COALESCE(MAX(id), 0) AS id FROM execution_traces',
).get() as { id: number | null };
return Number(row.id ?? 0);
}
/** Sum persisted model cost from a timestamp through an inclusive trace-id boundary. */
getTotalCostSince(since: string, throughId: number = Number.MAX_SAFE_INTEGER): number {
const settledSince = new Date(since).toISOString();
const raw = this.db.getDatabase();
const ledger = raw.prepare(`
SELECT COALESCE(SUM(s.cost_usd), 0) AS total
FROM execution_trace_spend s
JOIN execution_traces t ON t.id = s.trace_id
WHERE s.settled_at >= ? AND t.id <= ?
`).get(settledSince, throughId) as { total: number | null };
const pending = raw.prepare(`
SELECT COALESCE(SUM(estimated_cost_usd), 0) AS total
FROM execution_trace_spend_reservations
WHERE state = 'pending' AND reserved_at >= ? AND trace_id <= ?
`).get(settledSince, throughId) as { total: number | null };
const legacy = raw.prepare(`
SELECT COALESCE(SUM(t.cost_usd), 0) AS total
FROM execution_traces t
WHERE t.created_at >= datetime(?) AND t.id <= ?
AND NOT EXISTS (
SELECT 1 FROM execution_trace_spend s WHERE s.trace_id = t.id
)
`).get(since, throughId) as { total: number | null };
return Number(ledger.total ?? 0)
+ Number(pending.total ?? 0)
+ Number(legacy.total ?? 0);
}
/** Query traces with optional filters. */
query(filter: TraceQueryFilter = {}): ExecutionTrace[] {
const clauses: string[] = [];

View File

@@ -70,6 +70,22 @@ export class FrameStore {
this.db = db;
}
/**
* Run a write unit atomically. Top-level callers acquire the write lock up
* front and retry the whole closure on transient cross-process contention.
* Nested callers use better-sqlite3's savepoint behavior and never retry an
* inner closure against the same ambient snapshot.
*/
runInTransaction<T>(fn: () => T): T {
if (typeof fn !== 'function') {
throw new TypeError('FrameStore.runInTransaction requires a function');
}
const raw = this.db.getDatabase();
const transaction = raw.transaction(fn);
if (raw.inTransaction) return transaction();
return this.db.runWithBusyRetry(() => transaction.immediate());
}
createIFrame(
gopId: string,
content: string,
@@ -154,6 +170,12 @@ export class FrameStore {
return this.db.getDatabase().prepare('SELECT * FROM memory_frames WHERE id = ?').get(id) as MemoryFrame | undefined;
}
hasSession(gopId: string): boolean {
return this.db.getDatabase().prepare(
'SELECT 1 FROM sessions WHERE gop_id = ? LIMIT 1',
).get(gopId) !== undefined;
}
getLatestIFrame(gopId: string): MemoryFrame | undefined {
return this.db.getDatabase().prepare(`
SELECT * FROM memory_frames
@@ -287,6 +309,13 @@ export class FrameStore {
if (!existing) return undefined;
const newImportance = importance ?? existing.importance;
if (content === existing.content) {
if (newImportance !== existing.importance) {
raw.prepare('UPDATE memory_frames SET importance = ? WHERE id = ?')
.run(newImportance, id);
}
return this.getById(id);
}
// Update main table (content_hash maintained — oss-drift D3)
raw.prepare(`

View File

@@ -2,13 +2,14 @@
* In-process embedder using @huggingface/transformers (ONNX Runtime).
* Default provider for ALL desktop users — zero config, works offline.
* Model: Xenova/all-MiniLM-L6-v2 (384 native dims, normalized to target dims).
* Downloads ~23MB model on first use, cached in ~/.waggle/models/.
* Downloads ~90MB fp32 model on first use, cached in ~/.waggle/models/.
*/
import path from 'node:path';
import os from 'node:os';
import type { Embedder } from './embeddings.js';
import { createCoreLogger } from '../logger.js';
import { withTransformersModelLoad } from './transformers-model-load.js';
const log = createCoreLogger('inprocess-embedder');
@@ -32,13 +33,18 @@ export async function createInProcessEmbedder(config?: Partial<InProcessEmbedder
const cacheDir = config?.cacheDir ?? path.join(os.homedir(), '.waggle', 'models');
const targetDims = config?.targetDimensions ?? 1024;
log.info(`Loading in-process embedding model: ${model} (~23MB first download)`);
log.info(`Loading in-process embedding model: ${model} (~90MB fp32 first download)`);
const { pipeline, env } = await import('@huggingface/transformers');
env.cacheDir = cacheDir;
env.allowRemoteModels = true;
const extractor = await pipeline('feature-extraction', model, { dtype: 'fp32' });
const { pipeline } = await import('@huggingface/transformers');
const extractor = await withTransformersModelLoad({
cacheDir,
model,
load: (canonicalCacheDir) => pipeline('feature-extraction', model, {
dtype: 'fp32',
cache_dir: canonicalCacheDir,
}),
onQuarantine: () => log.warn(`Quarantined corrupt embedding model cache: ${model}`),
});
const nativeDims = 384; // all-MiniLM-L6-v2 output dimensions
log.info(`In-process embedder ready (${nativeDims} native dims → ${targetDims} normalized)`);

View File

@@ -21,6 +21,7 @@
import path from 'node:path';
import os from 'node:os';
import { createCoreLogger } from '../logger.js';
import { withTransformersModelLoad } from './transformers-model-load.js';
const log = createCoreLogger('inprocess-reranker');
@@ -56,18 +57,25 @@ export async function createInProcessReranker(
log.info(`Loading in-process reranker: ${model} (~22MB first download)`);
const { AutoTokenizer, AutoModelForSequenceClassification, env } = await import(
const { AutoTokenizer, AutoModelForSequenceClassification } = await import(
'@huggingface/transformers'
);
env.cacheDir = cacheDir;
env.allowRemoteModels = true;
// Cross-encoders need direct tokenizer + model access — pipeline API
// doesn't expose the (text, text_pair) input pattern cleanly across
// all transformers.js versions. Calling the model directly with
// tokenized pairs is the stable path.
const tokenizer = await AutoTokenizer.from_pretrained(model);
const seqModel = await AutoModelForSequenceClassification.from_pretrained(model, { dtype: 'fp32' });
const { tokenizer, seqModel } = await withTransformersModelLoad({
cacheDir,
model,
load: async (canonicalCacheDir) => ({
// Cross-encoders need direct tokenizer + model access — pipeline API
// doesn't expose the (text, text_pair) input pattern cleanly across
// all transformers.js versions. Calling the model directly with
// tokenized pairs is the stable path.
tokenizer: await AutoTokenizer.from_pretrained(model, { cache_dir: canonicalCacheDir }),
seqModel: await AutoModelForSequenceClassification.from_pretrained(model, {
dtype: 'fp32',
cache_dir: canonicalCacheDir,
}),
}),
onQuarantine: () => log.warn(`Quarantined corrupt reranker model cache: ${model}`),
});
log.info(`In-process reranker ready: ${model}`);
@@ -77,7 +85,7 @@ export async function createInProcessReranker(
text_pair: doc,
padding: true,
truncation: true,
return_tensors: 'pt',
return_tensor: true,
});
const out = await seqModel(inputs);
// ms-marco-MiniLM outputs a single logit per pair (1-class regression).
@@ -105,7 +113,7 @@ export async function createInProcessReranker(
text_pair: docs,
padding: true,
truncation: true,
return_tensors: 'pt',
return_tensor: true,
});
const out = await seqModel(inputs);
const logits = out.logits ?? out[0];

View File

@@ -59,6 +59,17 @@ export class KnowledgeGraph {
this.db = db;
}
/** Run a graph write unit atomically, nesting as a savepoint when needed. */
runInTransaction<T>(fn: () => T): T {
if (typeof fn !== 'function') {
throw new TypeError('KnowledgeGraph.runInTransaction requires a function');
}
const raw = this.db.getDatabase();
const transaction = raw.transaction(fn);
if (raw.inTransaction) return transaction();
return this.db.runWithBusyRetry(() => transaction.immediate());
}
setValidationSchema(schema: ValidationSchema): void {
this.schema = schema;
}
@@ -381,11 +392,16 @@ export class KnowledgeGraph {
/** Link an entity to a frame it was extracted from (kg_entity_frames bridge).
* Powers the 'contextual' scoring signal. Idempotent per (entity, frame). */
linkEntityToFrame(entityId: number, frameId: number): void {
try {
this.db.getDatabase().prepare(
'INSERT OR IGNORE INTO kg_entity_frames (entity_id, frame_id) VALUES (?, ?)'
).run(entityId, frameId);
} catch { /* bridge table absent on a pre-migration DB — best-effort */ }
try { this.linkEntityToFrameStrict(entityId, frameId); } catch {
/* bridge table absent on a pre-migration DB — best-effort */
}
}
/** Strict provenance link for atomic writers; true only for a new link. */
linkEntityToFrameStrict(entityId: number, frameId: number): boolean {
return this.db.getDatabase().prepare(
'INSERT OR IGNORE INTO kg_entity_frames (entity_id, frame_id) VALUES (?, ?)'
).run(entityId, frameId).changes === 1;
}
/** Seed entities whose name appears in free text (case-insensitive, name ≥3

View File

@@ -123,7 +123,7 @@ export async function fetchRawDetailLane(
const excludeIds = opts.excludeIds ?? new Set<number>();
// ── Pool ──────────────────────────────────────────────────────────────
let pool: FrameRow[] = [];
let pool: FrameRow[];
if (opts.window) {
pool = windowPool(db, opts.window.since, opts.window.until);
if (pool.length > WINDOW_POOL_MAX) {

View File

@@ -3,7 +3,7 @@ import type { Embedder } from './embeddings.js';
import type { MemoryFrame, Importance } from './frames.js';
import type { Reranker } from './inprocess-reranker.js';
import { chunkText, type ChunkOptions } from './chunker.js';
import { buildFtsOrQuery, hasUnsegmentedScript, sanitizeFtsToken } from './fts-sanitize.js';
import { buildFtsOrQuery, FTS_STOP_WORDS, hasUnsegmentedScript } from './fts-sanitize.js';
import { createCoreLogger } from '../logger.js';
import {
computeRelevance,
@@ -86,6 +86,7 @@ export function assessRetrievalConfidence(
}
const RRF_K = 60;
const MAX_PUNCTUATED_FTS_TERMS = 16;
const log = createCoreLogger('hybrid-search');
@@ -121,6 +122,27 @@ function escapeLikeTerm(term: string): string {
return term.replace(/[\\%_]/g, ch => `\\${ch}`);
}
/**
* Build a strict fallback MATCH expression from punctuation-delimited terms.
* Unlike the primary recall-oriented OR query, every surviving term is
* required. Refuse overlong expressions instead of truncating them into a
* broader query.
*/
function buildPunctuatedFtsAndQuery(query: string, minimumTerms = 2): string {
const tokens = query.match(/[\p{L}\p{N}_]+/gu) ?? [];
const terms = tokens.filter((token) => (
token.length > 2
&& !FTS_STOP_WORDS.has(token.toLowerCase())
&& !hasUnsegmentedScript(token)
));
if (terms.length < minimumTerms || terms.length > MAX_PUNCTUATED_FTS_TERMS) return '';
const uniqueTerms = [...new Set(terms)];
if (uniqueTerms.length < minimumTerms) return '';
return uniqueTerms.map(term => `"${term}"`).join(' AND ');
}
export class HybridSearch {
private db: MindDB;
private embedder: Embedder;
@@ -180,13 +202,13 @@ export class HybridSearch {
// Flag off → chunkResults is null without touching the chunk tables,
// so the lane below is byte-identical to pre-D1.
const chunkResults = chunkRetrievalEnabled()
? await this.vectorSearchChunks(query, laneFetch, gopId)
? await this.vectorSearchChunks(query, laneFetch, gopId, options.excludeDeprecated)
: null;
const [keywordResults, vectorResults] = await Promise.all([
this.keywordSearch(query, laneFetch, gopId),
this.keywordSearch(query, laneFetch, gopId, options.excludeDeprecated),
chunkResults !== null
? Promise.resolve(chunkResults)
: this.vectorSearch(query, laneFetch, gopId),
: this.vectorSearch(query, laneFetch, gopId, options.excludeDeprecated),
]);
// RRF fusion
@@ -321,7 +343,12 @@ export class HybridSearch {
return results.slice(0, limit);
}
async keywordSearch(query: string, limit: number, gopId?: string): Promise<number[]> {
async keywordSearch(
query: string,
limit: number,
gopId?: string,
excludeDeprecated = false,
): Promise<number[]> {
const raw = this.db.getDatabase();
// W3.6: Sanitize query for FTS5 with OR-based matching for better recall.
@@ -339,21 +366,29 @@ export class HybridSearch {
// unsegmented script (CJK) the emptiness is a sanitizer artifact, not a
// lack of signal: unicode61 cannot token-match CJK prose, but LIKE
// substring matching can, so route those to the fallback lane.
return hasUnsegmentedScript(query) ? this.likeFallbackSearch(query, limit, gopId) : [];
return hasUnsegmentedScript(query)
? this.likeFallbackSearch(query, limit, gopId, excludeDeprecated)
: [];
}
let sql: string;
let params: unknown[];
if (gopId) {
sql = `
SELECT mf.id FROM memory_frames_fts fts
JOIN memory_frames mf ON mf.id = fts.rowid
WHERE fts.content MATCH ? AND mf.gop_id = ?
ORDER BY rank
WHERE fts.content MATCH ? AND mf.gop_id = ?${excludeDeprecated ? " AND mf.importance != 'deprecated'" : ''}
ORDER BY fts.rank
LIMIT ?
`;
} else if (excludeDeprecated) {
sql = `
SELECT mf.id FROM memory_frames_fts fts
JOIN memory_frames mf ON mf.id = fts.rowid
WHERE fts.content MATCH ? AND mf.importance != 'deprecated'
ORDER BY fts.rank
LIMIT ?
`;
params = [safeQuery, gopId, limit];
} else {
sql = `
SELECT rowid as id FROM memory_frames_fts
@@ -361,68 +396,112 @@ export class HybridSearch {
ORDER BY rank
LIMIT ?
`;
params = [safeQuery, limit];
}
try {
const runMatch = (matchQuery: string): number[] => {
const params = gopId
? [matchQuery, gopId, limit]
: [matchQuery, limit];
const rows = raw.prepare(sql).all(...params) as { id: number }[];
return rows.map(r => r.id);
return rows.map(row => row.id);
};
const runPunctuationFallback = (allowSingleTerm = false): number[] => {
// Preserve the complete identifier first. This is the most precise lane
// and the only safe behavior when the token count exceeds the FTS bound.
const literalIds = this.likeFallbackSearch(query, limit, gopId, excludeDeprecated);
if (literalIds.length > 0) return literalIds;
// SQLite LIKE only case-folds ASCII. A strict unicode61 MATCH over every
// punctuation-delimited term supplies Unicode case-insensitive recall
// without broad OR matches.
const boundaryQuery = buildPunctuatedFtsAndQuery(query, allowSingleTerm ? 1 : 2);
if (!boundaryQuery) return [];
try {
return runMatch(boundaryQuery);
} catch {
return [];
}
};
try {
const ids = runMatch(safeQuery);
if (ids.length === 0 && /[^\p{L}\p{N}_\s]/u.test(query)) {
return runPunctuationFallback();
}
return ids;
} catch {
// FTS5 parse error (e.g. user query with FTS5-special chars that survived
// sanitization) — fall back to a LIKE keyword scan over the same column so
// we return best-effort matches instead of a false "no memory found".
return this.likeFallbackSearch(query, limit, gopId);
// FTS5 parse error (for example, an unmatched quote): retry through the
// same precise literal-plus-strict-boundary fallback used for zero hits.
return runPunctuationFallback(true);
}
}
/**
* LIKE-based keyword fallback over memory_frames.content. Used when the FTS5
* MATCH query throws a parse error (e.g. an unbalanced quote or other FTS5
* operator the user typed literally). The raw query is split into word tokens
* — stripping the punctuation that caused the FTS5 error, mirroring the
* primary sanitizer — and matched with OR-ed LIKE clauses for best-effort
* recall. Bound parameters only (the term is never interpolated) and LIKE
* metachars (`%`, `_`, `\`) are escaped with an ESCAPE clause so each token
* matches literally. If no usable token survives, a single literal LIKE over
* the whole escaped query is used.
* Whole-query LIKE fallback over memory_frames.content. Bound parameters only
* (the term is never interpolated), with LIKE metachars (`%`, `_`, `\`)
* escaped so punctuation-delimited identifiers stay literal. Unicode
* case-insensitive fallback is handled separately by strict unicode61 FTS.
*/
private likeFallbackSearch(query: string, limit: number, gopId?: string): number[] {
private likeFallbackSearch(
query: string,
limit: number,
gopId?: string,
excludeDeprecated = false,
): number[] {
const raw = this.db.getDatabase();
const tokens = query
.split(/\s+/)
.map(sanitizeFtsToken) // strip punctuation (incl. FTS5 operators), Unicode-aware
.filter(w => w.length > 0);
const terms = (tokens.length > 0 ? tokens : [query]).map(t => `%${escapeLikeTerm(t)}%`);
const likeClause = terms.map(() => `content LIKE ? ESCAPE '\\'`).join(' OR ');
const term = `%${escapeLikeTerm(query)}%`;
const deprecatedFilter = excludeDeprecated ? " AND importance != 'deprecated'" : '';
try {
if (gopId) {
const rows = raw.prepare(
`SELECT id FROM memory_frames
WHERE (${likeClause}) AND gop_id = ?
WHERE content LIKE ? ESCAPE '\\' AND gop_id = ?${deprecatedFilter}
ORDER BY created_at DESC LIMIT ?`
).all(...terms, gopId, limit) as { id: number }[];
).all(term, gopId, limit) as { id: number }[];
return rows.map(r => r.id);
}
const rows = raw.prepare(
`SELECT id FROM memory_frames
WHERE (${likeClause})
WHERE content LIKE ? ESCAPE '\\'${deprecatedFilter}
ORDER BY created_at DESC LIMIT ?`
).all(...terms, limit) as { id: number }[];
).all(term, limit) as { id: number }[];
return rows.map(r => r.id);
} catch {
return [];
}
}
async vectorSearch(query: string, limit: number, gopId?: string): Promise<number[]> {
async vectorSearch(
query: string,
limit: number,
gopId?: string,
excludeDeprecated = false,
): Promise<number[]> {
this.ensureFingerprint();
const embedding = await this.embedder.embed(query);
const blob = f32ToBlob(embedding);
const raw = this.db.getDatabase();
if (excludeDeprecated) {
const gopFilter = gopId ? ' AND gop_id = ?' : '';
try {
const rows = raw.prepare(`
SELECT rowid as id FROM memory_frames_vec
WHERE embedding MATCH ? AND k = ?
AND rowid IN (
SELECT id FROM memory_frames
WHERE importance != 'deprecated'${gopFilter}
)
ORDER BY distance
`).all(blob, limit, ...(gopId ? [gopId] : [])) as { id: number }[];
return rows.map((row) => row.id);
} catch {
return [];
}
}
if (gopId) {
// Two-step: get candidates from vec, then filter by GOP
try {
@@ -597,7 +676,12 @@ export class HybridSearch {
* chunk index is empty (or the tables are missing), returns null so callers
* can cleanly fall back to the whole-frame vectorSearch path.
*/
async vectorSearchChunks(query: string, limit: number, gopId?: string): Promise<number[] | null> {
async vectorSearchChunks(
query: string,
limit: number,
gopId?: string,
excludeDeprecated = false,
): Promise<number[] | null> {
this.ensureFingerprint();
const raw = this.db.getDatabase();
// Cheap probe — avoid embedding the query when chunks aren't populated.
@@ -618,15 +702,28 @@ export class HybridSearch {
// Over-fetch chunks (limit * 5) so dedup-to-frame still leaves enough
// candidates after collapsing multiple chunks of the same frame.
try {
const gopFilter = gopId ? ' AND mf.gop_id = ?' : '';
const candidateFilter = excludeDeprecated
? ` AND v.rowid IN (
SELECT c2.id
FROM memory_frame_chunks c2
JOIN memory_frames mf ON mf.id = c2.frame_id
WHERE mf.importance != 'deprecated'${gopFilter}
)`
: '';
const chunkRows = raw
.prepare(
`SELECT v.rowid AS chunk_id, c.frame_id
FROM memory_frame_chunks_vec v
JOIN memory_frame_chunks c ON c.id = v.rowid
WHERE v.embedding MATCH ? AND k = ?
WHERE v.embedding MATCH ? AND k = ?${candidateFilter}
ORDER BY distance`
)
.all(blob, Math.max(limit * 5, 25)) as Array<{ chunk_id: number; frame_id: number }>;
.all(
blob,
Math.max(limit * 5, 25),
...(excludeDeprecated && gopId ? [gopId] : []),
) as Array<{ chunk_id: number; frame_id: number }>;
if (chunkRows.length === 0) return [];
@@ -640,7 +737,7 @@ export class HybridSearch {
if (frameIds.length >= limit) break;
}
if (gopId) {
if (gopId && !excludeDeprecated) {
const placeholders = frameIds.map(() => '?').join(',');
const filtered = raw
.prepare(

View File

@@ -51,6 +51,19 @@
import type { MindDB } from './db.js';
import type { FrameStore, FrameSource, MemoryFrame } from './frames.js';
import { evaluateExternalMemoryIngress } from '../memory-ingress-guard.js';
export const MAX_CONSOLIDATION_OBSERVATIONS = 400;
const MAX_PROMPT_CHARS = 100_000;
const MAX_RESPONSE_CHARS = 100_000;
const MAX_LABEL_CHARS = 256;
const MAX_CURRENT_VALUE_CHARS = 4_000;
const CREATED_AT_SORT_EXPR = `julianday(CASE
WHEN created_at GLOB '*[+-][0-9][0-9][0-9][0-9]'
THEN substr(created_at, 1, length(created_at) - 5)
|| substr(created_at, -5, 3) || ':' || substr(created_at, -2)
ELSE created_at
END)`;
/**
* LLM callback the consolidation passes inject. Given a system + user message,
@@ -98,7 +111,7 @@ export interface ConsolidationResult {
export interface CollectObservationsOptions {
/** Scope to a single GOP session; omit for the whole mind. */
gopId?: string;
/** Cap the number of observations (keeps the LLM prompt bounded). */
/** Select the latest N eligible observations, returned chronologically. */
limit?: number;
/**
* Which frame source to include. Defaults to 'agent_inferred' (the
@@ -124,19 +137,26 @@ const GROUP_SYSTEM =
* an empty object when nothing parses — the callers treat "no intents" as a
* valid, non-fatal outcome rather than throwing on model chatter.
*/
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
function parseLlmJson(raw: string): Record<string, unknown> {
if (typeof raw !== 'string') return {};
if (raw.length > MAX_RESPONSE_CHARS) return {};
const trimmed = raw.trim();
if (!trimmed) return {};
try {
return JSON.parse(trimmed) as Record<string, unknown>;
const parsed: unknown = JSON.parse(trimmed);
return isRecord(parsed) ? parsed : {};
} catch {
// Not a bare JSON document — try to recover an embedded object below.
}
const match = trimmed.match(/\{[\s\S]*\}/);
if (match) {
try {
return JSON.parse(match[0]) as Record<string, unknown>;
const parsed: unknown = JSON.parse(match[0]);
return isRecord(parsed) ? parsed : {};
} catch {
// Embedded block was also malformed — fall through to the empty result.
}
@@ -154,20 +174,83 @@ function assertObservations(observations: unknown): asserts observations is Obse
throw new TypeError('consolidate: each observation must be an object');
}
const rec = o as Record<string, unknown>;
if (!Number.isInteger(rec.id)) {
throw new TypeError('consolidate: observation.id must be an integer');
if (!Number.isSafeInteger(rec.id) || Number(rec.id) <= 0) {
throw new TypeError('consolidate: observation.id must be a positive safe integer');
}
if (typeof rec.content !== 'string') {
throw new TypeError('consolidate: observation.content must be a string');
}
if (typeof rec.created_at !== 'string') {
throw new TypeError('consolidate: observation.created_at must be a string');
}
if (!Number.isFinite(observationTime(rec.created_at))) {
throw new TypeError('consolidate: observation.created_at must be a valid timestamp');
}
}
}
/** Render the observations as a `N. [YYYY-MM-DD] content` numbered list. */
function numberObservations(observations: Observation[]): string {
return observations
.map((o, idx) => `${idx + 1}. [${String(o.created_at ?? '').slice(0, 10)}] ${o.content}`)
.join('\n');
const SQLITE_TIMESTAMP_RE =
/^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})(?:\.(\d+))?(?:Z|([+-])(\d{2}):?(\d{2}))?$/;
function observationTime(value: string): number {
const match = SQLITE_TIMESTAMP_RE.exec(value.trim());
if (!match) return Number.NaN;
const [, date, time, fraction = '', sign, hour, minute] = match;
if (hour && (Number(hour) > 14 || Number(minute) > 59)) return Number.NaN;
const firstThree = Number((fraction + '000').slice(0, 3));
const millis = Math.min(999, firstThree + ((fraction[3] ?? '0') >= '5' ? 1 : 0));
const zone = sign ? `${sign}${hour}:${minute}` : 'Z';
return Date.parse(`${date}T${time}.${String(millis).padStart(3, '0')}${zone}`);
}
function normalizeObservations(observations: Observation[]): Observation[] {
const sorted = [...observations].sort((a, b) => {
const timeDelta = observationTime(a.created_at) - observationTime(b.created_at);
return timeDelta || a.id - b.id;
});
const seen = new Set<number>();
return sorted.filter(({ id }) => {
if (seen.has(id)) return false;
seen.add(id);
return true;
});
}
function prepareDetectionPrompt(
observations: Observation[],
system: string,
operation: string,
): { normalized: Observation[]; user: string } {
assertObservations(observations);
if (observations.length > MAX_CONSOLIDATION_OBSERVATIONS) {
throw new RangeError(
`${operation}: at most ${MAX_CONSOLIDATION_OBSERVATIONS} observations are allowed`,
);
}
const normalized = normalizeObservations(observations);
const lines: string[] = [];
let userChars = 0;
for (let index = 0; index < normalized.length; index += 1) {
const observation = normalized[index];
const prefix = `${index + 1}. [${observation.created_at.slice(0, 10)}] `;
const addedChars = (index > 0 ? 1 : 0) + prefix.length + observation.content.length;
if (system.length + userChars + addedChars > MAX_PROMPT_CHARS) {
throw new RangeError(`${operation}: prompt exceeds ${MAX_PROMPT_CHARS} characters`);
}
lines.push(prefix + observation.content);
userChars += addedChars;
}
return { normalized, user: lines.join('\n') };
}
function safeModelText(value: unknown, maxChars: number): string | null {
if (typeof value !== 'string') return '';
const text = value.trim();
if (text.length > maxChars) return null;
if (text && evaluateExternalMemoryIngress({ content: text }).action !== 'allow') return null;
return text;
}
/**
@@ -186,8 +269,9 @@ function mapNumbersToFrameIds(
const indices: number[] = [];
const seen = new Set<number>();
for (const n of numbers) {
const idx = Number(n);
if (!Number.isInteger(idx) || idx < 1 || idx > observations.length) continue;
if (typeof n !== 'number' || !Number.isSafeInteger(n)) continue;
const idx = n;
if (idx < 1 || idx > observations.length) continue;
if (seen.has(idx)) continue;
seen.add(idx);
indices.push(idx);
@@ -208,13 +292,17 @@ export async function detectSupersessionChains(
observations: Observation[],
llm: ConsolidationLlm,
): Promise<SupersessionChain[]> {
assertObservations(observations);
if (typeof llm !== 'function') {
throw new TypeError('detectSupersessionChains: llm must be a function');
}
if (observations.length < 2) return [];
const { normalized, user } = prepareDetectionPrompt(
observations,
SUPERSESSION_SYSTEM,
'detectSupersessionChains',
);
if (normalized.length < 2) return [];
const raw = await llm(SUPERSESSION_SYSTEM, numberObservations(observations));
const raw = await llm(SUPERSESSION_SYSTEM, user);
const parsed = parseLlmJson(raw);
const rawChains = Array.isArray(parsed.chains) ? parsed.chains : [];
@@ -222,10 +310,11 @@ export async function detectSupersessionChains(
for (const entry of rawChains) {
if (!entry || typeof entry !== 'object') continue;
const rec = entry as Record<string, unknown>;
const frameIds = mapNumbersToFrameIds(rec.ids, observations, true);
const frameIds = mapNumbersToFrameIds(rec.ids, normalized, true);
if (frameIds.length < 2) continue;
const attribute = typeof rec.attribute === 'string' ? rec.attribute.trim() : '';
const currentValue = typeof rec.current_value === 'string' ? rec.current_value.trim() : '';
const attribute = safeModelText(rec.attribute, MAX_LABEL_CHARS);
const currentValue = safeModelText(rec.current_value, MAX_CURRENT_VALUE_CHARS);
if (attribute === null || currentValue === null) continue;
chains.push({ attribute: attribute || 'value', currentValue, frameIds });
}
return chains;
@@ -240,13 +329,17 @@ export async function detectEntityGroups(
observations: Observation[],
llm: ConsolidationLlm,
): Promise<EntityGroup[]> {
assertObservations(observations);
if (typeof llm !== 'function') {
throw new TypeError('detectEntityGroups: llm must be a function');
}
if (observations.length < 2) return [];
const { normalized, user } = prepareDetectionPrompt(
observations,
GROUP_SYSTEM,
'detectEntityGroups',
);
if (normalized.length < 2) return [];
const raw = await llm(GROUP_SYSTEM, numberObservations(observations));
const raw = await llm(GROUP_SYSTEM, user);
const parsed = parseLlmJson(raw);
const rawGroups = Array.isArray(parsed.groups) ? parsed.groups : [];
@@ -254,9 +347,10 @@ export async function detectEntityGroups(
for (const entry of rawGroups) {
if (!entry || typeof entry !== 'object') continue;
const rec = entry as Record<string, unknown>;
const frameIds = mapNumbersToFrameIds(rec.ids, observations, false);
const frameIds = mapNumbersToFrameIds(rec.ids, normalized, false);
if (frameIds.length < 2) continue;
const label = typeof rec.label === 'string' ? rec.label.trim() : '';
const label = safeModelText(rec.label, MAX_LABEL_CHARS);
if (label === null) continue;
groups.push({ label: label || 'group', frameIds });
}
return groups;
@@ -284,54 +378,201 @@ export function applyConsolidation(
groups: EntityGroup[],
gopId: string,
): ConsolidationResult {
if (!frames || typeof frames.createPFrame !== 'function') {
if (
!frames
|| typeof frames.createPFrame !== 'function'
|| typeof frames.runInTransaction !== 'function'
) {
throw new TypeError('applyConsolidation: frames must be a FrameStore');
}
if (typeof gopId !== 'string' || !gopId) {
throw new Error('applyConsolidation: gopId is required');
}
const pframes: MemoryFrame[] = [];
const bframes: MemoryFrame[] = [];
const deprecated: number[] = [];
for (const chain of chains ?? []) {
const ids = chain.frameIds;
if (!Array.isArray(ids) || ids.length < 2) continue;
const baseId = ids[0];
const newestId = ids[ids.length - 1];
const newest = frames.getById(newestId);
if (!newest) continue; // newest member gone — cannot anchor a current value
// Deprecate every stale member (all but the newest).
for (const staleId of ids.slice(0, -1)) {
const stale = frames.getById(staleId);
if (!stale) continue;
frames.update(staleId, stale.content, 'deprecated');
deprecated.push(staleId);
return frames.runInTransaction(() => {
if (!Array.isArray(chains) || chains.length > MAX_CONSOLIDATION_OBSERVATIONS) {
throw new TypeError('applyConsolidation: chains must be a bounded array');
}
if (!Array.isArray(groups) || groups.length > MAX_CONSOLIDATION_OBSERVATIONS) {
throw new TypeError('applyConsolidation: groups must be a bounded array');
}
if (!frames.hasSession(gopId)) {
throw new Error(`applyConsolidation: destination session does not exist: ${gopId}`);
}
// Boost the surviving newest so it wins recall ties.
frames.update(newestId, newest.content, 'critical');
// Emit the current-value P-frame (base = oldest), preferring the model's
// clean value and falling back to the newest frame's raw content.
const cleanValue = chain.currentValue.trim() ? chain.currentValue.trim() : newest.content;
const attribute = chain.attribute.trim() ? chain.attribute.trim() : 'value';
const asOf = String(newest.created_at).slice(0, 10);
const pContent = `[current] ${attribute}: ${cleanValue} (as of ${asOf})`;
pframes.push(frames.createPFrame(gopId, pContent, baseId, 'critical', 'agent_inferred'));
}
const frameCache = new Map<number, MemoryFrame>();
const requireFrame = (id: number): MemoryFrame => {
const cached = frameCache.get(id);
if (cached) return cached;
const frame = frames.getById(id);
if (!frame) throw new Error(`applyConsolidation: missing frame ${id}`);
if (frame.importance === 'deprecated') {
throw new Error(`applyConsolidation: frame ${id} is deprecated`);
}
frameCache.set(id, frame);
return frame;
};
const requireIds = (value: unknown, kind: string): number[] => {
if (
!Array.isArray(value)
|| value.length < 2
|| value.length > MAX_CONSOLIDATION_OBSERVATIONS
) {
throw new TypeError(`applyConsolidation: ${kind}.frameIds must contain 2-${MAX_CONSOLIDATION_OBSERVATIONS} ids`);
}
const seen = new Set<number>();
for (const id of value) {
if (typeof id !== 'number' || !Number.isSafeInteger(id) || id <= 0) {
throw new TypeError(`applyConsolidation: ${kind}.frameIds must be positive safe integers`);
}
if (seen.has(id)) {
throw new TypeError(`applyConsolidation: ${kind}.frameIds must be unique`);
}
seen.add(id);
}
return [...value] as number[];
};
for (const group of groups ?? []) {
const ids = group.frameIds;
if (!Array.isArray(ids) || ids.length < 2) continue;
const label = group.label.trim() ? group.label.trim() : 'group';
const desc = `${label} (${ids.length} members)`;
bframes.push(frames.createBFrame(gopId, desc, ids[0], ids));
}
const chainPlans: Array<{
baseId: number;
pContent: string;
}> = [];
const groupPlans: Array<{ ids: number[]; desc: string }> = [];
const chainPlanByIdentity = new Map<string, { pContent: string }>();
const groupPlanByIdentity = new Map<string, { desc: string }>();
const staleRoles = new Set<number>();
const newestRoles = new Set<number>();
return { pframes, bframes, deprecated };
for (const entry of chains) {
if (!isRecord(entry)) {
throw new TypeError('applyConsolidation: each chain must be an object');
}
if (typeof entry.attribute !== 'string' || typeof entry.currentValue !== 'string') {
throw new TypeError('applyConsolidation: chain labels and values must be strings');
}
const ids = requireIds(entry.frameIds, 'chain');
const members = ids.map(requireFrame);
for (let index = 1; index < members.length; index += 1) {
const previous = members[index - 1];
const current = members[index];
const previousTime = observationTime(previous.created_at);
const currentTime = observationTime(current.created_at);
if (
!Number.isFinite(previousTime)
|| !Number.isFinite(currentTime)
|| previousTime > currentTime
|| (previousTime === currentTime && previous.id >= current.id)
) {
throw new Error('applyConsolidation: chain ids must be chronological oldest to newest');
}
}
const newest = members[members.length - 1];
const cleanValue = safeModelText(entry.currentValue, MAX_CURRENT_VALUE_CHARS);
const cleanAttribute = safeModelText(entry.attribute, MAX_LABEL_CHARS);
if (cleanValue === null || cleanAttribute === null) {
throw new Error('applyConsolidation: unsafe chain label or value');
}
const value = cleanValue || newest.content;
const attribute = cleanAttribute || 'value';
const asOf = String(newest.created_at).slice(0, 10);
const pContent = `[current] ${attribute}: ${value} (as of ${asOf})`;
if (evaluateExternalMemoryIngress({ content: pContent }).action !== 'allow') {
throw new Error('applyConsolidation: unsafe P-frame payload');
}
const chainIdentity = JSON.stringify([
attribute.normalize('NFKC').replace(/\s+/g, ' ').toLowerCase(),
ids,
]);
const existingChain = chainPlanByIdentity.get(chainIdentity);
if (existingChain) {
if (existingChain.pContent !== pContent) {
throw new Error('applyConsolidation: conflicting duplicate chain');
}
continue;
}
chainPlanByIdentity.set(chainIdentity, { pContent });
const staleIds = ids.slice(0, -1);
for (const staleId of staleIds) staleRoles.add(staleId);
newestRoles.add(newest.id);
chainPlans.push({ baseId: ids[0], pContent });
}
for (const staleId of staleRoles) {
if (newestRoles.has(staleId)) {
throw new Error(`applyConsolidation: conflicting stale/newest role for frame ${staleId}`);
}
}
for (const entry of groups) {
if (!isRecord(entry)) {
throw new TypeError('applyConsolidation: each group must be an object');
}
if (typeof entry.label !== 'string') {
throw new TypeError('applyConsolidation: group label must be a string');
}
const ids = requireIds(entry.frameIds, 'group');
ids.forEach(requireFrame);
const canonicalIds = [...ids].sort((a, b) => a - b);
const cleanLabel = safeModelText(entry.label, MAX_LABEL_CHARS);
if (cleanLabel === null) {
throw new Error('applyConsolidation: unsafe group label');
}
const desc = `${cleanLabel || 'group'} (${ids.length} members)`;
const persisted = JSON.stringify({ description: desc, references: canonicalIds });
if (
evaluateExternalMemoryIngress({ content: desc }).action !== 'allow'
|| evaluateExternalMemoryIngress({ content: persisted }).action !== 'allow'
) {
throw new Error('applyConsolidation: unsafe B-frame payload');
}
const groupIdentity = JSON.stringify([
(cleanLabel || 'group').normalize('NFKC').replace(/\s+/g, ' ').toLowerCase(),
canonicalIds,
]);
const existingGroup = groupPlanByIdentity.get(groupIdentity);
if (existingGroup) {
if (existingGroup.desc !== desc) {
throw new Error('applyConsolidation: conflicting duplicate group');
}
continue;
}
groupPlanByIdentity.set(groupIdentity, { desc });
groupPlans.push({ ids: canonicalIds, desc });
}
const pframes: MemoryFrame[] = [];
const bframes: MemoryFrame[] = [];
const deprecated = [...staleRoles];
for (const staleId of staleRoles) {
const stale = frameCache.get(staleId)!;
if (!frames.update(staleId, stale.content, 'deprecated')) {
throw new Error(`applyConsolidation: frame ${staleId} disappeared during update`);
}
}
for (const newestId of newestRoles) {
const newest = frameCache.get(newestId)!;
if (!frames.update(newestId, newest.content, 'critical')) {
throw new Error(`applyConsolidation: frame ${newestId} disappeared during update`);
}
}
for (const plan of chainPlans) {
pframes.push(frames.createPFrame(
gopId,
plan.pContent,
plan.baseId,
'critical',
'agent_inferred',
));
}
for (const plan of groupPlans) {
bframes.push(frames.createBFrame(gopId, plan.desc, plan.ids[0], plan.ids));
}
return { pframes, bframes, deprecated };
});
}
// ── Read helpers ───────────────────────────────────────────────────────────
@@ -359,11 +600,22 @@ export function collectObservations(
params.push(options.gopId);
}
let sql = `SELECT id, content, created_at FROM memory_frames WHERE ${conditions.join(' AND ')} ORDER BY created_at, id`;
if (options.limit && options.limit > 0) {
sql += ' LIMIT ?';
params.push(options.limit);
const sql = `
SELECT id, content, created_at
FROM (
SELECT id, content, created_at
FROM memory_frames
WHERE ${conditions.join(' AND ')}
ORDER BY ${CREATED_AT_SORT_EXPR} DESC, id DESC
LIMIT ?
)
ORDER BY ${CREATED_AT_SORT_EXPR} ASC, id ASC
`;
return raw.prepare(sql).all(...params) as Observation[];
}
const sql = `SELECT id, content, created_at FROM memory_frames WHERE ${conditions.join(' AND ')} ORDER BY ${CREATED_AT_SORT_EXPR}, id`;
return raw.prepare(sql).all(...params) as Observation[];
}
@@ -379,15 +631,25 @@ export function getCurrentValues(db: MindDB, gopId?: string): string[] {
gopId
? raw
.prepare(
"SELECT content FROM memory_frames WHERE frame_type = 'P' AND gop_id = ? ORDER BY created_at, id",
`SELECT content, importance FROM memory_frames WHERE frame_type = 'P' AND substr(content, 1, 10) = '[current] ' AND gop_id = ? ORDER BY ${CREATED_AT_SORT_EXPR} DESC, id DESC`,
)
.all(gopId)
: raw
.prepare("SELECT content FROM memory_frames WHERE frame_type = 'P' ORDER BY created_at, id")
.prepare(`SELECT content, importance FROM memory_frames WHERE frame_type = 'P' AND substr(content, 1, 10) = '[current] ' ORDER BY ${CREATED_AT_SORT_EXPR} DESC, id DESC`)
.all()
) as Array<{ content: string }>;
) as Array<{ content: string; importance: string }>;
return rows
.map((r) => String(r.content).replace(/^\[current\]\s*/, '').trim())
.filter(Boolean);
const seen = new Set<string>();
const current: string[] = [];
for (const row of rows) {
const line = String(row.content).replace(/^\[current\]\s*/, '').trim();
const colon = line.indexOf(':');
if (colon <= 0) continue;
const key = line.slice(0, colon).trim().replace(/\s+/g, ' ').toLowerCase();
if (!key || seen.has(key)) continue;
seen.add(key);
if (row.importance === 'deprecated') continue;
current.push(line);
}
return current.reverse();
}

View File

@@ -0,0 +1,258 @@
import { createHash, randomUUID } from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { performance } from 'node:perf_hooks';
import Database from 'better-sqlite3';
const LOCK_WAIT_TIMEOUT_MS = 12 * 60 * 1_000;
const LOCK_RETRY_MIN_MS = 35;
const LOCK_RETRY_JITTER_MS = 30;
const SAFE_HUGGING_FACE_MODEL_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*(?:\/[A-Za-z0-9][A-Za-z0-9._-]*)?$/;
const CORRUPT_ONNX_ERROR = /^Load model from (.+\.onnx) failed:\s*Protobuf parsing failed\.?$/i;
const LOCK_DIRECTORY = '.waggle-model-locks';
export interface TransformersModelLoadOptions<T> {
cacheDir: string;
model: string;
load: (canonicalCacheDir: string) => Promise<T>;
lockTimeoutMs?: number;
onQuarantine?: (quarantineDir: string) => void | Promise<void>;
}
function normalizeLockKey(value: string): string {
return process.platform === 'win32' ? value.toLocaleLowerCase('en-US') : value;
}
export function modelLoadLockPath(cacheDir: string, model: string): string {
const modelHash = createHash('sha256')
.update(normalizeLockKey(model))
.digest('hex')
.slice(0, 16);
return path.join(path.resolve(cacheDir), LOCK_DIRECTORY, `${modelHash}.sqlite`);
}
function isSqliteBusy(error: unknown): boolean {
if (!(error instanceof Error) || !('code' in error)) return false;
const code = String((error as Error & { code?: unknown }).code);
return code === 'SQLITE_BUSY'
|| code === 'SQLITE_BUSY_SNAPSHOT'
|| code === 'SQLITE_LOCKED';
}
async function acquireCrossProcessLock(
database: Database.Database,
timeoutMs: number,
): Promise<void> {
const deadline = performance.now() + timeoutMs;
for (;;) {
try {
database.exec('BEGIN IMMEDIATE');
return;
} catch (error) {
if (!isSqliteBusy(error)) throw error;
if (performance.now() >= deadline) {
throw new Error(
`Timed out waiting ${timeoutMs}ms for local model cache lock`,
{ cause: error },
);
}
}
const retryMs = LOCK_RETRY_MIN_MS + Math.random() * LOCK_RETRY_JITTER_MS;
await new Promise<void>((resolve) => setTimeout(resolve, retryMs));
}
}
function isWithin(root: string, candidate: string): boolean {
const relative = path.relative(root, candidate);
return relative !== ''
&& relative !== '..'
&& !relative.startsWith(`..${path.sep}`)
&& !path.isAbsolute(relative);
}
function isWithinOrEqual(root: string, candidate: string): boolean {
return root === candidate || isWithin(root, candidate);
}
function assertUnlinkedPath(root: string, target: string): void {
const relative = path.relative(root, target);
if (relative === '' || relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
throw new Error('Model cache path escapes its cache root');
}
let current = root;
for (const segment of relative.split(path.sep)) {
current = path.join(current, segment);
const stats = fs.lstatSync(current);
if (stats.isSymbolicLink()) {
throw new Error(`Model cache path crosses a filesystem link: ${current}`);
}
}
}
function validateQuarantinePaths(
canonicalCacheDir: string,
modelDir: string,
reportedOnnxPath: string,
): { modelDir: string; quarantineRoot: string } | null {
if (!path.isAbsolute(reportedOnnxPath)) return null;
if (!fs.existsSync(modelDir) || !fs.existsSync(reportedOnnxPath)) return null;
assertUnlinkedPath(canonicalCacheDir, modelDir);
assertUnlinkedPath(canonicalCacheDir, reportedOnnxPath);
const quarantineRoot = path.dirname(modelDir);
const modelStats = fs.lstatSync(modelDir);
const reportStats = fs.lstatSync(reportedOnnxPath);
const rootStats = fs.lstatSync(quarantineRoot);
if (!modelStats.isDirectory() || !reportStats.isFile() || !rootStats.isDirectory()) return null;
const realCacheDir = fs.realpathSync.native(canonicalCacheDir);
const realQuarantineRoot = fs.realpathSync.native(quarantineRoot);
const realModelDir = fs.realpathSync.native(modelDir);
const realOnnxPath = fs.realpathSync.native(reportedOnnxPath);
if (!isWithinOrEqual(realCacheDir, realQuarantineRoot)
|| !isWithin(realQuarantineRoot, realModelDir)
|| path.dirname(realModelDir) !== realQuarantineRoot
|| !isWithin(realModelDir, realOnnxPath)) {
return null;
}
return { modelDir: realModelDir, quarantineRoot: realQuarantineRoot };
}
function quarantineCorruptModel(
canonicalCacheDir: string,
model: string,
reportedOnnxPath: string,
): string | null {
if (!SAFE_HUGGING_FACE_MODEL_ID.test(model)) return null;
const modelDir = path.join(canonicalCacheDir, ...model.split('/'));
const firstValidation = validateQuarantinePaths(canonicalCacheDir, modelDir, reportedOnnxPath);
if (!firstValidation) return null;
// Re-resolve immediately before the move so a changed link/path cannot redirect it.
const finalValidation = validateQuarantinePaths(canonicalCacheDir, modelDir, reportedOnnxPath);
if (!finalValidation) return null;
const quarantineDir = path.join(
finalValidation.quarantineRoot,
`${path.basename(finalValidation.modelDir)}.corrupt-${Date.now()}-${randomUUID()}`,
);
fs.renameSync(finalValidation.modelDir, quarantineDir);
return quarantineDir;
}
function reportedCorruptOnnxPath(error: unknown): string | null {
if (!(error instanceof Error)) return null;
return error.message.trim().match(CORRUPT_ONNX_ERROR)?.[1] ?? null;
}
function notifyQuarantine(
callback: TransformersModelLoadOptions<unknown>['onQuarantine'],
quarantineDir: string,
): void {
try {
const notification = callback?.(quarantineDir);
if (notification) {
void Promise.resolve(notification).catch(() => undefined);
}
} catch {
// Notification is advisory and must not alter model recovery control flow.
}
}
function prepareLockDatabase(canonicalCacheDir: string, model: string): Database.Database {
const lockPath = modelLoadLockPath(canonicalCacheDir, model);
const lockDir = path.dirname(lockPath);
fs.mkdirSync(lockDir, { recursive: true });
const lockDirStats = fs.lstatSync(lockDir);
if (!lockDirStats.isDirectory() || lockDirStats.isSymbolicLink()) {
throw new Error('Local model lock path is not a regular directory');
}
const realLockDir = fs.realpathSync.native(lockDir);
if (!isWithin(canonicalCacheDir, realLockDir)) {
throw new Error('Local model lock path escapes its cache root');
}
if (fs.existsSync(lockPath)) {
const lockStats = fs.lstatSync(lockPath);
if (!lockStats.isFile() || lockStats.isSymbolicLink()) {
throw new Error('Local model lock database is not a regular file');
}
}
return new Database(lockPath, { timeout: 0 });
}
async function runModelLoad<T>(options: TransformersModelLoadOptions<T>): Promise<T> {
const timeoutMs = options.lockTimeoutMs ?? LOCK_WAIT_TIMEOUT_MS;
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
throw new Error('Local model cache lock timeout must be a non-negative finite number');
}
const requestedCacheDir = path.resolve(options.cacheDir);
fs.mkdirSync(requestedCacheDir, { recursive: true });
const canonicalCacheDir = fs.realpathSync.native(requestedCacheDir);
const lockDatabase = prepareLockDatabase(canonicalCacheDir, options.model);
let result!: T;
let primaryError: unknown;
let hasPrimaryError = false;
try {
await acquireCrossProcessLock(lockDatabase, timeoutMs);
try {
result = await options.load(canonicalCacheDir);
} catch (firstError) {
const reportedOnnxPath = reportedCorruptOnnxPath(firstError);
if (!reportedOnnxPath) throw firstError;
let quarantineDir: string | null = null;
try {
quarantineDir = quarantineCorruptModel(canonicalCacheDir, options.model, reportedOnnxPath);
} catch {
// Preserve the original loader error if safe quarantine cannot complete.
}
if (!quarantineDir) throw firstError;
notifyQuarantine(options.onQuarantine, quarantineDir);
result = await options.load(canonicalCacheDir);
}
} catch (error) {
primaryError = error;
hasPrimaryError = true;
}
let cleanupError: unknown;
let hasCleanupError = false;
if (lockDatabase.inTransaction) {
try {
lockDatabase.exec('ROLLBACK');
} catch (error) {
cleanupError = error;
hasCleanupError = true;
}
}
try {
lockDatabase.close();
} catch (error) {
if (!hasCleanupError) cleanupError = error;
hasCleanupError = true;
}
if (hasPrimaryError) throw primaryError;
if (hasCleanupError) throw cleanupError;
return result;
}
/**
* Serializes one model/cache load across workers and processes. SQLite owns
* the OS lock, so process termination releases it without PID/age heuristics.
*/
export function withTransformersModelLoad<T>(options: TransformersModelLoadOptions<T>): Promise<T> {
return runModelLoad(options);
}

View File

@@ -1,3 +1,4 @@
import fs from 'node:fs';
import path from 'node:path';
import { MindDB } from './mind/db.js';
import { createCoreLogger } from './logger.js';
@@ -61,8 +62,20 @@ export class MultiMindCache {
this.cache.delete(workspaceId);
}
const mindPath = this.getMindPath(workspaceId);
if (!mindPath) return null;
try {
const mindPath = this.getMindPath(workspaceId);
if (!mindPath) return null;
if (mindPath === ':memory:') {
if (this.cache.size >= this.maxOpen) this.evictLRU();
const recheck = this.cache.get(workspaceId);
if (recheck?.db.isOpen()) {
recheck.lastAccessed = Date.now();
return recheck.db;
}
const db = new MindDB(mindPath);
this.cache.set(workspaceId, { db, lastAccessed: Date.now(), pins: carriedPins });
return db;
}
// Review Critical #2: path-traversal guard. Defense-in-depth against an
// attacker-controlled workspaceId (e.g. from an LLM tool call with a misconfigured
@@ -78,9 +91,6 @@ export class MultiMindCache {
}
}
// Review Major #5: re-check after evictLRU — a concurrent call may have just
// inserted the same workspaceId between our initial .get() and here.
try {
if (this.cache.size >= this.maxOpen) {
this.evictLRU();
}
@@ -89,7 +99,23 @@ export class MultiMindCache {
recheck.lastAccessed = Date.now();
return recheck.db;
}
const db = new MindDB(mindPath);
const mindStat = fs.lstatSync(mindPath, { throwIfNoEntry: false });
let canonicalMind: string;
if (mindStat) {
if (!mindStat.isFile() || mindStat.isSymbolicLink() || mindStat.nlink !== 1) return null;
canonicalMind = fs.realpathSync.native(mindPath);
} else {
const canonicalParent = fs.realpathSync.native(path.dirname(mindPath));
canonicalMind = path.join(canonicalParent, path.basename(mindPath));
}
if (this.allowedRoot) {
const canonicalRoot = fs.realpathSync.native(this.allowedRoot);
const relative = path.relative(canonicalRoot, canonicalMind);
if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
return null;
}
}
const db = new MindDB(canonicalMind);
this.cache.set(workspaceId, { db, lastAccessed: Date.now(), pins: carriedPins });
return db;
} catch (err) {
@@ -130,7 +156,12 @@ export class MultiMindCache {
*/
release(workspaceId: string): void {
const entry = this.cache.get(workspaceId);
if (entry && entry.pins > 0) entry.pins -= 1;
if (!entry || entry.pins === 0) return;
entry.pins -= 1;
if (entry.pins === 0 && this.cache.size > this.maxOpen) {
this.evictLRU();
}
}
has(workspaceId: string): boolean {

View File

@@ -123,6 +123,13 @@ interface WorkspacesMeta {
defaultWorkspace?: string | null;
}
const WORKSPACE_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{0,199}$/;
function isContained(root: string, candidate: string): boolean {
const relative = path.relative(root, candidate);
return relative === '' || (!path.isAbsolute(relative) && relative !== '..' && !relative.startsWith(`..${path.sep}`));
}
/**
* WorkspaceManager manages workspace CRUD, groups, and directory structure.
* Each workspace lives under {baseDir}/workspaces/{id}/ with:
@@ -132,6 +139,7 @@ interface WorkspacesMeta {
*/
export class WorkspaceManager {
private readonly workspacesDir: string;
private readonly canonicalWorkspacesDir: string;
private readonly metaPath: string;
constructor(private readonly baseDir: string) {
@@ -141,6 +149,18 @@ export class WorkspaceManager {
if (!fs.existsSync(this.workspacesDir)) {
fs.mkdirSync(this.workspacesDir, { recursive: true });
}
const rootStat = fs.lstatSync(this.workspacesDir);
const canonicalBase = fs.realpathSync.native(baseDir);
const canonicalRoot = fs.realpathSync.native(this.workspacesDir);
if (
rootStat.isSymbolicLink()
|| !rootStat.isDirectory()
|| canonicalRoot === canonicalBase
|| !isContained(canonicalBase, canonicalRoot)
) {
throw new Error('Workspace root must be a regular directory inside the data directory');
}
this.canonicalWorkspacesDir = canonicalRoot;
}
/**
@@ -164,8 +184,19 @@ export class WorkspaceManager {
*/
// Reverse-ported from OSS hive-mind (oss-drift triage R4, 2026-06-11).
ensure(id: string, options: Partial<CreateWorkspaceOptions> = {}): WorkspaceConfig {
this.assertWorkspaceId(id);
const existing = this.get(id);
if (existing) return existing;
const workspacePath = path.join(this.resolveWorkspaceRoot(), id);
const workspaceStat = fs.lstatSync(workspacePath, { throwIfNoEntry: false });
if (workspaceStat) {
if (id !== 'default' || !this.isEmptyLegacyWorkspaceDirectory(workspacePath, workspaceStat)) {
throw new Error(`Workspace path already exists but is not a valid workspace: ${id}`);
}
const sessionsPath = path.join(workspacePath, 'sessions');
if (fs.lstatSync(sessionsPath, { throwIfNoEntry: false })) fs.rmdirSync(sessionsPath);
fs.rmdirSync(workspacePath);
}
return this.createWithId(id, {
...options,
@@ -178,13 +209,19 @@ export class WorkspaceManager {
* Shared create path: write directory structure + config for an exact id.
*/
private createWithId(id: string, options: CreateWorkspaceOptions): WorkspaceConfig {
const wsDir = path.join(this.workspacesDir, id);
this.assertWorkspaceId(id);
const canonicalRoot = this.resolveWorkspaceRoot();
const wsDir = path.join(canonicalRoot, id);
fs.mkdirSync(wsDir, { recursive: true });
fs.mkdirSync(path.join(wsDir, 'sessions'), { recursive: true });
fs.mkdirSync(wsDir);
const canonicalWorkspace = fs.realpathSync.native(wsDir);
if (!isContained(canonicalRoot, canonicalWorkspace)) {
throw new Error(`Workspace path escapes workspace root: ${id}`);
}
fs.mkdirSync(path.join(canonicalWorkspace, 'sessions'));
// Touch workspace.mind — MindDB will init schema when first opened
fs.writeFileSync(path.join(wsDir, 'workspace.mind'), '');
fs.writeFileSync(path.join(canonicalWorkspace, 'workspace.mind'), '', { flag: 'wx' });
const config: WorkspaceConfig = {
id,
@@ -214,9 +251,9 @@ export class WorkspaceManager {
};
fs.writeFileSync(
path.join(wsDir, 'workspace.json'),
path.join(canonicalWorkspace, 'workspace.json'),
JSON.stringify(config, null, 2),
'utf-8'
{ encoding: 'utf-8', flag: 'wx' }
);
return config;
@@ -226,18 +263,14 @@ export class WorkspaceManager {
* List all workspaces by reading workspace.json from each subdirectory.
*/
list(): WorkspaceConfig[] {
if (!fs.existsSync(this.workspacesDir)) return [];
const entries = fs.readdirSync(this.workspacesDir, { withFileTypes: true });
const root = this.resolveWorkspaceRoot();
const entries = fs.readdirSync(root, { withFileTypes: true });
const configs: WorkspaceConfig[] = [];
for (const entry of entries) {
if (!entry.isDirectory()) continue;
const configPath = path.join(this.workspacesDir, entry.name, 'workspace.json');
if (fs.existsSync(configPath)) {
const raw = fs.readFileSync(configPath, 'utf-8');
configs.push(JSON.parse(raw) as WorkspaceConfig);
}
if (!entry.isDirectory() || !WORKSPACE_ID.test(entry.name)) continue;
const config = this.get(entry.name);
if (config) configs.push(config);
}
return configs;
@@ -262,11 +295,22 @@ export class WorkspaceManager {
* Get a workspace by ID. Returns null if not found.
*/
get(id: string): WorkspaceConfig | null {
const configPath = path.join(this.workspacesDir, id, 'workspace.json');
if (!fs.existsSync(configPath)) return null;
if (!WORKSPACE_ID.test(id)) return null;
const raw = fs.readFileSync(configPath, 'utf-8');
return JSON.parse(raw) as WorkspaceConfig;
try {
const workspaceDir = this.resolveWorkspaceDir(id);
if (!workspaceDir) return null;
const configPath = path.join(workspaceDir, 'workspace.json');
const configStat = fs.lstatSync(configPath, { throwIfNoEntry: false });
if (!configStat?.isFile() || configStat.isSymbolicLink() || configStat.nlink !== 1) return null;
const canonicalConfig = fs.realpathSync.native(configPath);
if (!isContained(workspaceDir, canonicalConfig)) return null;
const config = JSON.parse(fs.readFileSync(canonicalConfig, 'utf-8')) as WorkspaceConfig;
return config.id === id ? config : null;
} catch {
return null;
}
}
/**
@@ -293,10 +337,11 @@ export class WorkspaceManager {
* Delete a workspace by removing its entire directory.
*/
delete(id: string): void {
const wsDir = path.join(this.workspacesDir, id);
if (fs.existsSync(wsDir)) {
fs.rmSync(wsDir, { recursive: true, force: true });
}
this.assertWorkspaceId(id);
if (!this.get(id)) return;
const workspaceDir = this.resolveWorkspaceDir(id);
if (!workspaceDir) return;
fs.rmSync(workspaceDir, { recursive: true, force: true });
}
/**
@@ -318,7 +363,85 @@ export class WorkspaceManager {
* Get the path to a workspace's .mind file.
*/
getMindPath(id: string): string {
return path.join(this.workspacesDir, id, 'workspace.mind');
this.assertWorkspaceId(id);
const lexicalMindPath = path.join(this.workspacesDir, id, 'workspace.mind');
const workspaceDir = this.resolveWorkspaceDir(id);
if (!workspaceDir) throw new Error(`Workspace not found: ${id}`);
const configPath = path.join(workspaceDir, 'workspace.json');
const configStat = fs.lstatSync(configPath, { throwIfNoEntry: false });
if (!configStat?.isFile() || configStat.isSymbolicLink() || configStat.nlink !== 1) {
throw new Error(`Workspace config not found: ${id}`);
}
const canonicalConfig = fs.realpathSync.native(configPath);
if (!isContained(workspaceDir, canonicalConfig)) {
throw new Error(`Workspace config escapes workspace directory: ${id}`);
}
const config = JSON.parse(fs.readFileSync(canonicalConfig, 'utf-8')) as WorkspaceConfig;
if (config.id !== id) throw new Error(`Workspace config id mismatch: ${id}`);
const mindPath = path.join(workspaceDir, 'workspace.mind');
const mindStat = fs.lstatSync(mindPath, { throwIfNoEntry: false });
if (!mindStat) return lexicalMindPath;
if (!mindStat?.isFile() || mindStat.isSymbolicLink() || mindStat.nlink !== 1) {
throw new Error(`Workspace mind is not a regular file: ${id}`);
}
const canonicalMind = fs.realpathSync.native(mindPath);
if (!isContained(workspaceDir, canonicalMind)) {
throw new Error(`Workspace mind escapes workspace directory: ${id}`);
}
return lexicalMindPath;
}
private assertWorkspaceId(id: string): void {
if (!WORKSPACE_ID.test(id)) throw new Error(`Invalid workspace id: ${id}`);
}
private isEmptyLegacyWorkspaceDirectory(workspacePath: string, stat: fs.Stats): boolean {
if (stat.isSymbolicLink() || !stat.isDirectory()) return false;
const canonicalRoot = this.resolveWorkspaceRoot();
const canonicalWorkspace = fs.realpathSync.native(workspacePath);
if (!isContained(canonicalRoot, canonicalWorkspace)) return false;
const entries = fs.readdirSync(workspacePath, { withFileTypes: true });
if (entries.length === 0) return true;
if (entries.length !== 1 || entries[0]?.name !== 'sessions' || !entries[0].isDirectory()) return false;
const sessionsPath = path.join(workspacePath, 'sessions');
const sessionsStat = fs.lstatSync(sessionsPath);
if (sessionsStat.isSymbolicLink()) return false;
const canonicalSessions = fs.realpathSync.native(sessionsPath);
return isContained(canonicalWorkspace, canonicalSessions) && fs.readdirSync(sessionsPath).length === 0;
}
private resolveWorkspaceDir(id: string): string | null {
this.assertWorkspaceId(id);
const lexicalRoot = this.resolveWorkspaceRoot();
const lexicalWorkspace = path.resolve(lexicalRoot, id);
if (!isContained(lexicalRoot, lexicalWorkspace)) {
throw new Error(`Workspace path escapes workspace root: ${id}`);
}
const workspaceStat = fs.lstatSync(lexicalWorkspace, { throwIfNoEntry: false });
if (!workspaceStat) return null;
if (workspaceStat.isSymbolicLink() || !workspaceStat.isDirectory()) {
throw new Error(`Workspace path is not a regular directory: ${id}`);
}
const canonicalWorkspace = fs.realpathSync.native(lexicalWorkspace);
if (!isContained(this.canonicalWorkspacesDir, canonicalWorkspace)) {
throw new Error(`Workspace path escapes workspace root: ${id}`);
}
return canonicalWorkspace;
}
private resolveWorkspaceRoot(): string {
const rootStat = fs.lstatSync(this.workspacesDir, { throwIfNoEntry: false });
if (!rootStat?.isDirectory() || rootStat.isSymbolicLink()) {
throw new Error('Workspace root is not a regular directory');
}
const canonicalRoot = fs.realpathSync.native(this.workspacesDir);
if (canonicalRoot !== this.canonicalWorkspacesDir) {
throw new Error('Workspace root changed after initialization');
}
return canonicalRoot;
}
/**
@@ -381,7 +504,7 @@ export class WorkspaceManager {
}
private workspaceExists(id: string): boolean {
return fs.existsSync(path.join(this.workspacesDir, id));
return fs.existsSync(path.join(this.resolveWorkspaceRoot(), id));
}
private loadMeta(): WorkspacesMeta {