moving
This commit is contained in:
@@ -20,8 +20,8 @@
|
||||
*/
|
||||
|
||||
import type { LLMCallFn } from './pipeline.js';
|
||||
import { scanForInjection } from '../injection-scanner.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
import { evaluateExternalMemoryIngress } from '../memory-ingress-guard.js';
|
||||
import { isNoiseName, normalizeEntityName } from '../mind/entity-normalizer.js';
|
||||
import type { KnowledgeGraph } from '../mind/knowledge.js';
|
||||
|
||||
@@ -138,6 +138,7 @@ function unwrapFencedBlock(text: string): string {
|
||||
*/
|
||||
function parseJsonlOutput(raw: string, validFrameIds: ReadonlySet<number>): KgEntity[] {
|
||||
const entities: KgEntity[] = [];
|
||||
const seenEntityFrames = new Set<string>();
|
||||
const cleaned = unwrapFencedBlock(raw);
|
||||
|
||||
for (const line of cleaned.split('\n')) {
|
||||
@@ -146,13 +147,16 @@ function parseJsonlOutput(raw: string, validFrameIds: ReadonlySet<number>): KgEn
|
||||
|
||||
let parsed: Record<string, unknown>;
|
||||
try {
|
||||
parsed = JSON.parse(trimmed) as Record<string, unknown>;
|
||||
const value: unknown = JSON.parse(trimmed);
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) continue;
|
||||
parsed = value as Record<string, unknown>;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
const frameId = Number(parsed.frame_id);
|
||||
if (!Number.isFinite(frameId) || !validFrameIds.has(frameId)) continue;
|
||||
const frameId = parsed.frame_id;
|
||||
if (typeof frameId !== 'number' || !Number.isSafeInteger(frameId) ||
|
||||
!validFrameIds.has(frameId)) continue;
|
||||
|
||||
const name = typeof parsed.name === 'string' ? parsed.name.trim() : '';
|
||||
if (name.length < 2) continue;
|
||||
@@ -163,12 +167,17 @@ function parseJsonlOutput(raw: string, validFrameIds: ReadonlySet<number>): KgEn
|
||||
const rawType = typeof parsed.type === 'string' ? parsed.type.toLowerCase().trim() : '';
|
||||
if (!(KG_ENTITY_TYPES as readonly string[]).includes(rawType)) continue;
|
||||
|
||||
const scan = scanForInjection(name, 'tool_output');
|
||||
if (!scan.safe) {
|
||||
log.warn('dropping extracted entity name with injection payload', { flags: scan.flags.join(',') });
|
||||
const ingress = evaluateExternalMemoryIngress({ content: name });
|
||||
if (ingress.action !== 'allow') {
|
||||
log.warn('dropping extracted entity name with injection payload', {
|
||||
flags: ingress.scan.flags.join(','),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const entityFrameKey = JSON.stringify([frameId, normalizeEntityName(name)]);
|
||||
if (seenEntityFrames.has(entityFrameKey)) continue;
|
||||
seenEntityFrames.add(entityFrameKey);
|
||||
entities.push({ frameId, name, type: rawType as KgEntityType });
|
||||
}
|
||||
|
||||
@@ -214,7 +223,14 @@ export interface WriteKgEntitiesResult {
|
||||
|
||||
function safeParseProps(raw: string | undefined | null): Record<string, unknown> {
|
||||
if (!raw) return {};
|
||||
try { return JSON.parse(raw) as Record<string, unknown>; } catch { return {}; }
|
||||
try {
|
||||
const value: unknown = JSON.parse(raw);
|
||||
return value && typeof value === 'object' && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -231,37 +247,54 @@ export function writeKgEntities(
|
||||
kg: KnowledgeGraph,
|
||||
extraction: KgEntityExtraction,
|
||||
): WriteKgEntitiesResult {
|
||||
const result: WriteKgEntitiesResult = { created: 0, updated: 0 };
|
||||
return kg.runInTransaction(() => {
|
||||
const result: WriteKgEntitiesResult = { created: 0, updated: 0 };
|
||||
|
||||
for (const entity of extraction.entities) {
|
||||
// Defense at the write seam (mirrors the cognify CLI): callers other than
|
||||
// extractKgEntities may not have noise-filtered.
|
||||
if (isNoiseName(entity.name)) continue;
|
||||
if (normalizeEntityName(entity.name).length < 3) continue;
|
||||
for (const entity of extraction.entities) {
|
||||
if (!entity || typeof entity !== 'object') continue;
|
||||
if (!Number.isSafeInteger(entity.frameId) || entity.frameId <= 0) continue;
|
||||
const name = typeof entity.name === 'string' ? entity.name.trim() : '';
|
||||
if (isNoiseName(name)) continue;
|
||||
if (normalizeEntityName(name).length < 3) continue;
|
||||
if (!(KG_ENTITY_TYPES as readonly string[]).includes(entity.type)) continue;
|
||||
if (evaluateExternalMemoryIngress({ content: name }).action !== 'allow') continue;
|
||||
|
||||
const existing = kg.findEntityByName(entity.name);
|
||||
if (existing) {
|
||||
const existingProps = safeParseProps(existing.properties);
|
||||
const seenCount = Number(existingProps.seen_count ?? 1) + 1;
|
||||
kg.updateEntity(existing.id, {
|
||||
properties: { ...existingProps, seen_count: seenCount },
|
||||
});
|
||||
kg.linkEntityToFrame(existing.id, entity.frameId);
|
||||
result.updated++;
|
||||
} else {
|
||||
try {
|
||||
const created = kg.createEntity(entity.type, entity.name, { seen_count: 1, source: 'cognify-llm' });
|
||||
kg.linkEntityToFrame(created.id, entity.frameId);
|
||||
result.created++;
|
||||
} catch (e: unknown) {
|
||||
// Ontology validation may reject — skip this entity, never abort the pass.
|
||||
log.warn('createEntity rejected extracted entity', {
|
||||
name: entity.name,
|
||||
error: e instanceof Error ? e.message : String(e),
|
||||
const existing = kg.findEntityByName(name);
|
||||
if (existing) {
|
||||
if (!kg.linkEntityToFrameStrict(existing.id, entity.frameId)) continue;
|
||||
const existingProps = safeParseProps(existing.properties);
|
||||
const previousSeenCount = Number(existingProps.seen_count ?? 1);
|
||||
const seenCount = (Number.isFinite(previousSeenCount) && previousSeenCount >= 0
|
||||
? previousSeenCount
|
||||
: 1) + 1;
|
||||
kg.updateEntity(existing.id, {
|
||||
properties: { ...existingProps, seen_count: seenCount },
|
||||
});
|
||||
result.updated++;
|
||||
} else {
|
||||
let created: { id: number };
|
||||
try {
|
||||
created = kg.createEntity(entity.type, name, {
|
||||
seen_count: 1,
|
||||
source: 'cognify-llm',
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message.startsWith('Validation failed:')) {
|
||||
log.warn('createEntity rejected extracted entity', {
|
||||
name,
|
||||
error: error.message,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (!kg.linkEntityToFrameStrict(created.id, entity.frameId)) {
|
||||
throw new Error(`KG writer failed to link entity ${created.id} to frame ${entity.frameId}`);
|
||||
}
|
||||
result.created++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -16,7 +16,10 @@ import type {
|
||||
} from './types.js';
|
||||
import { CLASSIFY_PROMPT, EXTRACT_PROMPT, SYNTHESIZE_PROMPT } from './prompts.js';
|
||||
import { dedup } from './dedup.js';
|
||||
import { scanForInjection } from '../injection-scanner.js';
|
||||
import {
|
||||
evaluateExternalMemoryIngress,
|
||||
projectExternalMemoryContent,
|
||||
} from '../memory-ingress-guard.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
|
||||
const log = createCoreLogger('harvest-pipeline');
|
||||
@@ -102,26 +105,34 @@ export class HarvestPipeline {
|
||||
const errors: string[] = [];
|
||||
log.info('harvest pipeline starting', { source, itemCount: items.length, batchSize: this.batchSize, concurrency: this.concurrency });
|
||||
|
||||
// Pass 0: Injection scan — drop any item whose title or content carries a
|
||||
// prompt-injection payload (role_override / prompt_extraction / instruction_injection).
|
||||
// Pass 0: Injection scan — drop any item whose untrusted title or message
|
||||
// text carries a prompt-injection payload. Structured conversation adapters
|
||||
// synthesize item.content with trusted `user:` / `assistant:` labels; scan
|
||||
// their original message text instead so those labels are not mistaken for
|
||||
// attacker-supplied authority markers. Unstructured items still scan their
|
||||
// complete content. The exact-serialization check prevents a partial
|
||||
// messages projection from hiding extra attacker-controlled content.
|
||||
// Harvest ingests UNTRUSTED external exports (ChatGPT/Claude/Gemini JSON dumps,
|
||||
// Perplexity shares, URL fetches). A hostile file must not flow through to the
|
||||
// LLM passes or into memory frames.
|
||||
const originalCount = items.length;
|
||||
items = items.filter((item) => {
|
||||
// Scan title + first 4KB of content — enough to catch payloads hidden in either field.
|
||||
// Using 'tool_output' context since imports are external data, weighted like tool output.
|
||||
const probe = `${item.title ?? ''}\n${(item.content ?? '').slice(0, 4000)}`;
|
||||
const scan = scanForInjection(probe, 'tool_output');
|
||||
if (!scan.safe) {
|
||||
const reason = scan.flags.join(',');
|
||||
const untrustedContent = projectExternalMemoryContent({
|
||||
content: item.content ?? '',
|
||||
messages: item.messages,
|
||||
parseMethod: item.metadata?.parseMethod,
|
||||
});
|
||||
const decision = evaluateExternalMemoryIngress({
|
||||
title: item.title,
|
||||
content: untrustedContent,
|
||||
});
|
||||
if (decision.action === 'block') {
|
||||
log.warn('dropping harvest item with injection payload', {
|
||||
itemId: item.id,
|
||||
title: item.title?.slice(0, 80),
|
||||
flags: scan.flags,
|
||||
score: scan.score,
|
||||
itemId: String(item.id).slice(0, 80),
|
||||
flags: decision.scan.flags,
|
||||
score: decision.scan.score,
|
||||
});
|
||||
errors.push(`Blocked item "${item.title?.slice(0, 40) ?? item.id}" — injection detected (${reason})`);
|
||||
errors.push('Blocked imported item due to unsafe content.');
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
import type { FrameStore } from '../mind/frames.js';
|
||||
import type { UniversalImportItem } from './types.js';
|
||||
import { HARVEST_FRAME_CONTENT_CAP } from './types.js';
|
||||
import { scanForInjection } from '../injection-scanner.js';
|
||||
import { evaluateExternalMemoryIngress } from '../memory-ingress-guard.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
|
||||
const log = createCoreLogger('raw-turns');
|
||||
@@ -44,6 +44,7 @@ export const MIND_RAWTURN_PREFIX = '[mind-rawturn';
|
||||
/** Hard per-conversation cap — backstop against pathological exports.
|
||||
* LoCoMo conversations run ~600 turns; 2000 leaves generous headroom. */
|
||||
export const MAX_TURNS_PER_ITEM = 2000;
|
||||
const MAX_INJECTION_DROP_LOGS = 8;
|
||||
|
||||
/** Env kill switch (checked by CALLERS, mirrored here for the recall lane). */
|
||||
export const RAWDETAIL_KILL_SWITCH = 'WAGGLE_RAWDETAIL';
|
||||
@@ -127,31 +128,38 @@ export function writeRawTurnFrames(
|
||||
const itemTs = isIsoTimestamp(item.timestamp) ? item.timestamp : undefined;
|
||||
|
||||
let turn = 0;
|
||||
let inspected = 0;
|
||||
let suppressedInjectionLogs = 0;
|
||||
for (const msg of messages) {
|
||||
if (msg.role !== 'user' && msg.role !== 'assistant') continue;
|
||||
if (inspected >= MAX_TURNS_PER_ITEM) {
|
||||
result.capped = true;
|
||||
break;
|
||||
}
|
||||
inspected++;
|
||||
const text = (msg.text ?? '').trim();
|
||||
if (text.length === 0) {
|
||||
result.skippedEmpty++;
|
||||
continue;
|
||||
}
|
||||
if (turn >= MAX_TURNS_PER_ITEM) {
|
||||
result.capped = true;
|
||||
break;
|
||||
}
|
||||
// Scan first 4KB — same probe budget as the harvest pipeline's Pass 0.
|
||||
const scan = scanForInjection(text.slice(0, 4000), 'tool_output');
|
||||
if (!scan.safe) {
|
||||
const storedText = text.slice(0, HARVEST_FRAME_CONTENT_CAP);
|
||||
const decision = evaluateExternalMemoryIngress({ content: storedText });
|
||||
if (decision.action === 'block') {
|
||||
result.injectionDropped++;
|
||||
log.warn('dropping raw turn with injection payload', {
|
||||
conv: convKey, turn, flags: scan.flags.join(','),
|
||||
});
|
||||
if (result.injectionDropped <= MAX_INJECTION_DROP_LOGS) {
|
||||
log.warn('dropping raw turn with injection payload', {
|
||||
conv: convKey, turn, flags: decision.scan.flags,
|
||||
});
|
||||
} else {
|
||||
suppressedInjectionLogs++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const speaker = sanitizeToken(msg.role, 24);
|
||||
const createdAt = isIsoTimestamp(msg.timestamp) ? msg.timestamp : itemTs;
|
||||
frames.createIFrame(
|
||||
gopId,
|
||||
`${rawTurnHeader(convKey, turn, speaker)}\n${text.slice(0, HARVEST_FRAME_CONTENT_CAP)}`,
|
||||
`${rawTurnHeader(convKey, turn, speaker)}\n${storedText}`,
|
||||
'normal',
|
||||
'import',
|
||||
createdAt,
|
||||
@@ -160,9 +168,14 @@ export function writeRawTurnFrames(
|
||||
turn++;
|
||||
}
|
||||
|
||||
if (suppressedInjectionLogs > 0) {
|
||||
log.warn('additional raw-turn injection warnings suppressed', {
|
||||
conv: convKey, suppressed: suppressedInjectionLogs,
|
||||
});
|
||||
}
|
||||
if (result.capped) {
|
||||
log.warn('raw-turn storage capped — conversation exceeds MAX_TURNS_PER_ITEM', {
|
||||
conv: convKey, stored: result.written, totalMessages: messages.length,
|
||||
conv: convKey, inspected, stored: result.written, totalMessages: messages.length,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
|
||||
@@ -7,8 +7,8 @@
|
||||
* internal services on the cloud/TEAMS deploy (sidecar binds 0.0.0.0) and can
|
||||
* exfiltrate instance-metadata IAM credentials.
|
||||
*
|
||||
* This module is a self-contained, dependency-free (node builtins only) copy of
|
||||
* the guard spec shared with `packages/agent/src/url-egress-guard.ts`. It lives
|
||||
* This module is a self-contained copy of the guard spec shared with
|
||||
* `packages/agent/src/url-egress-guard.ts`. It lives
|
||||
* here — rather than importing from @waggle/agent — because hive-mind-core is
|
||||
* OSS-mirrored and must not depend on Waggle-proprietary packages. Keep the two
|
||||
* implementations in sync; they share one spec.
|
||||
@@ -19,7 +19,8 @@
|
||||
*/
|
||||
|
||||
import { lookup as dnsLookup } from 'node:dns/promises';
|
||||
import { isIP } from 'node:net';
|
||||
import { isIP, type LookupFunction } from 'node:net';
|
||||
import { Agent } from 'undici';
|
||||
|
||||
export type AddressClass =
|
||||
| 'public'
|
||||
@@ -85,6 +86,7 @@ function classifyIpv4(ip: string): AddressClass {
|
||||
if (a >= 240) return 'reserved'; // 240.0.0.0/4 + 255.255.255.255
|
||||
if (a === 192 && b === 0 && c === 0) return 'reserved'; // 192.0.0.0/24
|
||||
if (a === 192 && b === 0 && c === 2) return 'reserved'; // TEST-NET-1
|
||||
if (a === 192 && b === 88 && c === 99) return 'reserved'; // Deprecated 6to4 relay anycast
|
||||
if (a === 198 && (b === 18 || b === 19)) return 'reserved'; // 198.18.0.0/15
|
||||
if (a === 198 && b === 51 && c === 100) return 'reserved'; // TEST-NET-2
|
||||
if (a === 203 && b === 0 && c === 113) return 'reserved'; // TEST-NET-3
|
||||
@@ -142,11 +144,20 @@ function classifyIpv6(ip: string): AddressClass {
|
||||
}
|
||||
|
||||
if ((h[0] & 0xffc0) === 0xfe80) return 'link-local'; // fe80::/10
|
||||
if ((h[0] & 0xffc0) === 0xfec0) return 'reserved'; // fec0::/10 deprecated site-local
|
||||
if ((h[0] & 0xfe00) === 0xfc00) return 'unique-local'; // fc00::/7 (ULA)
|
||||
if ((h[0] & 0xff00) === 0xff00) return 'multicast'; // ff00::/8
|
||||
if (h[0] === 0x2001 && h[1] === 0x0db8) return 'reserved'; // 2001:db8::/32 docs
|
||||
if (h[0] === 0x0064 && h[1] === 0xff9b) return 'reserved'; // 64:ff9b::/96 NAT64
|
||||
if (
|
||||
h[0] === 0x0064 && h[1] === 0xff9b
|
||||
&& ((h[2] === 0 && h[3] === 0 && h[4] === 0 && h[5] === 0) || h[2] === 1)
|
||||
) return 'reserved'; // 64:ff9b::/96 and 64:ff9b:1::/48 translation prefixes
|
||||
if (h[0] === 0x0100 && h[1] === 0 && h[2] === 0 && h[3] === 0) return 'reserved'; // 100::/64 discard
|
||||
if (h[0] === 0x0100 && h[1] === 0 && h[2] === 0 && h[3] === 1) return 'reserved'; // 100:0:0:1::/64 dummy
|
||||
if (h[0] === 0x2001 && h[1] === 2 && h[2] === 0) return 'reserved'; // 2001:2::/48 benchmark
|
||||
if (h[0] === 0x2002) return 'reserved'; // 2002::/16 deprecated 6to4
|
||||
if (h[0] === 0x3fff && (h[1] & 0xf000) === 0) return 'reserved'; // 3fff::/20 docs
|
||||
if (h[0] === 0x5f00) return 'reserved'; // 5f00::/16 SRv6 SIDs
|
||||
return 'public';
|
||||
}
|
||||
|
||||
@@ -169,6 +180,120 @@ async function defaultLookup(hostname: string): Promise<ResolvedAddress[]> {
|
||||
return results.map((r) => ({ address: r.address, family: r.family }));
|
||||
}
|
||||
|
||||
async function resolveHostname(
|
||||
hostname: string,
|
||||
rawUrl: string,
|
||||
lookupFn: LookupFn,
|
||||
): Promise<ResolvedAddress[]> {
|
||||
try {
|
||||
const addresses = await lookupFn(hostname);
|
||||
if (!addresses || addresses.length === 0) {
|
||||
throw new EgressBlockedError(
|
||||
`DNS resolution returned no addresses for "${hostname}"`,
|
||||
rawUrl,
|
||||
);
|
||||
}
|
||||
return addresses;
|
||||
} catch (err) {
|
||||
if (err instanceof EgressBlockedError) throw err;
|
||||
const detail = err instanceof Error ? err.message : String(err);
|
||||
throw new EgressBlockedError(
|
||||
`DNS resolution failed for "${hostname}": ${detail}`,
|
||||
rawUrl,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function validateResolvedAddresses(
|
||||
addresses: ResolvedAddress[],
|
||||
hostname: string,
|
||||
rawUrl: string,
|
||||
allowLocal: boolean,
|
||||
): void {
|
||||
for (const { address } of addresses) {
|
||||
const cls = classifyAddress(address);
|
||||
if (!isAllowed(cls, allowLocal)) {
|
||||
throw new EgressBlockedError(
|
||||
`Blocked egress to ${cls} address ${address} (host "${hostname}")`,
|
||||
rawUrl,
|
||||
cls,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function createGuardedLookup(
|
||||
allowLocal: boolean,
|
||||
lookupFn: LookupFn,
|
||||
): LookupFunction {
|
||||
return (hostname, options, callback) => {
|
||||
void resolveHostname(hostname, hostname, lookupFn)
|
||||
.then((addresses) => {
|
||||
validateResolvedAddresses(addresses, hostname, hostname, allowLocal);
|
||||
|
||||
const requestedFamily = options.family === 4 || options.family === 'IPv4'
|
||||
? 4
|
||||
: options.family === 6 || options.family === 'IPv6'
|
||||
? 6
|
||||
: 0;
|
||||
const candidates = requestedFamily === 0
|
||||
? addresses
|
||||
: addresses.filter(({ family }) => family === requestedFamily);
|
||||
if (candidates.length === 0) {
|
||||
throw new EgressBlockedError(
|
||||
`DNS resolution returned no IPv${requestedFamily} addresses for "${hostname}"`,
|
||||
hostname,
|
||||
);
|
||||
}
|
||||
|
||||
if (options.all) {
|
||||
callback(null, candidates);
|
||||
} else {
|
||||
const selected = candidates[0];
|
||||
callback(null, selected.address, selected.family);
|
||||
}
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
callback(err as NodeJS.ErrnoException, '');
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
function createGuardedAgent(allowLocal: boolean, lookupFn: LookupFn): Agent {
|
||||
return new Agent({
|
||||
autoSelectFamily: true,
|
||||
connect: { lookup: createGuardedLookup(allowLocal, lookupFn) },
|
||||
});
|
||||
}
|
||||
|
||||
const defaultGuardedAgents = new Map<boolean, Agent>();
|
||||
|
||||
function getDefaultGuardedAgent(allowLocal: boolean): Agent {
|
||||
const existing = defaultGuardedAgents.get(allowLocal);
|
||||
if (existing) return existing;
|
||||
const agent = createGuardedAgent(allowLocal, defaultLookup);
|
||||
defaultGuardedAgents.set(allowLocal, agent);
|
||||
return agent;
|
||||
}
|
||||
|
||||
function findEgressBlockedError(
|
||||
error: unknown,
|
||||
seen = new Set<unknown>(),
|
||||
): EgressBlockedError | null {
|
||||
if (error instanceof EgressBlockedError) return error;
|
||||
if (typeof error !== 'object' || error === null || seen.has(error)) return null;
|
||||
seen.add(error);
|
||||
|
||||
if (error instanceof AggregateError) {
|
||||
for (const nested of error.errors) {
|
||||
const blocked = findEgressBlockedError(nested, seen);
|
||||
if (blocked) return blocked;
|
||||
}
|
||||
}
|
||||
|
||||
return findEgressBlockedError((error as { cause?: unknown }).cause, seen);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that `rawUrl` is an http(s) URL whose host resolves only to public
|
||||
* addresses. Throws {@link EgressBlockedError} otherwise. Returns parsed URL.
|
||||
@@ -191,6 +316,10 @@ export async function assertUrlAllowed(
|
||||
);
|
||||
}
|
||||
|
||||
if (parsed.username || parsed.password) {
|
||||
throw new EgressBlockedError('Blocked URL credentials', rawUrl);
|
||||
}
|
||||
|
||||
// url.hostname keeps the surrounding brackets on an IPv6 literal ("[::1]"),
|
||||
// which isIP() does not recognize — strip them so the literal is classified
|
||||
// directly (loopback/private/link-local/…) instead of falling through to a DNS
|
||||
@@ -204,34 +333,11 @@ export async function assertUrlAllowed(
|
||||
addresses = [{ address: hostname, family: literalFamily }];
|
||||
} else {
|
||||
const lookupFn = options.lookup ?? defaultLookup;
|
||||
try {
|
||||
addresses = await lookupFn(hostname);
|
||||
} catch (err) {
|
||||
const detail = err instanceof Error ? err.message : String(err);
|
||||
throw new EgressBlockedError(
|
||||
`DNS resolution failed for "${hostname}": ${detail}`,
|
||||
rawUrl,
|
||||
);
|
||||
}
|
||||
if (!addresses || addresses.length === 0) {
|
||||
throw new EgressBlockedError(
|
||||
`DNS resolution returned no addresses for "${hostname}"`,
|
||||
rawUrl,
|
||||
);
|
||||
}
|
||||
addresses = await resolveHostname(hostname, rawUrl, lookupFn);
|
||||
}
|
||||
|
||||
const allowLocal = options.allowLocal ?? false;
|
||||
for (const { address } of addresses) {
|
||||
const cls = classifyAddress(address);
|
||||
if (!isAllowed(cls, allowLocal)) {
|
||||
throw new EgressBlockedError(
|
||||
`Blocked egress to ${cls} address ${address} (host "${hostname}")`,
|
||||
rawUrl,
|
||||
cls,
|
||||
);
|
||||
}
|
||||
}
|
||||
validateResolvedAddresses(addresses, hostname, rawUrl, allowLocal);
|
||||
|
||||
return parsed;
|
||||
}
|
||||
@@ -239,29 +345,119 @@ export async function assertUrlAllowed(
|
||||
export interface SafeFetchOptions extends EgressGuardOptions {
|
||||
/** Maximum redirect hops to follow (default 5). */
|
||||
maxRedirects?: number;
|
||||
/** Injectable fetch (tests). Defaults to globalThis.fetch. */
|
||||
fetchImpl?: typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
type FetchWithDispatcher = (
|
||||
input: string | URL | Request,
|
||||
init: RequestInit & { dispatcher: Agent },
|
||||
) => Promise<Response>;
|
||||
|
||||
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
||||
const CROSS_ORIGIN_SECRET_HEADERS = [
|
||||
'authorization',
|
||||
'proxy-authorization',
|
||||
'cookie',
|
||||
'cookie2',
|
||||
'x-api-key',
|
||||
'api-key',
|
||||
] as const;
|
||||
const REQUEST_BODY_HEADERS = [
|
||||
'content-encoding',
|
||||
'content-language',
|
||||
'content-length',
|
||||
'content-location',
|
||||
'content-type',
|
||||
] as const;
|
||||
|
||||
function isNonReplayableBody(body: BodyInit): boolean {
|
||||
const candidate = body as unknown as {
|
||||
getReader?: unknown;
|
||||
pipe?: unknown;
|
||||
[Symbol.asyncIterator]?: unknown;
|
||||
};
|
||||
return typeof candidate.getReader === 'function'
|
||||
|| typeof candidate.pipe === 'function'
|
||||
|| typeof candidate[Symbol.asyncIterator] === 'function';
|
||||
}
|
||||
|
||||
function redirectRequestInit(
|
||||
init: RequestInit,
|
||||
status: number,
|
||||
fromUrl: URL,
|
||||
toUrl: URL,
|
||||
): RequestInit {
|
||||
const next = { ...init };
|
||||
const method = (next.method ?? 'GET').toUpperCase();
|
||||
const rewriteToGet = ((status === 301 || status === 302) && method === 'POST')
|
||||
|| (status === 303 && method !== 'GET' && method !== 'HEAD');
|
||||
const headersToDelete = new Set<string>(['host']);
|
||||
|
||||
if (rewriteToGet) {
|
||||
next.method = 'GET';
|
||||
delete next.body;
|
||||
for (const name of REQUEST_BODY_HEADERS) headersToDelete.add(name);
|
||||
} else if (next.body !== undefined && next.body !== null && isNonReplayableBody(next.body)) {
|
||||
throw new TypeError('Cannot replay a streamed request body across a redirect');
|
||||
}
|
||||
|
||||
if (fromUrl.origin !== toUrl.origin) {
|
||||
for (const name of CROSS_ORIGIN_SECRET_HEADERS) headersToDelete.add(name);
|
||||
}
|
||||
|
||||
const headers = new Headers(next.headers);
|
||||
for (const name of headersToDelete) headers.delete(name);
|
||||
next.headers = headers;
|
||||
return next;
|
||||
}
|
||||
|
||||
/**
|
||||
* SSRF-safe fetch. Validates before the request and re-validates every redirect
|
||||
* hop (`redirect: 'manual'`). Caller-supplied `redirect` in `init` is ignored.
|
||||
* hop (`redirect: 'manual'`). Native fetch is mandatory; proxy transports need
|
||||
* an equivalent pinned connector rather than a global dispatcher override.
|
||||
* Caller-supplied `redirect` in `init` is ignored.
|
||||
*/
|
||||
export async function safeFetch(
|
||||
rawUrl: string,
|
||||
init: RequestInit = {},
|
||||
options: SafeFetchOptions = {},
|
||||
): Promise<Response> {
|
||||
if ('fetchImpl' in options) {
|
||||
throw new TypeError('safeFetch fetchImpl injection is not supported; socket pinning requires native fetch');
|
||||
}
|
||||
const maxRedirects = options.maxRedirects ?? 5;
|
||||
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
||||
|
||||
let currentUrl = rawUrl;
|
||||
let currentInit = { ...init };
|
||||
for (let hop = 0; hop <= maxRedirects; hop++) {
|
||||
await assertUrlAllowed(currentUrl, options);
|
||||
const allowLocal = options.allowLocal ?? false;
|
||||
const temporaryAgent = options.lookup !== undefined;
|
||||
const dispatcher = temporaryAgent
|
||||
? createGuardedAgent(allowLocal, options.lookup!)
|
||||
: getDefaultGuardedAgent(allowLocal);
|
||||
|
||||
const response = await fetchImpl(currentUrl, { ...init, redirect: 'manual' });
|
||||
let response: Response;
|
||||
try {
|
||||
response = await (globalThis.fetch as unknown as FetchWithDispatcher)(currentUrl, {
|
||||
...currentInit,
|
||||
redirect: 'manual',
|
||||
dispatcher,
|
||||
});
|
||||
} catch (err) {
|
||||
if (temporaryAgent) {
|
||||
await dispatcher.close().catch(() => undefined);
|
||||
}
|
||||
const blocked = findEgressBlockedError(err);
|
||||
if (blocked) {
|
||||
throw new EgressBlockedError(blocked.message, currentUrl, blocked.addressClass);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
const isRedirect = response.status >= 300 && response.status < 400;
|
||||
if (temporaryAgent) {
|
||||
void dispatcher.close().catch(() => undefined);
|
||||
}
|
||||
|
||||
const isRedirect = REDIRECT_STATUSES.has(response.status);
|
||||
const location = isRedirect ? response.headers.get('location') : null;
|
||||
if (!location) {
|
||||
return response;
|
||||
@@ -273,16 +469,22 @@ export async function safeFetch(
|
||||
/* best-effort; ignore */
|
||||
}
|
||||
|
||||
let nextUrl: string;
|
||||
let nextUrl: URL;
|
||||
try {
|
||||
nextUrl = new URL(location, currentUrl).toString();
|
||||
nextUrl = new URL(location, currentUrl);
|
||||
} catch {
|
||||
throw new EgressBlockedError(
|
||||
`Invalid redirect target "${location}"`,
|
||||
currentUrl,
|
||||
);
|
||||
}
|
||||
currentUrl = nextUrl;
|
||||
currentInit = redirectRequestInit(
|
||||
currentInit,
|
||||
response.status,
|
||||
new URL(currentUrl),
|
||||
nextUrl,
|
||||
);
|
||||
currentUrl = nextUrl.toString();
|
||||
}
|
||||
|
||||
throw new EgressBlockedError(
|
||||
|
||||
258
packages/hive-mind-core/src/hook-runtime.ts
Normal file
258
packages/hive-mind-core/src/hook-runtime.ts
Normal file
@@ -0,0 +1,258 @@
|
||||
/**
|
||||
* Minimal synchronous persistence path for short-lived IDE hooks.
|
||||
*
|
||||
* Deliberately imports only the SQLite-backed mind primitives. Hook latency
|
||||
* must not depend on loading the full core barrel, probing an embedding
|
||||
* provider, or starting an MCP server. FTS is committed synchronously;
|
||||
* vector enrichment remains repairable through reconcileVecIndex().
|
||||
*/
|
||||
|
||||
import {
|
||||
existsSync,
|
||||
lstatSync,
|
||||
mkdirSync,
|
||||
readFileSync,
|
||||
realpathSync,
|
||||
} from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { join, resolve, sep } from 'node:path';
|
||||
import { evaluateExternalMemoryIngress } from './memory-ingress-guard.js';
|
||||
import { MindDB } from './mind/db.js';
|
||||
import {
|
||||
FrameStore,
|
||||
type FrameSource,
|
||||
type Importance,
|
||||
type MemoryFrame,
|
||||
} from './mind/frames.js';
|
||||
import { SessionStore } from './mind/sessions.js';
|
||||
|
||||
export interface SaveHookFrameOptions {
|
||||
dataDir?: string;
|
||||
workspace?: string;
|
||||
content: string;
|
||||
importance: Exclude<Importance, 'deprecated'>;
|
||||
source: Exclude<FrameSource, 'import'>;
|
||||
}
|
||||
|
||||
export interface SaveHookFrameResult {
|
||||
id: string;
|
||||
success: true;
|
||||
workspace: string;
|
||||
}
|
||||
|
||||
export interface RecallHookFramesOptions {
|
||||
dataDir?: string;
|
||||
workspace?: string;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface HookMemoryHit {
|
||||
id: number;
|
||||
content: string;
|
||||
importance: string;
|
||||
source: string;
|
||||
score: number;
|
||||
created_at: string;
|
||||
from: string;
|
||||
}
|
||||
|
||||
const WORKSPACE_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{0,199}$/;
|
||||
const IMPORTANCE_SCORE: Record<Exclude<Importance, 'deprecated'>, number> = {
|
||||
critical: 1,
|
||||
important: 0.85,
|
||||
normal: 0.65,
|
||||
temporary: 0.45,
|
||||
};
|
||||
const ALLOWED_IMPORTANCE = new Set(Object.keys(IMPORTANCE_SCORE));
|
||||
const ALLOWED_SOURCE = new Set(['user_stated', 'tool_verified', 'agent_inferred', 'system']);
|
||||
|
||||
function resolveDataDir(override?: string): string {
|
||||
if (override !== undefined && override.trim() === '') {
|
||||
throw new Error('Hook data directory must not be blank');
|
||||
}
|
||||
const envDir = process.env.HIVE_MIND_DATA_DIR;
|
||||
const configured = override
|
||||
?? (envDir?.trim() ? envDir : join(homedir(), '.hive-mind'));
|
||||
const expanded = configured.startsWith('~')
|
||||
? join(homedir(), configured.slice(1))
|
||||
: configured;
|
||||
return resolve(expanded);
|
||||
}
|
||||
|
||||
function normalizedPath(value: string): string {
|
||||
return process.platform === 'win32' ? value.toLowerCase() : value;
|
||||
}
|
||||
|
||||
function isContained(root: string, candidate: string): boolean {
|
||||
const normalizedRoot = normalizedPath(root);
|
||||
const normalizedCandidate = normalizedPath(candidate);
|
||||
return normalizedCandidate === normalizedRoot
|
||||
|| normalizedCandidate.startsWith(`${normalizedRoot}${sep}`);
|
||||
}
|
||||
|
||||
function rejectLink(path: string, label: string): void {
|
||||
if (lstatSync(path).isSymbolicLink()) throw new Error(`${label} must not be a link`);
|
||||
}
|
||||
|
||||
function requireRegularFile(path: string, label: string): void {
|
||||
const stat = lstatSync(path);
|
||||
if (stat.isSymbolicLink()) throw new Error(`${label} must not be a link`);
|
||||
if (!stat.isFile()) throw new Error(`${label} must be a regular file`);
|
||||
if (stat.nlink !== 1) throw new Error(`${label} must not be a hard link`);
|
||||
}
|
||||
|
||||
function hasRegularFileEntry(path: string, label: string): boolean {
|
||||
const stat = lstatSync(path, { throwIfNoEntry: false });
|
||||
if (stat === undefined) return false;
|
||||
if (stat.isSymbolicLink()) throw new Error(`${label} must not be a link`);
|
||||
if (!stat.isFile()) throw new Error(`${label} must be a regular file`);
|
||||
if (stat.nlink !== 1) throw new Error(`${label} must not be a hard link`);
|
||||
return true;
|
||||
}
|
||||
|
||||
function requireSafeSqliteEntries(path: string, label: string): boolean {
|
||||
const hasDatabase = hasRegularFileEntry(path, label);
|
||||
for (const suffix of ['-wal', '-shm', '-journal']) {
|
||||
hasRegularFileEntry(`${path}${suffix}`, `${label}${suffix}`);
|
||||
}
|
||||
return hasDatabase;
|
||||
}
|
||||
|
||||
function resolveMind(options: { dataDir?: string; workspace?: string }): {
|
||||
path: string;
|
||||
workspace: string;
|
||||
} {
|
||||
const dataDir = resolveDataDir(options.dataDir);
|
||||
if (options.workspace === undefined) {
|
||||
mkdirSync(dataDir, { recursive: true });
|
||||
const canonicalDataDir = realpathSync(dataDir);
|
||||
const personalMind = join(canonicalDataDir, 'personal.mind');
|
||||
if (requireSafeSqliteEntries(personalMind, 'Personal mind')) {
|
||||
const canonicalMind = realpathSync(personalMind);
|
||||
if (!isContained(canonicalDataDir, canonicalMind)) {
|
||||
throw new Error('Personal mind escapes data directory');
|
||||
}
|
||||
return { path: canonicalMind, workspace: 'personal' };
|
||||
}
|
||||
return { path: personalMind, workspace: 'personal' };
|
||||
}
|
||||
|
||||
const id = options.workspace;
|
||||
if (!WORKSPACE_ID.test(id)) throw new Error(`Invalid workspace id: ${id}`);
|
||||
const workspacesRoot = resolve(dataDir, 'workspaces');
|
||||
const workspaceDir = resolve(workspacesRoot, id);
|
||||
if (!workspaceDir.startsWith(`${workspacesRoot}${sep}`)) {
|
||||
throw new Error(`Workspace path escapes data directory: ${id}`);
|
||||
}
|
||||
if (!existsSync(workspacesRoot) || !lstatSync(workspacesRoot).isDirectory()) {
|
||||
throw new Error(`Workspace not found: ${id}`);
|
||||
}
|
||||
rejectLink(workspacesRoot, 'Workspaces directory');
|
||||
if (!existsSync(workspaceDir) || !lstatSync(workspaceDir).isDirectory()) {
|
||||
throw new Error(`Workspace not found: ${id}`);
|
||||
}
|
||||
rejectLink(workspaceDir, 'Workspace directory');
|
||||
const canonicalDataDir = realpathSync(dataDir);
|
||||
const canonicalRoot = realpathSync(workspacesRoot);
|
||||
const canonicalWorkspace = realpathSync(workspaceDir);
|
||||
if (!isContained(canonicalDataDir, canonicalRoot)
|
||||
|| !isContained(canonicalRoot, canonicalWorkspace)) {
|
||||
throw new Error(`Workspace path escapes data directory: ${id}`);
|
||||
}
|
||||
const configPath = join(canonicalWorkspace, 'workspace.json');
|
||||
if (!existsSync(configPath)) {
|
||||
throw new Error(`Workspace not found: ${id}`);
|
||||
}
|
||||
requireRegularFile(configPath, 'Workspace config');
|
||||
const canonicalConfig = realpathSync(configPath);
|
||||
if (!isContained(canonicalWorkspace, canonicalConfig)) {
|
||||
throw new Error(`Workspace config escapes data directory: ${id}`);
|
||||
}
|
||||
let configuredId: unknown;
|
||||
try {
|
||||
configuredId = (JSON.parse(readFileSync(canonicalConfig, 'utf8')) as { id?: unknown }).id;
|
||||
} catch {
|
||||
throw new Error(`Workspace config is invalid: ${id}`);
|
||||
}
|
||||
if (configuredId !== id) throw new Error(`Workspace config id mismatch: ${id}`);
|
||||
const mindPath = join(canonicalWorkspace, 'workspace.mind');
|
||||
if (requireSafeSqliteEntries(mindPath, 'Workspace mind')) {
|
||||
const canonicalMind = realpathSync(mindPath);
|
||||
if (!isContained(canonicalWorkspace, canonicalMind)) {
|
||||
throw new Error(`Workspace mind escapes data directory: ${id}`);
|
||||
}
|
||||
return { path: canonicalMind, workspace: id };
|
||||
}
|
||||
return { path: mindPath, workspace: id };
|
||||
}
|
||||
|
||||
function assertSaveInput(options: SaveHookFrameOptions): void {
|
||||
if (typeof options.content !== 'string') throw new Error('Hook frame content must be a string');
|
||||
if (evaluateExternalMemoryIngress({ content: options.content }).action !== 'allow') {
|
||||
throw new Error('Hook frame content was rejected because it is unsafe.');
|
||||
}
|
||||
if (!ALLOWED_IMPORTANCE.has(options.importance)) {
|
||||
throw new Error(`Invalid hook frame importance: ${String(options.importance)}`);
|
||||
}
|
||||
if (!ALLOWED_SOURCE.has(options.source)) {
|
||||
throw new Error(`Invalid hook frame source: ${String(options.source)}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function saveHookFrame(options: SaveHookFrameOptions): SaveHookFrameResult {
|
||||
assertSaveInput(options);
|
||||
const target = resolveMind(options);
|
||||
const db = new MindDB(target.path);
|
||||
try {
|
||||
const raw = db.getDatabase();
|
||||
const save = raw.transaction(() => {
|
||||
const sessions = new SessionStore(db);
|
||||
const frames = new FrameStore(db);
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
const session = sessions.ensure(`mcp:${today}`, undefined, `MCP session ${today}`);
|
||||
return frames.createIFrame(
|
||||
session.gop_id,
|
||||
options.content,
|
||||
options.importance,
|
||||
options.source,
|
||||
);
|
||||
});
|
||||
const frame = db.runWithBusyRetry(save);
|
||||
return { id: String(frame.id), success: true, workspace: target.workspace };
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
export function recallHookFrames(options: RecallHookFramesOptions = {}): HookMemoryHit[] {
|
||||
const target = resolveMind(options);
|
||||
const requestedLimit = options.limit;
|
||||
const limit = typeof requestedLimit === 'number' && Number.isFinite(requestedLimit)
|
||||
? Math.min(100, Math.max(1, Math.floor(requestedLimit)))
|
||||
: 20;
|
||||
const db = new MindDB(target.path);
|
||||
try {
|
||||
const rows = db.getDatabase().prepare(`
|
||||
SELECT * FROM memory_frames
|
||||
WHERE importance != 'deprecated'
|
||||
ORDER BY CASE importance
|
||||
WHEN 'critical' THEN 4
|
||||
WHEN 'important' THEN 3
|
||||
WHEN 'normal' THEN 2
|
||||
ELSE 1
|
||||
END DESC, id DESC
|
||||
LIMIT ?
|
||||
`).all(limit) as MemoryFrame[];
|
||||
return rows.map((frame, index) => ({
|
||||
id: frame.id,
|
||||
content: frame.content,
|
||||
importance: frame.importance,
|
||||
source: frame.source,
|
||||
score: Math.max(0, (IMPORTANCE_SCORE[frame.importance as Exclude<Importance, 'deprecated'>] ?? 0) - index / 1000),
|
||||
created_at: frame.created_at,
|
||||
from: target.workspace === 'personal' ? 'personal' : `workspace:${target.workspace}`,
|
||||
}));
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,8 @@
|
||||
// @waggle/hive-mind-core — substrate package barrel.
|
||||
//
|
||||
// Distribution: Apache 2.0 OSS via `git subtree split` from waggle-os monorepo
|
||||
// to marolinik/hive-mind. Apps/web + Waggle agent harness stay proprietary in monorepo.
|
||||
// Distribution: Apache 2.0 OSS via a maintainer-curated forward-port from the
|
||||
// waggle-os monorepo to marolinik/hive-mind. Raw subtree-split branches are
|
||||
// inspection inputs only and must not be pushed as the public mirror.
|
||||
//
|
||||
// Contents: mind/ (substrate), harvest/ (ingestion pipeline), prompt-injection
|
||||
// scanner, structured logger.
|
||||
@@ -9,6 +10,13 @@
|
||||
// ── Logger + injection scanner (utilities used by substrate + Waggle agent) ──
|
||||
export { createCoreLogger, type CoreLogger } from './logger.js';
|
||||
export { scanForInjection, type ScanResult } from './injection-scanner.js';
|
||||
export {
|
||||
evaluateExternalMemoryIngress,
|
||||
projectExternalMemoryContent,
|
||||
type ExternalMemoryIngressDecision,
|
||||
type ExternalMemoryIngressInput,
|
||||
type ExternalMemoryProjectionInput,
|
||||
} from './memory-ingress-guard.js';
|
||||
|
||||
// ── mind/ — memory substrate (FrameStore, KnowledgeGraph, embedders, search, scoring) ──
|
||||
export {
|
||||
@@ -97,6 +105,7 @@ export {
|
||||
applyConsolidation,
|
||||
collectObservations,
|
||||
getCurrentValues,
|
||||
MAX_CONSOLIDATION_OBSERVATIONS,
|
||||
} from './mind/supersede.js';
|
||||
export type {
|
||||
ConsolidationLlm,
|
||||
|
||||
751
packages/hive-mind-core/src/memory-ingress-guard.ts
Normal file
751
packages/hive-mind-core/src/memory-ingress-guard.ts
Normal file
@@ -0,0 +1,751 @@
|
||||
import { scanForInjection, type ScanResult } from './injection-scanner.js';
|
||||
|
||||
export interface ExternalMemoryIngressInput {
|
||||
title?: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface ExternalMemoryProjectionInput {
|
||||
content: string;
|
||||
messages?: unknown;
|
||||
parseMethod?: unknown;
|
||||
maxChars?: number;
|
||||
}
|
||||
|
||||
export type ExternalMemoryIngressDecision =
|
||||
| { action: 'allow'; scan: ScanResult }
|
||||
| { action: 'block'; reason: 'prompt_injection'; scan: ScanResult };
|
||||
|
||||
// RawArchive supports one million characters per item. Keep the same explicit
|
||||
// public-ingress budget, checked before concatenation, scanning, or normalization.
|
||||
const MAX_EXTERNAL_MEMORY_INGRESS_CHARS = 1_000_000;
|
||||
|
||||
const NAMED_HTML_ENTITIES: Readonly<Record<string, string>> = Object.freeze({
|
||||
af: '\u2061',
|
||||
amp: '&',
|
||||
applyfunction: '\u2061',
|
||||
apos: "'",
|
||||
colon: ':',
|
||||
emsp: ' ',
|
||||
ensp: ' ',
|
||||
gt: '>',
|
||||
hairsp: ' ',
|
||||
ic: '\u2063',
|
||||
invisiblecomma: '\u2063',
|
||||
invisibletimes: '\u2062',
|
||||
it: '\u2062',
|
||||
lrm: '\u200e',
|
||||
lt: '<',
|
||||
negativemediumspace: '\u200b',
|
||||
negativethickspace: '\u200b',
|
||||
negativethinspace: '\u200b',
|
||||
negativeverythinspace: '\u200b',
|
||||
newline: '\n',
|
||||
nbsp: ' ',
|
||||
nobreak: '\u2060',
|
||||
quot: '"',
|
||||
rlm: '\u200f',
|
||||
shy: '\u00ad',
|
||||
tab: '\t',
|
||||
thinsp: ' ',
|
||||
zwj: '',
|
||||
zwnj: '',
|
||||
zwsp: '',
|
||||
zerowidthspace: '\u200b',
|
||||
});
|
||||
|
||||
type CanonicalMemoryMessage = {
|
||||
role: 'user' | 'assistant' | 'system';
|
||||
text: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Return only the attacker-controlled text represented by a stored adapter
|
||||
* projection. Role prefixes may be omitted only when plain canonical messages
|
||||
* exactly reproduce the full content and did not come from universal raw text.
|
||||
*/
|
||||
export function projectExternalMemoryContent(input: ExternalMemoryProjectionInput): string {
|
||||
let cappedContent = '';
|
||||
try {
|
||||
const content = typeof input.content === 'string' ? input.content : '';
|
||||
const maxChars = input.maxChars;
|
||||
cappedContent = maxChars === undefined
|
||||
|| !Number.isSafeInteger(maxChars)
|
||||
|| maxChars < 0
|
||||
? content
|
||||
: content.slice(0, maxChars);
|
||||
if (input.parseMethod === 'universal-text'
|
||||
|| !Array.isArray(input.messages)
|
||||
|| input.messages.length === 0) {
|
||||
return cappedContent;
|
||||
}
|
||||
|
||||
const messages: CanonicalMemoryMessage[] = [];
|
||||
for (const candidate of input.messages) {
|
||||
if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) {
|
||||
return cappedContent;
|
||||
}
|
||||
const prototype = Object.getPrototypeOf(candidate);
|
||||
if (prototype !== Object.prototype && prototype !== null) return cappedContent;
|
||||
const roleDescriptor = Object.getOwnPropertyDescriptor(candidate, 'role');
|
||||
const textDescriptor = Object.getOwnPropertyDescriptor(candidate, 'text');
|
||||
if (!roleDescriptor || !('value' in roleDescriptor)
|
||||
|| !textDescriptor || !('value' in textDescriptor)) {
|
||||
return cappedContent;
|
||||
}
|
||||
const role = roleDescriptor.value as unknown;
|
||||
const text = textDescriptor.value as unknown;
|
||||
if ((role !== 'user' && role !== 'assistant' && role !== 'system')
|
||||
|| typeof text !== 'string') {
|
||||
return cappedContent;
|
||||
}
|
||||
messages.push({ role, text });
|
||||
}
|
||||
|
||||
const serialized = messages
|
||||
.map(message => `${message.role}: ${message.text}`)
|
||||
.join('\n\n');
|
||||
if (serialized !== content) return cappedContent;
|
||||
|
||||
const parts: string[] = [];
|
||||
let cursor = 0;
|
||||
let offset = 0;
|
||||
for (const [index, message] of messages.entries()) {
|
||||
if (index > 0) offset += 2;
|
||||
const prefixStart = offset;
|
||||
const prefixEnd = prefixStart + `${message.role}: `.length;
|
||||
if (prefixStart >= cappedContent.length) break;
|
||||
if (message.role === 'system') return cappedContent;
|
||||
parts.push(cappedContent.slice(cursor, prefixStart));
|
||||
cursor = Math.min(prefixEnd, cappedContent.length);
|
||||
offset = prefixEnd + message.text.length;
|
||||
}
|
||||
parts.push(cappedContent.slice(cursor));
|
||||
return parts.join('');
|
||||
} catch {
|
||||
return cappedContent;
|
||||
}
|
||||
}
|
||||
|
||||
function decodeHtmlEntities(value: string): string {
|
||||
if (!value.includes('&')) return value;
|
||||
return value
|
||||
.replace(/&(?:amp;){2,}/gi, '&')
|
||||
.replace(/&#(?:x([0-9a-f]{1,6})|([0-9]{1,7}));?/gi, (match, hex: string, decimal: string) => {
|
||||
const codePoint = Number.parseInt(hex ?? decimal, hex ? 16 : 10);
|
||||
if (!Number.isInteger(codePoint)
|
||||
|| codePoint <= 0
|
||||
|| codePoint > 0x10ffff
|
||||
|| (codePoint >= 0xd800 && codePoint <= 0xdfff)) {
|
||||
return match;
|
||||
}
|
||||
return String.fromCodePoint(codePoint);
|
||||
})
|
||||
.replace(/&([a-z][a-z0-9]+);/gi, (match, name: string) =>
|
||||
NAMED_HTML_ENTITIES[name.toLowerCase()] ?? match)
|
||||
.replace(
|
||||
/&(amp|apos|colon|emsp|ensp|gt|hairsp|lt|newline|nbsp|quot|tab|thinsp|zwj|zwnj|zwsp)(?=[^a-z0-9;]|$)/gi,
|
||||
(_match, name: string) => NAMED_HTML_ENTITIES[name.toLowerCase()],
|
||||
);
|
||||
}
|
||||
|
||||
function decodePercentEncoding(value: string): string {
|
||||
if (!/[+%]/.test(value)) return value;
|
||||
const withSpaces = value.replace(/\+/g, ' ');
|
||||
return withSpaces.replace(/(?:%[0-9a-f]{2})+/gi, (run) => {
|
||||
try {
|
||||
return decodeURIComponent(run);
|
||||
} catch {
|
||||
const bytes = Uint8Array.from(
|
||||
run.match(/[0-9a-f]{2}/gi) ?? [],
|
||||
hex => Number.parseInt(hex, 16),
|
||||
);
|
||||
return new TextDecoder('utf-8', { fatal: false }).decode(bytes);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function decodeUnicodeEscapes(value: string): string {
|
||||
if (!/\\u/i.test(value)) return value;
|
||||
return value.replace(
|
||||
/\\u(?:\{([0-9a-f]{1,6})\}|([0-9a-f]{4}))/gi,
|
||||
(match, braced: string | undefined, fixed: string | undefined) => {
|
||||
const codePoint = Number.parseInt(braced ?? fixed ?? '', 16);
|
||||
if (!Number.isInteger(codePoint) || codePoint > 0x10ffff) return match;
|
||||
if (braced !== undefined) {
|
||||
if (codePoint >= 0xd800 && codePoint <= 0xdfff) return match;
|
||||
return String.fromCodePoint(codePoint);
|
||||
}
|
||||
return String.fromCharCode(codePoint);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function decodeHexEscapes(value: string): string {
|
||||
if (!/\\x/i.test(value)) return value;
|
||||
return value.replace(/\\x([0-9a-f]{2})/gi, (_match, hex: string) =>
|
||||
String.fromCharCode(Number.parseInt(hex, 16)));
|
||||
}
|
||||
|
||||
const MIXED_SCRIPT_CONFUSABLES: Readonly<Record<string, string>> = Object.freeze({
|
||||
'\u0391': 'A',
|
||||
'\u0392': 'B',
|
||||
'\u0395': 'E',
|
||||
'\u0396': 'Z',
|
||||
'\u0397': 'H',
|
||||
'\u0399': 'I',
|
||||
'\u039a': 'K',
|
||||
'\u039c': 'M',
|
||||
'\u039d': 'N',
|
||||
'\u039f': 'O',
|
||||
'\u03a1': 'P',
|
||||
'\u03a4': 'T',
|
||||
'\u03a5': 'Y',
|
||||
'\u03a7': 'X',
|
||||
'\u03b1': 'a',
|
||||
'\u03b5': 'e',
|
||||
'\u03b9': 'i',
|
||||
'\u03bf': 'o',
|
||||
'\u03c1': 'p',
|
||||
'\u03c7': 'x',
|
||||
'\u03f2': 'c',
|
||||
'\u03f9': 'C',
|
||||
'\u0405': 'S',
|
||||
'\u0406': 'I',
|
||||
'\u0408': 'J',
|
||||
'\u0410': 'A',
|
||||
'\u0412': 'B',
|
||||
'\u0415': 'E',
|
||||
'\u041a': 'K',
|
||||
'\u041c': 'M',
|
||||
'\u041d': 'H',
|
||||
'\u041e': 'O',
|
||||
'\u0420': 'P',
|
||||
'\u0421': 'C',
|
||||
'\u0422': 'T',
|
||||
'\u0425': 'X',
|
||||
'\u0430': 'a',
|
||||
'\u0435': 'e',
|
||||
'\u043e': 'o',
|
||||
'\u0440': 'p',
|
||||
'\u0441': 'c',
|
||||
'\u0443': 'y',
|
||||
'\u0445': 'x',
|
||||
'\u0455': 's',
|
||||
'\u0456': 'i',
|
||||
'\u0458': 'j',
|
||||
});
|
||||
|
||||
const MIXED_SCRIPT_CONFUSABLE_PATTERN = /[\u0391\u0392\u0395\u0396\u0397\u0399\u039a\u039c\u039d\u039f\u03a1\u03a4\u03a5\u03a7\u03b1\u03b5\u03b9\u03bf\u03c1\u03c7\u03f2\u03f9\u0405\u0406\u0408\u0410\u0412\u0415\u041a\u041c\u041d\u041e\u0420\u0421\u0422\u0425\u0430\u0435\u043e\u0440\u0441\u0443\u0445\u0455\u0456\u0458]/;
|
||||
const MIXED_SCRIPT_CONFUSABLE_REPLACE_PATTERN = new RegExp(
|
||||
MIXED_SCRIPT_CONFUSABLE_PATTERN.source,
|
||||
'g',
|
||||
);
|
||||
|
||||
function projectMixedScriptConfusables(value: string): string | undefined {
|
||||
if (!MIXED_SCRIPT_CONFUSABLE_PATTERN.test(value)) return undefined;
|
||||
let changed = false;
|
||||
const projected = value.replace(/[\p{L}\p{M}]+/gu, (token) => {
|
||||
if (!/[A-Za-z]/.test(token) || !MIXED_SCRIPT_CONFUSABLE_PATTERN.test(token)) return token;
|
||||
changed = true;
|
||||
return token.replace(
|
||||
MIXED_SCRIPT_CONFUSABLE_REPLACE_PATTERN,
|
||||
char => MIXED_SCRIPT_CONFUSABLES[char] ?? char,
|
||||
);
|
||||
});
|
||||
return changed ? projected : undefined;
|
||||
}
|
||||
|
||||
const BASE64_CANDIDATE_PATTERN = /(?:^|[^A-Za-z0-9+/_=-])([A-Za-z0-9+/_-]{24,}={0,2})(?=$|[^A-Za-z0-9+/_=-])/g;
|
||||
const MAX_BASE64_CANDIDATES = 16;
|
||||
const MAX_BASE64_CANDIDATE_CHARS = 262_144;
|
||||
const MAX_BASE64_TOTAL_CHARS = 524_288;
|
||||
const MAX_BASE64_DEPTH = 4;
|
||||
|
||||
type Base64Candidate = {
|
||||
value: string;
|
||||
start: number;
|
||||
end: number;
|
||||
};
|
||||
|
||||
function collectBase64Candidates(source: string, allowImplicitWrapped = false): {
|
||||
candidates: Base64Candidate[];
|
||||
complete: boolean;
|
||||
} {
|
||||
const wrappedBlocks: Base64Candidate[] = [];
|
||||
|
||||
const trimmed = source.trim();
|
||||
const wrappedChunks = trimmed.split(/[ \t\r\n]+/);
|
||||
const wrapWidth = wrappedChunks[0]?.length ?? 0;
|
||||
if (allowImplicitWrapped
|
||||
&& wrappedChunks.length > 1
|
||||
&& wrapWidth >= 4
|
||||
&& wrapWidth <= 76
|
||||
&& wrapWidth % 4 === 0
|
||||
&& wrappedChunks.every(chunk => /^[A-Za-z0-9+/_-]+={0,2}$/.test(chunk))
|
||||
&& wrappedChunks.slice(0, -1).every(chunk => chunk.length === wrapWidth && !chunk.includes('='))
|
||||
&& wrappedChunks.at(-1)!.length <= wrapWidth) {
|
||||
const candidate = wrappedChunks.join('');
|
||||
if (candidate.length >= 24) {
|
||||
const start = source.length - source.trimStart().length;
|
||||
wrappedBlocks.push({ value: candidate, start, end: start + trimmed.length });
|
||||
}
|
||||
}
|
||||
|
||||
const directivePattern = /(?:\bdecode\b[^\r\n:]{0,160}\bbase64\b|\bbase64\b[^\r\n:]{0,160}\bdecode\b)[^\r\n:]{0,160}:/gi;
|
||||
for (const directive of source.matchAll(directivePattern)) {
|
||||
const tailStart = (directive.index ?? 0) + directive[0].length;
|
||||
const tail = source.slice(tailStart);
|
||||
const wrapped = tail.match(/^[ \t\r\n]*([A-Za-z0-9+/_=-]+(?:[ \t\r\n]+[A-Za-z0-9+/_=-]+)*)/);
|
||||
if (!wrapped) continue;
|
||||
const captured = wrapped[1];
|
||||
const start = tailStart + wrapped[0].indexOf(captured);
|
||||
let candidate = '';
|
||||
let end = start;
|
||||
for (const chunk of captured.matchAll(/[A-Za-z0-9+/_=-]+/g)) {
|
||||
candidate += chunk[0];
|
||||
end = start + (chunk.index ?? 0) + chunk[0].length;
|
||||
if (chunk[0].includes('=')) break;
|
||||
}
|
||||
if (candidate.length >= 24) wrappedBlocks.push({ value: candidate, start, end });
|
||||
}
|
||||
|
||||
const candidates: Base64Candidate[] = [];
|
||||
const contiguousPattern = new RegExp(BASE64_CANDIDATE_PATTERN.source, 'g');
|
||||
for (const match of source.matchAll(contiguousPattern)) {
|
||||
const value = match[1];
|
||||
const start = (match.index ?? 0) + match[0].length - value.length;
|
||||
const end = start + value.length;
|
||||
if (wrappedBlocks.some(block => start >= block.start && end <= block.end)) continue;
|
||||
candidates.push({ value, start, end });
|
||||
}
|
||||
candidates.push(...wrappedBlocks);
|
||||
candidates.sort((left, right) => left.start - right.start || left.end - right.end);
|
||||
return { candidates, complete: true };
|
||||
}
|
||||
|
||||
function decodeBase64Text(candidate: string): string | undefined {
|
||||
let normalized = candidate.replace(/-/g, '+').replace(/_/g, '/');
|
||||
const remainder = normalized.length % 4;
|
||||
if (remainder === 1) return undefined;
|
||||
if (remainder > 0) normalized += '='.repeat(4 - remainder);
|
||||
|
||||
try {
|
||||
const binary = atob(normalized);
|
||||
const bytes = Uint8Array.from(binary, char => char.charCodeAt(0));
|
||||
const decoded = new TextDecoder('utf-8').decode(bytes);
|
||||
if (!decoded) return undefined;
|
||||
|
||||
let printable = 0;
|
||||
let total = 0;
|
||||
for (const char of decoded) {
|
||||
total++;
|
||||
const codePoint = char.codePointAt(0) ?? 0;
|
||||
const isPrintable = char === '\n' || char === '\r' || char === '\t'
|
||||
|| (codePoint >= 0x20 && codePoint !== 0x7f);
|
||||
if (codePoint !== 0xfffd && isPrintable) {
|
||||
printable++;
|
||||
}
|
||||
}
|
||||
return total > 0 && printable / total >= 0.85 ? decoded : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function addProjection(projections: Set<string>, value: string): void {
|
||||
if (projections.has(value)) return;
|
||||
projections.add(value);
|
||||
const confusable = projectMixedScriptConfusables(value);
|
||||
if (confusable !== undefined) projections.add(confusable);
|
||||
}
|
||||
|
||||
const FORMAT_CHARACTER_PATTERN = /\p{Cf}/u;
|
||||
const DEFAULT_IGNORABLE_PATTERN = /\p{Default_Ignorable_Code_Point}/u;
|
||||
|
||||
function isHiddenSeparator(char: string): boolean {
|
||||
const codePoint = char.codePointAt(0) ?? 0;
|
||||
if (codePoint <= 0x9f) {
|
||||
return codePoint <= 0x08
|
||||
|| codePoint === 0x0b
|
||||
|| codePoint === 0x0c
|
||||
|| (codePoint >= 0x0e && codePoint <= 0x1f)
|
||||
|| codePoint >= 0x7f;
|
||||
}
|
||||
if (codePoint >= 0xd800 && codePoint <= 0xdfff) return true;
|
||||
return FORMAT_CHARACTER_PATTERN.test(char) || DEFAULT_IGNORABLE_PATTERN.test(char);
|
||||
}
|
||||
|
||||
function replaceHiddenSeparators(value: string, replacement: string): string {
|
||||
let containsHidden = false;
|
||||
for (const char of value) {
|
||||
if (isHiddenSeparator(char)) {
|
||||
containsHidden = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!containsHidden) return value;
|
||||
|
||||
let projected = '';
|
||||
for (const char of value) projected += isHiddenSeparator(char) ? replacement : char;
|
||||
return projected;
|
||||
}
|
||||
|
||||
function projectDelimitedWords(value: string): string | undefined {
|
||||
const projected = value.replace(
|
||||
/(\p{L})([\p{P}\p{S}\p{White_Space}]+)(?=\p{L})/gu,
|
||||
(match, letter: string, separators: string) =>
|
||||
/[\p{P}\p{S}]/u.test(separators) ? `${letter} ` : match,
|
||||
);
|
||||
return projected === value ? undefined : projected;
|
||||
}
|
||||
|
||||
type HtmlTagBoundary =
|
||||
| { kind: 'close'; index: number }
|
||||
| { kind: 'nested'; index: number }
|
||||
| { kind: 'eof'; index: number };
|
||||
|
||||
function findHtmlTagBoundary(value: string, start: number): HtmlTagBoundary {
|
||||
let quote: '"' | "'" | undefined;
|
||||
for (let index = start; index < value.length; index++) {
|
||||
const char = value[index];
|
||||
if (char === '<') return { kind: 'nested', index };
|
||||
if (quote) {
|
||||
if (char === quote) quote = undefined;
|
||||
} else if (char === '"' || char === "'") {
|
||||
quote = char;
|
||||
} else if (char === '>') {
|
||||
return { kind: 'close', index };
|
||||
}
|
||||
}
|
||||
return { kind: 'eof', index: value.length };
|
||||
}
|
||||
|
||||
function normalizeHtmlToken(value: string): string {
|
||||
return value.replace(/[:_-]+/g, ' ');
|
||||
}
|
||||
|
||||
function extractHtmlAttributeTokens(value: string, start: number, end: number): string {
|
||||
const tokens: string[] = [];
|
||||
let index = start;
|
||||
while (index < end) {
|
||||
while (index < end && /[\s/]/.test(value[index])) index++;
|
||||
const nameStart = index;
|
||||
while (index < end && !/[\s=/>]/.test(value[index])) index++;
|
||||
if (index === nameStart) {
|
||||
index++;
|
||||
continue;
|
||||
}
|
||||
const name = normalizeHtmlToken(value.slice(nameStart, index));
|
||||
while (index < end && /\s/.test(value[index])) index++;
|
||||
if (value[index] !== '=') {
|
||||
if (name) tokens.push(name);
|
||||
continue;
|
||||
}
|
||||
index++;
|
||||
while (index < end && /\s/.test(value[index])) index++;
|
||||
|
||||
const quote = value[index] === '"' || value[index] === "'"
|
||||
? value[index]
|
||||
: undefined;
|
||||
if (quote) index++;
|
||||
const valueStart = index;
|
||||
if (quote) {
|
||||
while (index < end && value[index] !== quote) index++;
|
||||
} else {
|
||||
while (index < end && !/[\s>]/.test(value[index])) index++;
|
||||
}
|
||||
if (index > valueStart) tokens.push(normalizeHtmlToken(value.slice(valueStart, index)));
|
||||
if (quote && index < end) index++;
|
||||
}
|
||||
return tokens.join(' ');
|
||||
}
|
||||
|
||||
function stripHtmlMarkup(value: string): {
|
||||
rendered: string;
|
||||
tagNames: string;
|
||||
attributes: string;
|
||||
lexical: string;
|
||||
} {
|
||||
if (!value.includes('<')) {
|
||||
return { rendered: value, tagNames: value, attributes: value, lexical: value };
|
||||
}
|
||||
|
||||
const rendered: string[] = [];
|
||||
const tagNames: string[] = [];
|
||||
const attributes: string[] = [];
|
||||
const lexical: string[] = [];
|
||||
for (let index = 0; index < value.length;) {
|
||||
if (value.startsWith('<!--', index)) {
|
||||
const commentEnd = value.indexOf('-->', index + 4);
|
||||
if (commentEnd === -1) {
|
||||
const visibleTail = value.slice(index + 4);
|
||||
rendered.push(visibleTail);
|
||||
tagNames.push(visibleTail);
|
||||
attributes.push(visibleTail);
|
||||
lexical.push(visibleTail);
|
||||
break;
|
||||
}
|
||||
const commentText = value.slice(index + 4, commentEnd);
|
||||
if (commentText) {
|
||||
tagNames.push(' ', commentText, ' ');
|
||||
attributes.push(' ', commentText, ' ');
|
||||
lexical.push(' ', commentText, ' ');
|
||||
}
|
||||
index = commentEnd + 3;
|
||||
continue;
|
||||
}
|
||||
if (value.startsWith('-->', index)) {
|
||||
index += 3;
|
||||
continue;
|
||||
}
|
||||
if (value[index] === '<' && /[!/A-Za-z?]/.test(value[index + 1] ?? '')) {
|
||||
let tagNameEnd = index + 1;
|
||||
if (value[tagNameEnd] === '/') tagNameEnd++;
|
||||
const tagNameStart = tagNameEnd;
|
||||
while (/[A-Za-z0-9:!_-]/.test(value[tagNameEnd] ?? '')) tagNameEnd++;
|
||||
const tagName = normalizeHtmlToken(value.slice(tagNameStart, tagNameEnd));
|
||||
const boundary = findHtmlTagBoundary(value, tagNameEnd);
|
||||
if (boundary.kind === 'close') {
|
||||
const attributeTokens = extractHtmlAttributeTokens(
|
||||
value,
|
||||
tagNameEnd,
|
||||
boundary.index,
|
||||
);
|
||||
if (tagName) tagNames.push(' ', tagName, ' ');
|
||||
if (attributeTokens) attributes.push(' ', attributeTokens, ' ');
|
||||
if (tagName || attributeTokens) {
|
||||
lexical.push(' ', tagName, ' ', attributeTokens, ' ');
|
||||
}
|
||||
index = boundary.index + 1;
|
||||
continue;
|
||||
}
|
||||
const visibleTail = value.slice(tagNameEnd, boundary.index);
|
||||
rendered.push(visibleTail);
|
||||
if (tagName) tagNames.push(' ', tagName, ' ');
|
||||
tagNames.push(visibleTail);
|
||||
attributes.push(visibleTail);
|
||||
if (tagName) lexical.push(' ', tagName, ' ');
|
||||
lexical.push(visibleTail);
|
||||
if (boundary.kind === 'eof') break;
|
||||
index = boundary.index;
|
||||
continue;
|
||||
}
|
||||
rendered.push(value[index]);
|
||||
tagNames.push(value[index]);
|
||||
attributes.push(value[index]);
|
||||
lexical.push(value[index]);
|
||||
index++;
|
||||
}
|
||||
return {
|
||||
rendered: rendered.join(''),
|
||||
tagNames: tagNames.join(''),
|
||||
attributes: attributes.join(''),
|
||||
lexical: lexical.join(''),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizedIngressProjections(value: string): {
|
||||
projections: string[];
|
||||
complete: boolean;
|
||||
};
|
||||
function normalizedIngressProjections(value: string, includeBase64: boolean): {
|
||||
projections: string[];
|
||||
complete: boolean;
|
||||
};
|
||||
function normalizedIngressProjections(value: string, includeBase64 = true): {
|
||||
projections: string[];
|
||||
complete: boolean;
|
||||
} {
|
||||
const maxPasses = 64;
|
||||
const maxWork = MAX_EXTERNAL_MEMORY_INGRESS_CHARS + 1;
|
||||
const originalProjection = value.normalize('NFKC');
|
||||
const normalizationStages = new Set([originalProjection]);
|
||||
let decodedProjection = originalProjection;
|
||||
let complete = false;
|
||||
let work = 0;
|
||||
for (let pass = 0; pass < maxPasses; pass++) {
|
||||
work += decodedProjection.length;
|
||||
if (work > maxWork) break;
|
||||
const decodedText = decodeUnicodeEscapes(decodeHexEscapes(
|
||||
decodeHtmlEntities(decodePercentEncoding(decodedProjection)),
|
||||
));
|
||||
const decoded = decodedText === decodedProjection
|
||||
? decodedProjection
|
||||
: decodedText.normalize('NFKC');
|
||||
if (decoded === decodedProjection) {
|
||||
complete = true;
|
||||
break;
|
||||
}
|
||||
normalizationStages.add(decoded);
|
||||
decodedProjection = decoded;
|
||||
}
|
||||
if (!complete) return { projections: [decodedProjection], complete: false };
|
||||
|
||||
const htmlProjections = stripHtmlMarkup(decodedProjection);
|
||||
const projections = new Set<string>();
|
||||
for (const projection of normalizationStages) addProjection(projections, projection);
|
||||
for (const projection of new Set([
|
||||
htmlProjections.rendered,
|
||||
htmlProjections.tagNames,
|
||||
htmlProjections.attributes,
|
||||
htmlProjections.lexical,
|
||||
])) {
|
||||
let unformatted = projection;
|
||||
if (unformatted.includes('[')) {
|
||||
unformatted = unformatted
|
||||
.replace(/!\[([^\]]*)\]\([^)]*\)/g, '$1')
|
||||
.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
|
||||
.replace(/\[([^\]]+)\]\[[^\]]*\]/g, '$1');
|
||||
}
|
||||
if (/[*_~`]/.test(unformatted)) unformatted = unformatted.replace(/[*_~`]/g, '');
|
||||
const compact = replaceHiddenSeparators(unformatted, '');
|
||||
addProjection(projections, compact);
|
||||
if (compact !== unformatted) {
|
||||
addProjection(projections, replaceHiddenSeparators(unformatted, ' '));
|
||||
}
|
||||
}
|
||||
|
||||
if (!includeBase64) return { projections: [...projections], complete: true };
|
||||
|
||||
const candidateLineages = new Map<number, Map<string, number[]>>();
|
||||
let nextCandidateLineage = 1;
|
||||
const resolveCandidateLineage = (
|
||||
parentLineage: number,
|
||||
candidate: string,
|
||||
ordinal: number,
|
||||
): { lineage: number; created: boolean } => {
|
||||
let candidates = candidateLineages.get(parentLineage);
|
||||
if (!candidates) {
|
||||
candidates = new Map();
|
||||
candidateLineages.set(parentLineage, candidates);
|
||||
}
|
||||
let lineages = candidates.get(candidate);
|
||||
if (!lineages) {
|
||||
lineages = [];
|
||||
candidates.set(candidate, lineages);
|
||||
}
|
||||
const existing = lineages[ordinal];
|
||||
if (existing !== undefined) return { lineage: existing, created: false };
|
||||
const lineage = nextCandidateLineage++;
|
||||
lineages[ordinal] = lineage;
|
||||
return { lineage, created: true };
|
||||
};
|
||||
let decodedCandidateCount = 0;
|
||||
let decodedCandidateChars = 0;
|
||||
let base64Sources = [...projections].map(value => ({ value, lineage: 0 }));
|
||||
let reachedDepthLimit = true;
|
||||
for (let depth = 0; depth < MAX_BASE64_DEPTH; depth++) {
|
||||
const decodedValues: Array<{ value: string; lineage: number }> = [];
|
||||
for (const source of base64Sources) {
|
||||
const collected = collectBase64Candidates(source.value, depth > 0);
|
||||
if (!collected.complete) return { projections: [...projections], complete: false };
|
||||
const occurrenceCounts = new Map<string, number>();
|
||||
for (const occurrence of collected.candidates) {
|
||||
const candidate = occurrence.value;
|
||||
const ordinal = occurrenceCounts.get(candidate) ?? 0;
|
||||
occurrenceCounts.set(candidate, ordinal + 1);
|
||||
const occurrenceLineage = resolveCandidateLineage(
|
||||
source.lineage,
|
||||
candidate,
|
||||
ordinal,
|
||||
);
|
||||
if (!occurrenceLineage.created) continue;
|
||||
if (candidate.length > MAX_BASE64_CANDIDATE_CHARS) {
|
||||
return { projections: [...projections], complete: false };
|
||||
}
|
||||
const decoded = decodeBase64Text(candidate);
|
||||
if (decoded === undefined) continue;
|
||||
decodedCandidateCount++;
|
||||
decodedCandidateChars += candidate.length;
|
||||
if (decodedCandidateCount > MAX_BASE64_CANDIDATES
|
||||
|| decodedCandidateChars > MAX_BASE64_TOTAL_CHARS) {
|
||||
return { projections: [...projections], complete: false };
|
||||
}
|
||||
decodedValues.push({ value: decoded, lineage: occurrenceLineage.lineage });
|
||||
}
|
||||
}
|
||||
if (decodedValues.length === 0) {
|
||||
reachedDepthLimit = false;
|
||||
break;
|
||||
}
|
||||
|
||||
const nextSources: Array<{ value: string; lineage: number }> = [];
|
||||
for (const decoded of decodedValues) {
|
||||
const nested = normalizedIngressProjections(decoded.value, false);
|
||||
if (!nested.complete) return { projections: [...projections], complete: false };
|
||||
for (const projection of nested.projections) {
|
||||
addProjection(projections, projection);
|
||||
nextSources.push({ value: projection, lineage: decoded.lineage });
|
||||
}
|
||||
}
|
||||
base64Sources = nextSources;
|
||||
}
|
||||
|
||||
if (reachedDepthLimit && base64Sources.length > 0) {
|
||||
for (const source of base64Sources) {
|
||||
const collected = collectBase64Candidates(source.value, true);
|
||||
if (!collected.complete) return { projections: [...projections], complete: false };
|
||||
const occurrenceCounts = new Map<string, number>();
|
||||
for (const occurrence of collected.candidates) {
|
||||
const candidate = occurrence.value;
|
||||
const ordinal = occurrenceCounts.get(candidate) ?? 0;
|
||||
occurrenceCounts.set(candidate, ordinal + 1);
|
||||
const occurrenceSeen = candidateLineages
|
||||
.get(source.lineage)
|
||||
?.get(candidate)?.[ordinal] !== undefined;
|
||||
if (!occurrenceSeen
|
||||
&& decodeBase64Text(candidate) !== undefined) {
|
||||
return { projections: [...projections], complete: false };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { projections: [...projections], complete: true };
|
||||
}
|
||||
|
||||
/** Evaluate untrusted content before it can enter persistent memory. */
|
||||
export function evaluateExternalMemoryIngress(
|
||||
input: ExternalMemoryIngressInput,
|
||||
): ExternalMemoryIngressDecision {
|
||||
const title = input.title ?? '';
|
||||
if (title.length > MAX_EXTERNAL_MEMORY_INGRESS_CHARS
|
||||
|| input.content.length > MAX_EXTERNAL_MEMORY_INGRESS_CHARS - title.length) {
|
||||
return {
|
||||
action: 'block',
|
||||
reason: 'prompt_injection',
|
||||
scan: { safe: false, score: 0.6, flags: ['normalization_limit'] },
|
||||
};
|
||||
}
|
||||
|
||||
const projection = `${title}\n${input.content}`;
|
||||
let scan = scanForInjection(projection, 'tool_output');
|
||||
if (scan.safe) {
|
||||
const normalizedIngress = normalizedIngressProjections(projection);
|
||||
for (const normalized of normalizedIngress.projections) {
|
||||
if (normalized !== projection) {
|
||||
const normalizedScan = scanForInjection(normalized, 'tool_output');
|
||||
if (!normalizedScan.safe) {
|
||||
scan = normalizedScan;
|
||||
break;
|
||||
}
|
||||
}
|
||||
const delimited = projectDelimitedWords(normalized);
|
||||
if (delimited !== undefined) {
|
||||
const delimitedScan = scanForInjection(delimited, 'tool_output');
|
||||
if (!delimitedScan.safe) {
|
||||
scan = delimitedScan;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (scan.safe && !normalizedIngress.complete) {
|
||||
scan = { safe: false, score: 0.6, flags: ['normalization_limit'] };
|
||||
}
|
||||
}
|
||||
|
||||
return scan.safe
|
||||
? { action: 'allow', scan }
|
||||
: { action: 'block', reason: 'prompt_injection', scan };
|
||||
}
|
||||
@@ -109,6 +109,7 @@ export interface StartTraceInput {
|
||||
export interface FinalizeTraceInput {
|
||||
outcome: TraceOutcome;
|
||||
output: string;
|
||||
model?: string | null;
|
||||
reasoning?: TraceReasoningStep[];
|
||||
toolCalls?: TraceToolCall[];
|
||||
artifacts?: string[];
|
||||
@@ -158,6 +159,30 @@ const EXECUTION_TRACES_DDL: string[] = [
|
||||
`CREATE INDEX IF NOT EXISTS idx_traces_persona ON execution_traces (persona_id, outcome)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_traces_outcome ON execution_traces (outcome, created_at DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_traces_workspace ON execution_traces (workspace_id, created_at DESC)`,
|
||||
`CREATE TABLE IF NOT EXISTS execution_trace_spend (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
trace_id INTEGER NOT NULL REFERENCES execution_traces(id) ON DELETE CASCADE,
|
||||
cost_usd REAL NOT NULL CHECK (cost_usd > 0),
|
||||
settled_at TEXT NOT NULL
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_trace_spend_settled ON execution_trace_spend (settled_at, trace_id)`,
|
||||
`CREATE TABLE IF NOT EXISTS execution_trace_spend_reservations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
trace_id INTEGER NOT NULL REFERENCES execution_traces(id) ON DELETE CASCADE,
|
||||
estimated_cost_usd REAL NOT NULL CHECK (estimated_cost_usd > 0),
|
||||
actual_cost_usd REAL CHECK (actual_cost_usd IS NULL OR actual_cost_usd >= 0),
|
||||
state TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (state IN ('pending', 'settled', 'released')),
|
||||
reserved_at TEXT NOT NULL,
|
||||
resolved_at TEXT,
|
||||
CHECK (
|
||||
(state = 'pending' AND actual_cost_usd IS NULL AND resolved_at IS NULL) OR
|
||||
(state = 'settled' AND actual_cost_usd IS NOT NULL AND resolved_at IS NOT NULL) OR
|
||||
(state = 'released' AND actual_cost_usd IS NULL AND resolved_at IS NOT NULL)
|
||||
)
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_trace_spend_reservations_time
|
||||
ON execution_trace_spend_reservations (reserved_at, trace_id, state)`,
|
||||
];
|
||||
|
||||
/** Exported DDL concatenated — kept for anyone who needs the full table SQL. */
|
||||
@@ -175,10 +200,6 @@ export class ExecutionTraceStore {
|
||||
private ensureTable(): void {
|
||||
try {
|
||||
const raw = this.db.getDatabase();
|
||||
const exists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='execution_traces'",
|
||||
).get();
|
||||
if (exists) return;
|
||||
for (const stmt of EXECUTION_TRACES_DDL) {
|
||||
raw.prepare(stmt).run();
|
||||
}
|
||||
@@ -249,6 +270,141 @@ export class ExecutionTraceStore {
|
||||
.run(JSON.stringify(payload), id);
|
||||
}
|
||||
|
||||
/** Persist one settled model charge while a long-running trace is pending. */
|
||||
recordCost(id: number, costUsd: number, timestamp = new Date().toISOString()): void {
|
||||
if (!Number.isFinite(costUsd) || costUsd <= 0) {
|
||||
throw new RangeError('Trace cost entry must be a positive finite number');
|
||||
}
|
||||
const settledMs = Date.parse(timestamp);
|
||||
if (!Number.isFinite(settledMs)) {
|
||||
throw new RangeError('Trace cost timestamp must be a valid ISO date');
|
||||
}
|
||||
const settledAt = new Date(settledMs).toISOString();
|
||||
const raw = this.db.getDatabase();
|
||||
raw.transaction(() => {
|
||||
const updated = raw.prepare(`
|
||||
UPDATE execution_traces SET cost_usd = cost_usd + ? WHERE id = ?
|
||||
`).run(costUsd, id);
|
||||
if (updated.changes === 0) return;
|
||||
raw.prepare(`
|
||||
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
|
||||
VALUES (?, ?, ?)
|
||||
`).run(id, costUsd, settledAt);
|
||||
})();
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist a conservative model-spend reservation before provider dispatch.
|
||||
* Pending reservations intentionally have no settled-spend row, so restart
|
||||
* recovery counts the estimate through the reservation ledger.
|
||||
*/
|
||||
reserveCost(
|
||||
id: number,
|
||||
estimatedCostUsd: number,
|
||||
timestamp = new Date().toISOString(),
|
||||
): number {
|
||||
if (!Number.isFinite(estimatedCostUsd) || estimatedCostUsd <= 0) {
|
||||
throw new RangeError('Trace cost reservation must be positive and finite');
|
||||
}
|
||||
const reservedMs = Date.parse(timestamp);
|
||||
if (!Number.isFinite(reservedMs)) {
|
||||
throw new RangeError('Trace cost reservation timestamp must be valid ISO date');
|
||||
}
|
||||
const result = this.db.getDatabase().prepare(`
|
||||
INSERT INTO execution_trace_spend_reservations
|
||||
(trace_id, estimated_cost_usd, state, reserved_at)
|
||||
SELECT id, ?, 'pending', ?
|
||||
FROM execution_traces
|
||||
WHERE id = ? AND outcome = 'pending'
|
||||
`).run(estimatedCostUsd, new Date(reservedMs).toISOString(), id);
|
||||
if (result.changes !== 1) {
|
||||
throw new Error(`Pending execution trace ${id} does not exist`);
|
||||
}
|
||||
return Number(result.lastInsertRowid);
|
||||
}
|
||||
|
||||
/** Replace a pending estimate with one authoritative settled charge. */
|
||||
settleReservedCost(
|
||||
reservationId: number,
|
||||
actualCostUsd: number,
|
||||
timestamp = new Date().toISOString(),
|
||||
): boolean {
|
||||
if (!Number.isFinite(actualCostUsd) || actualCostUsd < 0) {
|
||||
throw new RangeError('Settled trace cost must be non-negative and finite');
|
||||
}
|
||||
const settledMs = Date.parse(timestamp);
|
||||
if (!Number.isFinite(settledMs)) {
|
||||
throw new RangeError('Trace cost timestamp must be valid ISO date');
|
||||
}
|
||||
const settledAt = new Date(settledMs).toISOString();
|
||||
const raw = this.db.getDatabase();
|
||||
return raw.transaction(() => {
|
||||
const current = raw.prepare(`
|
||||
SELECT trace_id AS traceId, reserved_at AS reservedAt,
|
||||
state, actual_cost_usd AS actualCostUsd
|
||||
FROM execution_trace_spend_reservations
|
||||
WHERE id = ?
|
||||
`).get(reservationId) as {
|
||||
traceId: number;
|
||||
reservedAt: string;
|
||||
state: 'pending' | 'settled' | 'released';
|
||||
actualCostUsd: number | null;
|
||||
} | undefined;
|
||||
if (!current) throw new Error(`Cost reservation ${reservationId} does not exist`);
|
||||
if (current.state === 'settled' && current.actualCostUsd === actualCostUsd) return false;
|
||||
if (current.state !== 'pending') {
|
||||
throw new Error(`Cost reservation ${reservationId} is already ${current.state}`);
|
||||
}
|
||||
const updated = raw.prepare(`
|
||||
UPDATE execution_trace_spend_reservations
|
||||
SET state = 'settled', actual_cost_usd = ?, resolved_at = ?
|
||||
WHERE id = ? AND state = 'pending'
|
||||
`).run(actualCostUsd, settledAt, reservationId);
|
||||
if (updated.changes !== 1) return false;
|
||||
if (actualCostUsd > 0) {
|
||||
raw.prepare(`
|
||||
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
|
||||
VALUES (?, ?, ?)
|
||||
`).run(current.traceId, actualCostUsd, current.reservedAt);
|
||||
raw.prepare(`
|
||||
UPDATE execution_traces SET cost_usd = cost_usd + ? WHERE id = ?
|
||||
`).run(actualCostUsd, current.traceId);
|
||||
}
|
||||
return true;
|
||||
})();
|
||||
}
|
||||
|
||||
/** Release a definitely pre-inference reservation without recording spend. */
|
||||
releaseReservedCost(reservationId: number, timestamp = new Date().toISOString()): boolean {
|
||||
const resolvedMs = Date.parse(timestamp);
|
||||
if (!Number.isFinite(resolvedMs)) {
|
||||
throw new RangeError('Trace cost timestamp must be valid ISO date');
|
||||
}
|
||||
const raw = this.db.getDatabase();
|
||||
return raw.transaction(() => {
|
||||
const current = raw.prepare(`
|
||||
SELECT id, state
|
||||
FROM execution_trace_spend_reservations
|
||||
WHERE id = ?
|
||||
`).get(reservationId) as {
|
||||
id: number;
|
||||
state: 'pending' | 'settled' | 'released';
|
||||
} | undefined;
|
||||
if (!current) throw new Error(`Cost reservation ${reservationId} does not exist`);
|
||||
if (current.state === 'released') return false;
|
||||
if (current.state !== 'pending') {
|
||||
throw new Error(`Cost reservation ${reservationId} is already ${current.state}`);
|
||||
}
|
||||
const result = raw.prepare(`
|
||||
UPDATE execution_trace_spend_reservations
|
||||
SET state = 'released', actual_cost_usd = NULL, resolved_at = ?
|
||||
WHERE id = ? AND state = 'pending'
|
||||
`).run(new Date(resolvedMs).toISOString(), reservationId);
|
||||
if (result.changes !== 1) return false;
|
||||
return true;
|
||||
})();
|
||||
}
|
||||
|
||||
/** Finalize a trace — set outcome, merge payload, record cost + duration. */
|
||||
finalize(id: number, input: FinalizeTraceInput): ExecutionTrace | undefined {
|
||||
const current = this.get(id);
|
||||
@@ -270,22 +426,42 @@ export class ExecutionTraceStore {
|
||||
const createdMs = Date.parse(current.created_at + 'Z');
|
||||
const now = Date.now();
|
||||
const durationMs = Number.isFinite(createdMs) ? Math.max(0, now - createdMs) : 0;
|
||||
const finalModel = input.model === undefined ? current.model : input.model;
|
||||
|
||||
this.db.getDatabase().prepare(`
|
||||
UPDATE execution_traces
|
||||
SET outcome = ?,
|
||||
trace_json = ?,
|
||||
cost_usd = ?,
|
||||
duration_ms = ?,
|
||||
finalized_at = datetime('now')
|
||||
WHERE id = ?
|
||||
`).run(
|
||||
input.outcome,
|
||||
JSON.stringify(merged),
|
||||
input.costUsd ?? current.cost_usd,
|
||||
durationMs,
|
||||
id,
|
||||
);
|
||||
const raw = this.db.getDatabase();
|
||||
raw.transaction(() => {
|
||||
const finalCost = Math.max(current.cost_usd, input.costUsd ?? current.cost_usd);
|
||||
if (input.costUsd !== undefined && finalCost > 0) {
|
||||
const ledger = raw.prepare(`
|
||||
SELECT COUNT(*) AS entries, COALESCE(SUM(cost_usd), 0) AS total
|
||||
FROM execution_trace_spend WHERE trace_id = ?
|
||||
`).get(id) as { entries: number; total: number | null };
|
||||
const missingCost = Math.max(0, finalCost - Number(ledger.total ?? 0));
|
||||
if (ledger.entries > 0 && missingCost > 0) {
|
||||
raw.prepare(`
|
||||
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
|
||||
VALUES (?, ?, ?)
|
||||
`).run(id, missingCost, new Date().toISOString());
|
||||
}
|
||||
}
|
||||
raw.prepare(`
|
||||
UPDATE execution_traces
|
||||
SET outcome = ?,
|
||||
model = ?,
|
||||
trace_json = ?,
|
||||
cost_usd = ?,
|
||||
duration_ms = ?,
|
||||
finalized_at = datetime('now')
|
||||
WHERE id = ?
|
||||
`).run(
|
||||
input.outcome,
|
||||
finalModel,
|
||||
JSON.stringify(merged),
|
||||
finalCost,
|
||||
durationMs,
|
||||
id,
|
||||
);
|
||||
})();
|
||||
|
||||
return this.get(id);
|
||||
}
|
||||
@@ -320,6 +496,42 @@ export class ExecutionTraceStore {
|
||||
return row ? toParsed(row) : undefined;
|
||||
}
|
||||
|
||||
/** Highest trace id present when a consumer starts its process-local ledger. */
|
||||
getLatestId(): number {
|
||||
const row = this.db.getDatabase().prepare(
|
||||
'SELECT COALESCE(MAX(id), 0) AS id FROM execution_traces',
|
||||
).get() as { id: number | null };
|
||||
return Number(row.id ?? 0);
|
||||
}
|
||||
|
||||
/** Sum persisted model cost from a timestamp through an inclusive trace-id boundary. */
|
||||
getTotalCostSince(since: string, throughId: number = Number.MAX_SAFE_INTEGER): number {
|
||||
const settledSince = new Date(since).toISOString();
|
||||
const raw = this.db.getDatabase();
|
||||
const ledger = raw.prepare(`
|
||||
SELECT COALESCE(SUM(s.cost_usd), 0) AS total
|
||||
FROM execution_trace_spend s
|
||||
JOIN execution_traces t ON t.id = s.trace_id
|
||||
WHERE s.settled_at >= ? AND t.id <= ?
|
||||
`).get(settledSince, throughId) as { total: number | null };
|
||||
const pending = raw.prepare(`
|
||||
SELECT COALESCE(SUM(estimated_cost_usd), 0) AS total
|
||||
FROM execution_trace_spend_reservations
|
||||
WHERE state = 'pending' AND reserved_at >= ? AND trace_id <= ?
|
||||
`).get(settledSince, throughId) as { total: number | null };
|
||||
const legacy = raw.prepare(`
|
||||
SELECT COALESCE(SUM(t.cost_usd), 0) AS total
|
||||
FROM execution_traces t
|
||||
WHERE t.created_at >= datetime(?) AND t.id <= ?
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM execution_trace_spend s WHERE s.trace_id = t.id
|
||||
)
|
||||
`).get(since, throughId) as { total: number | null };
|
||||
return Number(ledger.total ?? 0)
|
||||
+ Number(pending.total ?? 0)
|
||||
+ Number(legacy.total ?? 0);
|
||||
}
|
||||
|
||||
/** Query traces with optional filters. */
|
||||
query(filter: TraceQueryFilter = {}): ExecutionTrace[] {
|
||||
const clauses: string[] = [];
|
||||
|
||||
@@ -70,6 +70,22 @@ export class FrameStore {
|
||||
this.db = db;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a write unit atomically. Top-level callers acquire the write lock up
|
||||
* front and retry the whole closure on transient cross-process contention.
|
||||
* Nested callers use better-sqlite3's savepoint behavior and never retry an
|
||||
* inner closure against the same ambient snapshot.
|
||||
*/
|
||||
runInTransaction<T>(fn: () => T): T {
|
||||
if (typeof fn !== 'function') {
|
||||
throw new TypeError('FrameStore.runInTransaction requires a function');
|
||||
}
|
||||
const raw = this.db.getDatabase();
|
||||
const transaction = raw.transaction(fn);
|
||||
if (raw.inTransaction) return transaction();
|
||||
return this.db.runWithBusyRetry(() => transaction.immediate());
|
||||
}
|
||||
|
||||
createIFrame(
|
||||
gopId: string,
|
||||
content: string,
|
||||
@@ -154,6 +170,12 @@ export class FrameStore {
|
||||
return this.db.getDatabase().prepare('SELECT * FROM memory_frames WHERE id = ?').get(id) as MemoryFrame | undefined;
|
||||
}
|
||||
|
||||
hasSession(gopId: string): boolean {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT 1 FROM sessions WHERE gop_id = ? LIMIT 1',
|
||||
).get(gopId) !== undefined;
|
||||
}
|
||||
|
||||
getLatestIFrame(gopId: string): MemoryFrame | undefined {
|
||||
return this.db.getDatabase().prepare(`
|
||||
SELECT * FROM memory_frames
|
||||
@@ -287,6 +309,13 @@ export class FrameStore {
|
||||
if (!existing) return undefined;
|
||||
|
||||
const newImportance = importance ?? existing.importance;
|
||||
if (content === existing.content) {
|
||||
if (newImportance !== existing.importance) {
|
||||
raw.prepare('UPDATE memory_frames SET importance = ? WHERE id = ?')
|
||||
.run(newImportance, id);
|
||||
}
|
||||
return this.getById(id);
|
||||
}
|
||||
|
||||
// Update main table (content_hash maintained — oss-drift D3)
|
||||
raw.prepare(`
|
||||
|
||||
@@ -2,13 +2,14 @@
|
||||
* In-process embedder using @huggingface/transformers (ONNX Runtime).
|
||||
* Default provider for ALL desktop users — zero config, works offline.
|
||||
* Model: Xenova/all-MiniLM-L6-v2 (384 native dims, normalized to target dims).
|
||||
* Downloads ~23MB model on first use, cached in ~/.waggle/models/.
|
||||
* Downloads ~90MB fp32 model on first use, cached in ~/.waggle/models/.
|
||||
*/
|
||||
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import type { Embedder } from './embeddings.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
import { withTransformersModelLoad } from './transformers-model-load.js';
|
||||
|
||||
const log = createCoreLogger('inprocess-embedder');
|
||||
|
||||
@@ -32,13 +33,18 @@ export async function createInProcessEmbedder(config?: Partial<InProcessEmbedder
|
||||
const cacheDir = config?.cacheDir ?? path.join(os.homedir(), '.waggle', 'models');
|
||||
const targetDims = config?.targetDimensions ?? 1024;
|
||||
|
||||
log.info(`Loading in-process embedding model: ${model} (~23MB first download)`);
|
||||
log.info(`Loading in-process embedding model: ${model} (~90MB fp32 first download)`);
|
||||
|
||||
const { pipeline, env } = await import('@huggingface/transformers');
|
||||
env.cacheDir = cacheDir;
|
||||
env.allowRemoteModels = true;
|
||||
|
||||
const extractor = await pipeline('feature-extraction', model, { dtype: 'fp32' });
|
||||
const { pipeline } = await import('@huggingface/transformers');
|
||||
const extractor = await withTransformersModelLoad({
|
||||
cacheDir,
|
||||
model,
|
||||
load: (canonicalCacheDir) => pipeline('feature-extraction', model, {
|
||||
dtype: 'fp32',
|
||||
cache_dir: canonicalCacheDir,
|
||||
}),
|
||||
onQuarantine: () => log.warn(`Quarantined corrupt embedding model cache: ${model}`),
|
||||
});
|
||||
const nativeDims = 384; // all-MiniLM-L6-v2 output dimensions
|
||||
|
||||
log.info(`In-process embedder ready (${nativeDims} native dims → ${targetDims} normalized)`);
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
import { withTransformersModelLoad } from './transformers-model-load.js';
|
||||
|
||||
const log = createCoreLogger('inprocess-reranker');
|
||||
|
||||
@@ -56,18 +57,25 @@ export async function createInProcessReranker(
|
||||
|
||||
log.info(`Loading in-process reranker: ${model} (~22MB first download)`);
|
||||
|
||||
const { AutoTokenizer, AutoModelForSequenceClassification, env } = await import(
|
||||
const { AutoTokenizer, AutoModelForSequenceClassification } = await import(
|
||||
'@huggingface/transformers'
|
||||
);
|
||||
env.cacheDir = cacheDir;
|
||||
env.allowRemoteModels = true;
|
||||
|
||||
// Cross-encoders need direct tokenizer + model access — pipeline API
|
||||
// doesn't expose the (text, text_pair) input pattern cleanly across
|
||||
// all transformers.js versions. Calling the model directly with
|
||||
// tokenized pairs is the stable path.
|
||||
const tokenizer = await AutoTokenizer.from_pretrained(model);
|
||||
const seqModel = await AutoModelForSequenceClassification.from_pretrained(model, { dtype: 'fp32' });
|
||||
const { tokenizer, seqModel } = await withTransformersModelLoad({
|
||||
cacheDir,
|
||||
model,
|
||||
load: async (canonicalCacheDir) => ({
|
||||
// Cross-encoders need direct tokenizer + model access — pipeline API
|
||||
// doesn't expose the (text, text_pair) input pattern cleanly across
|
||||
// all transformers.js versions. Calling the model directly with
|
||||
// tokenized pairs is the stable path.
|
||||
tokenizer: await AutoTokenizer.from_pretrained(model, { cache_dir: canonicalCacheDir }),
|
||||
seqModel: await AutoModelForSequenceClassification.from_pretrained(model, {
|
||||
dtype: 'fp32',
|
||||
cache_dir: canonicalCacheDir,
|
||||
}),
|
||||
}),
|
||||
onQuarantine: () => log.warn(`Quarantined corrupt reranker model cache: ${model}`),
|
||||
});
|
||||
|
||||
log.info(`In-process reranker ready: ${model}`);
|
||||
|
||||
@@ -77,7 +85,7 @@ export async function createInProcessReranker(
|
||||
text_pair: doc,
|
||||
padding: true,
|
||||
truncation: true,
|
||||
return_tensors: 'pt',
|
||||
return_tensor: true,
|
||||
});
|
||||
const out = await seqModel(inputs);
|
||||
// ms-marco-MiniLM outputs a single logit per pair (1-class regression).
|
||||
@@ -105,7 +113,7 @@ export async function createInProcessReranker(
|
||||
text_pair: docs,
|
||||
padding: true,
|
||||
truncation: true,
|
||||
return_tensors: 'pt',
|
||||
return_tensor: true,
|
||||
});
|
||||
const out = await seqModel(inputs);
|
||||
const logits = out.logits ?? out[0];
|
||||
|
||||
@@ -59,6 +59,17 @@ export class KnowledgeGraph {
|
||||
this.db = db;
|
||||
}
|
||||
|
||||
/** Run a graph write unit atomically, nesting as a savepoint when needed. */
|
||||
runInTransaction<T>(fn: () => T): T {
|
||||
if (typeof fn !== 'function') {
|
||||
throw new TypeError('KnowledgeGraph.runInTransaction requires a function');
|
||||
}
|
||||
const raw = this.db.getDatabase();
|
||||
const transaction = raw.transaction(fn);
|
||||
if (raw.inTransaction) return transaction();
|
||||
return this.db.runWithBusyRetry(() => transaction.immediate());
|
||||
}
|
||||
|
||||
setValidationSchema(schema: ValidationSchema): void {
|
||||
this.schema = schema;
|
||||
}
|
||||
@@ -381,11 +392,16 @@ export class KnowledgeGraph {
|
||||
/** Link an entity to a frame it was extracted from (kg_entity_frames bridge).
|
||||
* Powers the 'contextual' scoring signal. Idempotent per (entity, frame). */
|
||||
linkEntityToFrame(entityId: number, frameId: number): void {
|
||||
try {
|
||||
this.db.getDatabase().prepare(
|
||||
'INSERT OR IGNORE INTO kg_entity_frames (entity_id, frame_id) VALUES (?, ?)'
|
||||
).run(entityId, frameId);
|
||||
} catch { /* bridge table absent on a pre-migration DB — best-effort */ }
|
||||
try { this.linkEntityToFrameStrict(entityId, frameId); } catch {
|
||||
/* bridge table absent on a pre-migration DB — best-effort */
|
||||
}
|
||||
}
|
||||
|
||||
/** Strict provenance link for atomic writers; true only for a new link. */
|
||||
linkEntityToFrameStrict(entityId: number, frameId: number): boolean {
|
||||
return this.db.getDatabase().prepare(
|
||||
'INSERT OR IGNORE INTO kg_entity_frames (entity_id, frame_id) VALUES (?, ?)'
|
||||
).run(entityId, frameId).changes === 1;
|
||||
}
|
||||
|
||||
/** Seed entities whose name appears in free text (case-insensitive, name ≥3
|
||||
|
||||
@@ -123,7 +123,7 @@ export async function fetchRawDetailLane(
|
||||
const excludeIds = opts.excludeIds ?? new Set<number>();
|
||||
|
||||
// ── Pool ──────────────────────────────────────────────────────────────
|
||||
let pool: FrameRow[] = [];
|
||||
let pool: FrameRow[];
|
||||
if (opts.window) {
|
||||
pool = windowPool(db, opts.window.since, opts.window.until);
|
||||
if (pool.length > WINDOW_POOL_MAX) {
|
||||
|
||||
@@ -3,7 +3,7 @@ import type { Embedder } from './embeddings.js';
|
||||
import type { MemoryFrame, Importance } from './frames.js';
|
||||
import type { Reranker } from './inprocess-reranker.js';
|
||||
import { chunkText, type ChunkOptions } from './chunker.js';
|
||||
import { buildFtsOrQuery, hasUnsegmentedScript, sanitizeFtsToken } from './fts-sanitize.js';
|
||||
import { buildFtsOrQuery, FTS_STOP_WORDS, hasUnsegmentedScript } from './fts-sanitize.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
import {
|
||||
computeRelevance,
|
||||
@@ -86,6 +86,7 @@ export function assessRetrievalConfidence(
|
||||
}
|
||||
|
||||
const RRF_K = 60;
|
||||
const MAX_PUNCTUATED_FTS_TERMS = 16;
|
||||
|
||||
const log = createCoreLogger('hybrid-search');
|
||||
|
||||
@@ -121,6 +122,27 @@ function escapeLikeTerm(term: string): string {
|
||||
return term.replace(/[\\%_]/g, ch => `\\${ch}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a strict fallback MATCH expression from punctuation-delimited terms.
|
||||
* Unlike the primary recall-oriented OR query, every surviving term is
|
||||
* required. Refuse overlong expressions instead of truncating them into a
|
||||
* broader query.
|
||||
*/
|
||||
function buildPunctuatedFtsAndQuery(query: string, minimumTerms = 2): string {
|
||||
const tokens = query.match(/[\p{L}\p{N}_]+/gu) ?? [];
|
||||
const terms = tokens.filter((token) => (
|
||||
token.length > 2
|
||||
&& !FTS_STOP_WORDS.has(token.toLowerCase())
|
||||
&& !hasUnsegmentedScript(token)
|
||||
));
|
||||
|
||||
if (terms.length < minimumTerms || terms.length > MAX_PUNCTUATED_FTS_TERMS) return '';
|
||||
|
||||
const uniqueTerms = [...new Set(terms)];
|
||||
if (uniqueTerms.length < minimumTerms) return '';
|
||||
return uniqueTerms.map(term => `"${term}"`).join(' AND ');
|
||||
}
|
||||
|
||||
export class HybridSearch {
|
||||
private db: MindDB;
|
||||
private embedder: Embedder;
|
||||
@@ -180,13 +202,13 @@ export class HybridSearch {
|
||||
// Flag off → chunkResults is null without touching the chunk tables,
|
||||
// so the lane below is byte-identical to pre-D1.
|
||||
const chunkResults = chunkRetrievalEnabled()
|
||||
? await this.vectorSearchChunks(query, laneFetch, gopId)
|
||||
? await this.vectorSearchChunks(query, laneFetch, gopId, options.excludeDeprecated)
|
||||
: null;
|
||||
const [keywordResults, vectorResults] = await Promise.all([
|
||||
this.keywordSearch(query, laneFetch, gopId),
|
||||
this.keywordSearch(query, laneFetch, gopId, options.excludeDeprecated),
|
||||
chunkResults !== null
|
||||
? Promise.resolve(chunkResults)
|
||||
: this.vectorSearch(query, laneFetch, gopId),
|
||||
: this.vectorSearch(query, laneFetch, gopId, options.excludeDeprecated),
|
||||
]);
|
||||
|
||||
// RRF fusion
|
||||
@@ -321,7 +343,12 @@ export class HybridSearch {
|
||||
return results.slice(0, limit);
|
||||
}
|
||||
|
||||
async keywordSearch(query: string, limit: number, gopId?: string): Promise<number[]> {
|
||||
async keywordSearch(
|
||||
query: string,
|
||||
limit: number,
|
||||
gopId?: string,
|
||||
excludeDeprecated = false,
|
||||
): Promise<number[]> {
|
||||
const raw = this.db.getDatabase();
|
||||
|
||||
// W3.6: Sanitize query for FTS5 with OR-based matching for better recall.
|
||||
@@ -339,21 +366,29 @@ export class HybridSearch {
|
||||
// unsegmented script (CJK) the emptiness is a sanitizer artifact, not a
|
||||
// lack of signal: unicode61 cannot token-match CJK prose, but LIKE
|
||||
// substring matching can, so route those to the fallback lane.
|
||||
return hasUnsegmentedScript(query) ? this.likeFallbackSearch(query, limit, gopId) : [];
|
||||
return hasUnsegmentedScript(query)
|
||||
? this.likeFallbackSearch(query, limit, gopId, excludeDeprecated)
|
||||
: [];
|
||||
}
|
||||
|
||||
let sql: string;
|
||||
let params: unknown[];
|
||||
|
||||
if (gopId) {
|
||||
sql = `
|
||||
SELECT mf.id FROM memory_frames_fts fts
|
||||
JOIN memory_frames mf ON mf.id = fts.rowid
|
||||
WHERE fts.content MATCH ? AND mf.gop_id = ?
|
||||
ORDER BY rank
|
||||
WHERE fts.content MATCH ? AND mf.gop_id = ?${excludeDeprecated ? " AND mf.importance != 'deprecated'" : ''}
|
||||
ORDER BY fts.rank
|
||||
LIMIT ?
|
||||
`;
|
||||
} else if (excludeDeprecated) {
|
||||
sql = `
|
||||
SELECT mf.id FROM memory_frames_fts fts
|
||||
JOIN memory_frames mf ON mf.id = fts.rowid
|
||||
WHERE fts.content MATCH ? AND mf.importance != 'deprecated'
|
||||
ORDER BY fts.rank
|
||||
LIMIT ?
|
||||
`;
|
||||
params = [safeQuery, gopId, limit];
|
||||
} else {
|
||||
sql = `
|
||||
SELECT rowid as id FROM memory_frames_fts
|
||||
@@ -361,68 +396,112 @@ export class HybridSearch {
|
||||
ORDER BY rank
|
||||
LIMIT ?
|
||||
`;
|
||||
params = [safeQuery, limit];
|
||||
}
|
||||
|
||||
try {
|
||||
const runMatch = (matchQuery: string): number[] => {
|
||||
const params = gopId
|
||||
? [matchQuery, gopId, limit]
|
||||
: [matchQuery, limit];
|
||||
const rows = raw.prepare(sql).all(...params) as { id: number }[];
|
||||
return rows.map(r => r.id);
|
||||
return rows.map(row => row.id);
|
||||
};
|
||||
|
||||
const runPunctuationFallback = (allowSingleTerm = false): number[] => {
|
||||
// Preserve the complete identifier first. This is the most precise lane
|
||||
// and the only safe behavior when the token count exceeds the FTS bound.
|
||||
const literalIds = this.likeFallbackSearch(query, limit, gopId, excludeDeprecated);
|
||||
if (literalIds.length > 0) return literalIds;
|
||||
|
||||
// SQLite LIKE only case-folds ASCII. A strict unicode61 MATCH over every
|
||||
// punctuation-delimited term supplies Unicode case-insensitive recall
|
||||
// without broad OR matches.
|
||||
const boundaryQuery = buildPunctuatedFtsAndQuery(query, allowSingleTerm ? 1 : 2);
|
||||
if (!boundaryQuery) return [];
|
||||
try {
|
||||
return runMatch(boundaryQuery);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
const ids = runMatch(safeQuery);
|
||||
if (ids.length === 0 && /[^\p{L}\p{N}_\s]/u.test(query)) {
|
||||
return runPunctuationFallback();
|
||||
}
|
||||
return ids;
|
||||
} catch {
|
||||
// FTS5 parse error (e.g. user query with FTS5-special chars that survived
|
||||
// sanitization) — fall back to a LIKE keyword scan over the same column so
|
||||
// we return best-effort matches instead of a false "no memory found".
|
||||
return this.likeFallbackSearch(query, limit, gopId);
|
||||
// FTS5 parse error (for example, an unmatched quote): retry through the
|
||||
// same precise literal-plus-strict-boundary fallback used for zero hits.
|
||||
return runPunctuationFallback(true);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* LIKE-based keyword fallback over memory_frames.content. Used when the FTS5
|
||||
* MATCH query throws a parse error (e.g. an unbalanced quote or other FTS5
|
||||
* operator the user typed literally). The raw query is split into word tokens
|
||||
* — stripping the punctuation that caused the FTS5 error, mirroring the
|
||||
* primary sanitizer — and matched with OR-ed LIKE clauses for best-effort
|
||||
* recall. Bound parameters only (the term is never interpolated) and LIKE
|
||||
* metachars (`%`, `_`, `\`) are escaped with an ESCAPE clause so each token
|
||||
* matches literally. If no usable token survives, a single literal LIKE over
|
||||
* the whole escaped query is used.
|
||||
* Whole-query LIKE fallback over memory_frames.content. Bound parameters only
|
||||
* (the term is never interpolated), with LIKE metachars (`%`, `_`, `\`)
|
||||
* escaped so punctuation-delimited identifiers stay literal. Unicode
|
||||
* case-insensitive fallback is handled separately by strict unicode61 FTS.
|
||||
*/
|
||||
private likeFallbackSearch(query: string, limit: number, gopId?: string): number[] {
|
||||
private likeFallbackSearch(
|
||||
query: string,
|
||||
limit: number,
|
||||
gopId?: string,
|
||||
excludeDeprecated = false,
|
||||
): number[] {
|
||||
const raw = this.db.getDatabase();
|
||||
|
||||
const tokens = query
|
||||
.split(/\s+/)
|
||||
.map(sanitizeFtsToken) // strip punctuation (incl. FTS5 operators), Unicode-aware
|
||||
.filter(w => w.length > 0);
|
||||
const terms = (tokens.length > 0 ? tokens : [query]).map(t => `%${escapeLikeTerm(t)}%`);
|
||||
|
||||
const likeClause = terms.map(() => `content LIKE ? ESCAPE '\\'`).join(' OR ');
|
||||
const term = `%${escapeLikeTerm(query)}%`;
|
||||
const deprecatedFilter = excludeDeprecated ? " AND importance != 'deprecated'" : '';
|
||||
|
||||
try {
|
||||
if (gopId) {
|
||||
const rows = raw.prepare(
|
||||
`SELECT id FROM memory_frames
|
||||
WHERE (${likeClause}) AND gop_id = ?
|
||||
WHERE content LIKE ? ESCAPE '\\' AND gop_id = ?${deprecatedFilter}
|
||||
ORDER BY created_at DESC LIMIT ?`
|
||||
).all(...terms, gopId, limit) as { id: number }[];
|
||||
).all(term, gopId, limit) as { id: number }[];
|
||||
return rows.map(r => r.id);
|
||||
}
|
||||
const rows = raw.prepare(
|
||||
`SELECT id FROM memory_frames
|
||||
WHERE (${likeClause})
|
||||
WHERE content LIKE ? ESCAPE '\\'${deprecatedFilter}
|
||||
ORDER BY created_at DESC LIMIT ?`
|
||||
).all(...terms, limit) as { id: number }[];
|
||||
).all(term, limit) as { id: number }[];
|
||||
return rows.map(r => r.id);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async vectorSearch(query: string, limit: number, gopId?: string): Promise<number[]> {
|
||||
async vectorSearch(
|
||||
query: string,
|
||||
limit: number,
|
||||
gopId?: string,
|
||||
excludeDeprecated = false,
|
||||
): Promise<number[]> {
|
||||
this.ensureFingerprint();
|
||||
const embedding = await this.embedder.embed(query);
|
||||
const blob = f32ToBlob(embedding);
|
||||
const raw = this.db.getDatabase();
|
||||
|
||||
if (excludeDeprecated) {
|
||||
const gopFilter = gopId ? ' AND gop_id = ?' : '';
|
||||
try {
|
||||
const rows = raw.prepare(`
|
||||
SELECT rowid as id FROM memory_frames_vec
|
||||
WHERE embedding MATCH ? AND k = ?
|
||||
AND rowid IN (
|
||||
SELECT id FROM memory_frames
|
||||
WHERE importance != 'deprecated'${gopFilter}
|
||||
)
|
||||
ORDER BY distance
|
||||
`).all(blob, limit, ...(gopId ? [gopId] : [])) as { id: number }[];
|
||||
return rows.map((row) => row.id);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
if (gopId) {
|
||||
// Two-step: get candidates from vec, then filter by GOP
|
||||
try {
|
||||
@@ -597,7 +676,12 @@ export class HybridSearch {
|
||||
* chunk index is empty (or the tables are missing), returns null so callers
|
||||
* can cleanly fall back to the whole-frame vectorSearch path.
|
||||
*/
|
||||
async vectorSearchChunks(query: string, limit: number, gopId?: string): Promise<number[] | null> {
|
||||
async vectorSearchChunks(
|
||||
query: string,
|
||||
limit: number,
|
||||
gopId?: string,
|
||||
excludeDeprecated = false,
|
||||
): Promise<number[] | null> {
|
||||
this.ensureFingerprint();
|
||||
const raw = this.db.getDatabase();
|
||||
// Cheap probe — avoid embedding the query when chunks aren't populated.
|
||||
@@ -618,15 +702,28 @@ export class HybridSearch {
|
||||
// Over-fetch chunks (limit * 5) so dedup-to-frame still leaves enough
|
||||
// candidates after collapsing multiple chunks of the same frame.
|
||||
try {
|
||||
const gopFilter = gopId ? ' AND mf.gop_id = ?' : '';
|
||||
const candidateFilter = excludeDeprecated
|
||||
? ` AND v.rowid IN (
|
||||
SELECT c2.id
|
||||
FROM memory_frame_chunks c2
|
||||
JOIN memory_frames mf ON mf.id = c2.frame_id
|
||||
WHERE mf.importance != 'deprecated'${gopFilter}
|
||||
)`
|
||||
: '';
|
||||
const chunkRows = raw
|
||||
.prepare(
|
||||
`SELECT v.rowid AS chunk_id, c.frame_id
|
||||
FROM memory_frame_chunks_vec v
|
||||
JOIN memory_frame_chunks c ON c.id = v.rowid
|
||||
WHERE v.embedding MATCH ? AND k = ?
|
||||
WHERE v.embedding MATCH ? AND k = ?${candidateFilter}
|
||||
ORDER BY distance`
|
||||
)
|
||||
.all(blob, Math.max(limit * 5, 25)) as Array<{ chunk_id: number; frame_id: number }>;
|
||||
.all(
|
||||
blob,
|
||||
Math.max(limit * 5, 25),
|
||||
...(excludeDeprecated && gopId ? [gopId] : []),
|
||||
) as Array<{ chunk_id: number; frame_id: number }>;
|
||||
|
||||
if (chunkRows.length === 0) return [];
|
||||
|
||||
@@ -640,7 +737,7 @@ export class HybridSearch {
|
||||
if (frameIds.length >= limit) break;
|
||||
}
|
||||
|
||||
if (gopId) {
|
||||
if (gopId && !excludeDeprecated) {
|
||||
const placeholders = frameIds.map(() => '?').join(',');
|
||||
const filtered = raw
|
||||
.prepare(
|
||||
|
||||
@@ -51,6 +51,19 @@
|
||||
|
||||
import type { MindDB } from './db.js';
|
||||
import type { FrameStore, FrameSource, MemoryFrame } from './frames.js';
|
||||
import { evaluateExternalMemoryIngress } from '../memory-ingress-guard.js';
|
||||
|
||||
export const MAX_CONSOLIDATION_OBSERVATIONS = 400;
|
||||
const MAX_PROMPT_CHARS = 100_000;
|
||||
const MAX_RESPONSE_CHARS = 100_000;
|
||||
const MAX_LABEL_CHARS = 256;
|
||||
const MAX_CURRENT_VALUE_CHARS = 4_000;
|
||||
const CREATED_AT_SORT_EXPR = `julianday(CASE
|
||||
WHEN created_at GLOB '*[+-][0-9][0-9][0-9][0-9]'
|
||||
THEN substr(created_at, 1, length(created_at) - 5)
|
||||
|| substr(created_at, -5, 3) || ':' || substr(created_at, -2)
|
||||
ELSE created_at
|
||||
END)`;
|
||||
|
||||
/**
|
||||
* LLM callback the consolidation passes inject. Given a system + user message,
|
||||
@@ -98,7 +111,7 @@ export interface ConsolidationResult {
|
||||
export interface CollectObservationsOptions {
|
||||
/** Scope to a single GOP session; omit for the whole mind. */
|
||||
gopId?: string;
|
||||
/** Cap the number of observations (keeps the LLM prompt bounded). */
|
||||
/** Select the latest N eligible observations, returned chronologically. */
|
||||
limit?: number;
|
||||
/**
|
||||
* Which frame source to include. Defaults to 'agent_inferred' (the
|
||||
@@ -124,19 +137,26 @@ const GROUP_SYSTEM =
|
||||
* an empty object when nothing parses — the callers treat "no intents" as a
|
||||
* valid, non-fatal outcome rather than throwing on model chatter.
|
||||
*/
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function parseLlmJson(raw: string): Record<string, unknown> {
|
||||
if (typeof raw !== 'string') return {};
|
||||
if (raw.length > MAX_RESPONSE_CHARS) return {};
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed) return {};
|
||||
try {
|
||||
return JSON.parse(trimmed) as Record<string, unknown>;
|
||||
const parsed: unknown = JSON.parse(trimmed);
|
||||
return isRecord(parsed) ? parsed : {};
|
||||
} catch {
|
||||
// Not a bare JSON document — try to recover an embedded object below.
|
||||
}
|
||||
const match = trimmed.match(/\{[\s\S]*\}/);
|
||||
if (match) {
|
||||
try {
|
||||
return JSON.parse(match[0]) as Record<string, unknown>;
|
||||
const parsed: unknown = JSON.parse(match[0]);
|
||||
return isRecord(parsed) ? parsed : {};
|
||||
} catch {
|
||||
// Embedded block was also malformed — fall through to the empty result.
|
||||
}
|
||||
@@ -154,20 +174,83 @@ function assertObservations(observations: unknown): asserts observations is Obse
|
||||
throw new TypeError('consolidate: each observation must be an object');
|
||||
}
|
||||
const rec = o as Record<string, unknown>;
|
||||
if (!Number.isInteger(rec.id)) {
|
||||
throw new TypeError('consolidate: observation.id must be an integer');
|
||||
if (!Number.isSafeInteger(rec.id) || Number(rec.id) <= 0) {
|
||||
throw new TypeError('consolidate: observation.id must be a positive safe integer');
|
||||
}
|
||||
if (typeof rec.content !== 'string') {
|
||||
throw new TypeError('consolidate: observation.content must be a string');
|
||||
}
|
||||
if (typeof rec.created_at !== 'string') {
|
||||
throw new TypeError('consolidate: observation.created_at must be a string');
|
||||
}
|
||||
if (!Number.isFinite(observationTime(rec.created_at))) {
|
||||
throw new TypeError('consolidate: observation.created_at must be a valid timestamp');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Render the observations as a `N. [YYYY-MM-DD] content` numbered list. */
|
||||
function numberObservations(observations: Observation[]): string {
|
||||
return observations
|
||||
.map((o, idx) => `${idx + 1}. [${String(o.created_at ?? '').slice(0, 10)}] ${o.content}`)
|
||||
.join('\n');
|
||||
const SQLITE_TIMESTAMP_RE =
|
||||
/^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})(?:\.(\d+))?(?:Z|([+-])(\d{2}):?(\d{2}))?$/;
|
||||
|
||||
function observationTime(value: string): number {
|
||||
const match = SQLITE_TIMESTAMP_RE.exec(value.trim());
|
||||
if (!match) return Number.NaN;
|
||||
|
||||
const [, date, time, fraction = '', sign, hour, minute] = match;
|
||||
if (hour && (Number(hour) > 14 || Number(minute) > 59)) return Number.NaN;
|
||||
|
||||
const firstThree = Number((fraction + '000').slice(0, 3));
|
||||
const millis = Math.min(999, firstThree + ((fraction[3] ?? '0') >= '5' ? 1 : 0));
|
||||
const zone = sign ? `${sign}${hour}:${minute}` : 'Z';
|
||||
return Date.parse(`${date}T${time}.${String(millis).padStart(3, '0')}${zone}`);
|
||||
}
|
||||
|
||||
function normalizeObservations(observations: Observation[]): Observation[] {
|
||||
const sorted = [...observations].sort((a, b) => {
|
||||
const timeDelta = observationTime(a.created_at) - observationTime(b.created_at);
|
||||
return timeDelta || a.id - b.id;
|
||||
});
|
||||
const seen = new Set<number>();
|
||||
return sorted.filter(({ id }) => {
|
||||
if (seen.has(id)) return false;
|
||||
seen.add(id);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function prepareDetectionPrompt(
|
||||
observations: Observation[],
|
||||
system: string,
|
||||
operation: string,
|
||||
): { normalized: Observation[]; user: string } {
|
||||
assertObservations(observations);
|
||||
if (observations.length > MAX_CONSOLIDATION_OBSERVATIONS) {
|
||||
throw new RangeError(
|
||||
`${operation}: at most ${MAX_CONSOLIDATION_OBSERVATIONS} observations are allowed`,
|
||||
);
|
||||
}
|
||||
const normalized = normalizeObservations(observations);
|
||||
const lines: string[] = [];
|
||||
let userChars = 0;
|
||||
for (let index = 0; index < normalized.length; index += 1) {
|
||||
const observation = normalized[index];
|
||||
const prefix = `${index + 1}. [${observation.created_at.slice(0, 10)}] `;
|
||||
const addedChars = (index > 0 ? 1 : 0) + prefix.length + observation.content.length;
|
||||
if (system.length + userChars + addedChars > MAX_PROMPT_CHARS) {
|
||||
throw new RangeError(`${operation}: prompt exceeds ${MAX_PROMPT_CHARS} characters`);
|
||||
}
|
||||
lines.push(prefix + observation.content);
|
||||
userChars += addedChars;
|
||||
}
|
||||
return { normalized, user: lines.join('\n') };
|
||||
}
|
||||
|
||||
function safeModelText(value: unknown, maxChars: number): string | null {
|
||||
if (typeof value !== 'string') return '';
|
||||
const text = value.trim();
|
||||
if (text.length > maxChars) return null;
|
||||
if (text && evaluateExternalMemoryIngress({ content: text }).action !== 'allow') return null;
|
||||
return text;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -186,8 +269,9 @@ function mapNumbersToFrameIds(
|
||||
const indices: number[] = [];
|
||||
const seen = new Set<number>();
|
||||
for (const n of numbers) {
|
||||
const idx = Number(n);
|
||||
if (!Number.isInteger(idx) || idx < 1 || idx > observations.length) continue;
|
||||
if (typeof n !== 'number' || !Number.isSafeInteger(n)) continue;
|
||||
const idx = n;
|
||||
if (idx < 1 || idx > observations.length) continue;
|
||||
if (seen.has(idx)) continue;
|
||||
seen.add(idx);
|
||||
indices.push(idx);
|
||||
@@ -208,13 +292,17 @@ export async function detectSupersessionChains(
|
||||
observations: Observation[],
|
||||
llm: ConsolidationLlm,
|
||||
): Promise<SupersessionChain[]> {
|
||||
assertObservations(observations);
|
||||
if (typeof llm !== 'function') {
|
||||
throw new TypeError('detectSupersessionChains: llm must be a function');
|
||||
}
|
||||
if (observations.length < 2) return [];
|
||||
const { normalized, user } = prepareDetectionPrompt(
|
||||
observations,
|
||||
SUPERSESSION_SYSTEM,
|
||||
'detectSupersessionChains',
|
||||
);
|
||||
if (normalized.length < 2) return [];
|
||||
|
||||
const raw = await llm(SUPERSESSION_SYSTEM, numberObservations(observations));
|
||||
const raw = await llm(SUPERSESSION_SYSTEM, user);
|
||||
const parsed = parseLlmJson(raw);
|
||||
const rawChains = Array.isArray(parsed.chains) ? parsed.chains : [];
|
||||
|
||||
@@ -222,10 +310,11 @@ export async function detectSupersessionChains(
|
||||
for (const entry of rawChains) {
|
||||
if (!entry || typeof entry !== 'object') continue;
|
||||
const rec = entry as Record<string, unknown>;
|
||||
const frameIds = mapNumbersToFrameIds(rec.ids, observations, true);
|
||||
const frameIds = mapNumbersToFrameIds(rec.ids, normalized, true);
|
||||
if (frameIds.length < 2) continue;
|
||||
const attribute = typeof rec.attribute === 'string' ? rec.attribute.trim() : '';
|
||||
const currentValue = typeof rec.current_value === 'string' ? rec.current_value.trim() : '';
|
||||
const attribute = safeModelText(rec.attribute, MAX_LABEL_CHARS);
|
||||
const currentValue = safeModelText(rec.current_value, MAX_CURRENT_VALUE_CHARS);
|
||||
if (attribute === null || currentValue === null) continue;
|
||||
chains.push({ attribute: attribute || 'value', currentValue, frameIds });
|
||||
}
|
||||
return chains;
|
||||
@@ -240,13 +329,17 @@ export async function detectEntityGroups(
|
||||
observations: Observation[],
|
||||
llm: ConsolidationLlm,
|
||||
): Promise<EntityGroup[]> {
|
||||
assertObservations(observations);
|
||||
if (typeof llm !== 'function') {
|
||||
throw new TypeError('detectEntityGroups: llm must be a function');
|
||||
}
|
||||
if (observations.length < 2) return [];
|
||||
const { normalized, user } = prepareDetectionPrompt(
|
||||
observations,
|
||||
GROUP_SYSTEM,
|
||||
'detectEntityGroups',
|
||||
);
|
||||
if (normalized.length < 2) return [];
|
||||
|
||||
const raw = await llm(GROUP_SYSTEM, numberObservations(observations));
|
||||
const raw = await llm(GROUP_SYSTEM, user);
|
||||
const parsed = parseLlmJson(raw);
|
||||
const rawGroups = Array.isArray(parsed.groups) ? parsed.groups : [];
|
||||
|
||||
@@ -254,9 +347,10 @@ export async function detectEntityGroups(
|
||||
for (const entry of rawGroups) {
|
||||
if (!entry || typeof entry !== 'object') continue;
|
||||
const rec = entry as Record<string, unknown>;
|
||||
const frameIds = mapNumbersToFrameIds(rec.ids, observations, false);
|
||||
const frameIds = mapNumbersToFrameIds(rec.ids, normalized, false);
|
||||
if (frameIds.length < 2) continue;
|
||||
const label = typeof rec.label === 'string' ? rec.label.trim() : '';
|
||||
const label = safeModelText(rec.label, MAX_LABEL_CHARS);
|
||||
if (label === null) continue;
|
||||
groups.push({ label: label || 'group', frameIds });
|
||||
}
|
||||
return groups;
|
||||
@@ -284,54 +378,201 @@ export function applyConsolidation(
|
||||
groups: EntityGroup[],
|
||||
gopId: string,
|
||||
): ConsolidationResult {
|
||||
if (!frames || typeof frames.createPFrame !== 'function') {
|
||||
if (
|
||||
!frames
|
||||
|| typeof frames.createPFrame !== 'function'
|
||||
|| typeof frames.runInTransaction !== 'function'
|
||||
) {
|
||||
throw new TypeError('applyConsolidation: frames must be a FrameStore');
|
||||
}
|
||||
if (typeof gopId !== 'string' || !gopId) {
|
||||
throw new Error('applyConsolidation: gopId is required');
|
||||
}
|
||||
|
||||
const pframes: MemoryFrame[] = [];
|
||||
const bframes: MemoryFrame[] = [];
|
||||
const deprecated: number[] = [];
|
||||
|
||||
for (const chain of chains ?? []) {
|
||||
const ids = chain.frameIds;
|
||||
if (!Array.isArray(ids) || ids.length < 2) continue;
|
||||
|
||||
const baseId = ids[0];
|
||||
const newestId = ids[ids.length - 1];
|
||||
const newest = frames.getById(newestId);
|
||||
if (!newest) continue; // newest member gone — cannot anchor a current value
|
||||
|
||||
// Deprecate every stale member (all but the newest).
|
||||
for (const staleId of ids.slice(0, -1)) {
|
||||
const stale = frames.getById(staleId);
|
||||
if (!stale) continue;
|
||||
frames.update(staleId, stale.content, 'deprecated');
|
||||
deprecated.push(staleId);
|
||||
return frames.runInTransaction(() => {
|
||||
if (!Array.isArray(chains) || chains.length > MAX_CONSOLIDATION_OBSERVATIONS) {
|
||||
throw new TypeError('applyConsolidation: chains must be a bounded array');
|
||||
}
|
||||
if (!Array.isArray(groups) || groups.length > MAX_CONSOLIDATION_OBSERVATIONS) {
|
||||
throw new TypeError('applyConsolidation: groups must be a bounded array');
|
||||
}
|
||||
if (!frames.hasSession(gopId)) {
|
||||
throw new Error(`applyConsolidation: destination session does not exist: ${gopId}`);
|
||||
}
|
||||
// Boost the surviving newest so it wins recall ties.
|
||||
frames.update(newestId, newest.content, 'critical');
|
||||
|
||||
// Emit the current-value P-frame (base = oldest), preferring the model's
|
||||
// clean value and falling back to the newest frame's raw content.
|
||||
const cleanValue = chain.currentValue.trim() ? chain.currentValue.trim() : newest.content;
|
||||
const attribute = chain.attribute.trim() ? chain.attribute.trim() : 'value';
|
||||
const asOf = String(newest.created_at).slice(0, 10);
|
||||
const pContent = `[current] ${attribute}: ${cleanValue} (as of ${asOf})`;
|
||||
pframes.push(frames.createPFrame(gopId, pContent, baseId, 'critical', 'agent_inferred'));
|
||||
}
|
||||
const frameCache = new Map<number, MemoryFrame>();
|
||||
const requireFrame = (id: number): MemoryFrame => {
|
||||
const cached = frameCache.get(id);
|
||||
if (cached) return cached;
|
||||
const frame = frames.getById(id);
|
||||
if (!frame) throw new Error(`applyConsolidation: missing frame ${id}`);
|
||||
if (frame.importance === 'deprecated') {
|
||||
throw new Error(`applyConsolidation: frame ${id} is deprecated`);
|
||||
}
|
||||
frameCache.set(id, frame);
|
||||
return frame;
|
||||
};
|
||||
const requireIds = (value: unknown, kind: string): number[] => {
|
||||
if (
|
||||
!Array.isArray(value)
|
||||
|| value.length < 2
|
||||
|| value.length > MAX_CONSOLIDATION_OBSERVATIONS
|
||||
) {
|
||||
throw new TypeError(`applyConsolidation: ${kind}.frameIds must contain 2-${MAX_CONSOLIDATION_OBSERVATIONS} ids`);
|
||||
}
|
||||
const seen = new Set<number>();
|
||||
for (const id of value) {
|
||||
if (typeof id !== 'number' || !Number.isSafeInteger(id) || id <= 0) {
|
||||
throw new TypeError(`applyConsolidation: ${kind}.frameIds must be positive safe integers`);
|
||||
}
|
||||
if (seen.has(id)) {
|
||||
throw new TypeError(`applyConsolidation: ${kind}.frameIds must be unique`);
|
||||
}
|
||||
seen.add(id);
|
||||
}
|
||||
return [...value] as number[];
|
||||
};
|
||||
|
||||
for (const group of groups ?? []) {
|
||||
const ids = group.frameIds;
|
||||
if (!Array.isArray(ids) || ids.length < 2) continue;
|
||||
const label = group.label.trim() ? group.label.trim() : 'group';
|
||||
const desc = `${label} (${ids.length} members)`;
|
||||
bframes.push(frames.createBFrame(gopId, desc, ids[0], ids));
|
||||
}
|
||||
const chainPlans: Array<{
|
||||
baseId: number;
|
||||
pContent: string;
|
||||
}> = [];
|
||||
const groupPlans: Array<{ ids: number[]; desc: string }> = [];
|
||||
const chainPlanByIdentity = new Map<string, { pContent: string }>();
|
||||
const groupPlanByIdentity = new Map<string, { desc: string }>();
|
||||
const staleRoles = new Set<number>();
|
||||
const newestRoles = new Set<number>();
|
||||
|
||||
return { pframes, bframes, deprecated };
|
||||
for (const entry of chains) {
|
||||
if (!isRecord(entry)) {
|
||||
throw new TypeError('applyConsolidation: each chain must be an object');
|
||||
}
|
||||
if (typeof entry.attribute !== 'string' || typeof entry.currentValue !== 'string') {
|
||||
throw new TypeError('applyConsolidation: chain labels and values must be strings');
|
||||
}
|
||||
const ids = requireIds(entry.frameIds, 'chain');
|
||||
const members = ids.map(requireFrame);
|
||||
for (let index = 1; index < members.length; index += 1) {
|
||||
const previous = members[index - 1];
|
||||
const current = members[index];
|
||||
const previousTime = observationTime(previous.created_at);
|
||||
const currentTime = observationTime(current.created_at);
|
||||
if (
|
||||
!Number.isFinite(previousTime)
|
||||
|| !Number.isFinite(currentTime)
|
||||
|| previousTime > currentTime
|
||||
|| (previousTime === currentTime && previous.id >= current.id)
|
||||
) {
|
||||
throw new Error('applyConsolidation: chain ids must be chronological oldest to newest');
|
||||
}
|
||||
}
|
||||
|
||||
const newest = members[members.length - 1];
|
||||
const cleanValue = safeModelText(entry.currentValue, MAX_CURRENT_VALUE_CHARS);
|
||||
const cleanAttribute = safeModelText(entry.attribute, MAX_LABEL_CHARS);
|
||||
if (cleanValue === null || cleanAttribute === null) {
|
||||
throw new Error('applyConsolidation: unsafe chain label or value');
|
||||
}
|
||||
const value = cleanValue || newest.content;
|
||||
const attribute = cleanAttribute || 'value';
|
||||
const asOf = String(newest.created_at).slice(0, 10);
|
||||
const pContent = `[current] ${attribute}: ${value} (as of ${asOf})`;
|
||||
if (evaluateExternalMemoryIngress({ content: pContent }).action !== 'allow') {
|
||||
throw new Error('applyConsolidation: unsafe P-frame payload');
|
||||
}
|
||||
|
||||
const chainIdentity = JSON.stringify([
|
||||
attribute.normalize('NFKC').replace(/\s+/g, ' ').toLowerCase(),
|
||||
ids,
|
||||
]);
|
||||
const existingChain = chainPlanByIdentity.get(chainIdentity);
|
||||
if (existingChain) {
|
||||
if (existingChain.pContent !== pContent) {
|
||||
throw new Error('applyConsolidation: conflicting duplicate chain');
|
||||
}
|
||||
continue;
|
||||
}
|
||||
chainPlanByIdentity.set(chainIdentity, { pContent });
|
||||
|
||||
const staleIds = ids.slice(0, -1);
|
||||
for (const staleId of staleIds) staleRoles.add(staleId);
|
||||
newestRoles.add(newest.id);
|
||||
chainPlans.push({ baseId: ids[0], pContent });
|
||||
}
|
||||
|
||||
for (const staleId of staleRoles) {
|
||||
if (newestRoles.has(staleId)) {
|
||||
throw new Error(`applyConsolidation: conflicting stale/newest role for frame ${staleId}`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const entry of groups) {
|
||||
if (!isRecord(entry)) {
|
||||
throw new TypeError('applyConsolidation: each group must be an object');
|
||||
}
|
||||
if (typeof entry.label !== 'string') {
|
||||
throw new TypeError('applyConsolidation: group label must be a string');
|
||||
}
|
||||
const ids = requireIds(entry.frameIds, 'group');
|
||||
ids.forEach(requireFrame);
|
||||
const canonicalIds = [...ids].sort((a, b) => a - b);
|
||||
const cleanLabel = safeModelText(entry.label, MAX_LABEL_CHARS);
|
||||
if (cleanLabel === null) {
|
||||
throw new Error('applyConsolidation: unsafe group label');
|
||||
}
|
||||
const desc = `${cleanLabel || 'group'} (${ids.length} members)`;
|
||||
const persisted = JSON.stringify({ description: desc, references: canonicalIds });
|
||||
if (
|
||||
evaluateExternalMemoryIngress({ content: desc }).action !== 'allow'
|
||||
|| evaluateExternalMemoryIngress({ content: persisted }).action !== 'allow'
|
||||
) {
|
||||
throw new Error('applyConsolidation: unsafe B-frame payload');
|
||||
}
|
||||
|
||||
const groupIdentity = JSON.stringify([
|
||||
(cleanLabel || 'group').normalize('NFKC').replace(/\s+/g, ' ').toLowerCase(),
|
||||
canonicalIds,
|
||||
]);
|
||||
const existingGroup = groupPlanByIdentity.get(groupIdentity);
|
||||
if (existingGroup) {
|
||||
if (existingGroup.desc !== desc) {
|
||||
throw new Error('applyConsolidation: conflicting duplicate group');
|
||||
}
|
||||
continue;
|
||||
}
|
||||
groupPlanByIdentity.set(groupIdentity, { desc });
|
||||
groupPlans.push({ ids: canonicalIds, desc });
|
||||
}
|
||||
|
||||
const pframes: MemoryFrame[] = [];
|
||||
const bframes: MemoryFrame[] = [];
|
||||
const deprecated = [...staleRoles];
|
||||
for (const staleId of staleRoles) {
|
||||
const stale = frameCache.get(staleId)!;
|
||||
if (!frames.update(staleId, stale.content, 'deprecated')) {
|
||||
throw new Error(`applyConsolidation: frame ${staleId} disappeared during update`);
|
||||
}
|
||||
}
|
||||
for (const newestId of newestRoles) {
|
||||
const newest = frameCache.get(newestId)!;
|
||||
if (!frames.update(newestId, newest.content, 'critical')) {
|
||||
throw new Error(`applyConsolidation: frame ${newestId} disappeared during update`);
|
||||
}
|
||||
}
|
||||
for (const plan of chainPlans) {
|
||||
pframes.push(frames.createPFrame(
|
||||
gopId,
|
||||
plan.pContent,
|
||||
plan.baseId,
|
||||
'critical',
|
||||
'agent_inferred',
|
||||
));
|
||||
}
|
||||
for (const plan of groupPlans) {
|
||||
bframes.push(frames.createBFrame(gopId, plan.desc, plan.ids[0], plan.ids));
|
||||
}
|
||||
return { pframes, bframes, deprecated };
|
||||
});
|
||||
}
|
||||
|
||||
// ── Read helpers ───────────────────────────────────────────────────────────
|
||||
@@ -359,11 +600,22 @@ export function collectObservations(
|
||||
params.push(options.gopId);
|
||||
}
|
||||
|
||||
let sql = `SELECT id, content, created_at FROM memory_frames WHERE ${conditions.join(' AND ')} ORDER BY created_at, id`;
|
||||
if (options.limit && options.limit > 0) {
|
||||
sql += ' LIMIT ?';
|
||||
params.push(options.limit);
|
||||
const sql = `
|
||||
SELECT id, content, created_at
|
||||
FROM (
|
||||
SELECT id, content, created_at
|
||||
FROM memory_frames
|
||||
WHERE ${conditions.join(' AND ')}
|
||||
ORDER BY ${CREATED_AT_SORT_EXPR} DESC, id DESC
|
||||
LIMIT ?
|
||||
)
|
||||
ORDER BY ${CREATED_AT_SORT_EXPR} ASC, id ASC
|
||||
`;
|
||||
return raw.prepare(sql).all(...params) as Observation[];
|
||||
}
|
||||
const sql = `SELECT id, content, created_at FROM memory_frames WHERE ${conditions.join(' AND ')} ORDER BY ${CREATED_AT_SORT_EXPR}, id`;
|
||||
return raw.prepare(sql).all(...params) as Observation[];
|
||||
}
|
||||
|
||||
@@ -379,15 +631,25 @@ export function getCurrentValues(db: MindDB, gopId?: string): string[] {
|
||||
gopId
|
||||
? raw
|
||||
.prepare(
|
||||
"SELECT content FROM memory_frames WHERE frame_type = 'P' AND gop_id = ? ORDER BY created_at, id",
|
||||
`SELECT content, importance FROM memory_frames WHERE frame_type = 'P' AND substr(content, 1, 10) = '[current] ' AND gop_id = ? ORDER BY ${CREATED_AT_SORT_EXPR} DESC, id DESC`,
|
||||
)
|
||||
.all(gopId)
|
||||
: raw
|
||||
.prepare("SELECT content FROM memory_frames WHERE frame_type = 'P' ORDER BY created_at, id")
|
||||
.prepare(`SELECT content, importance FROM memory_frames WHERE frame_type = 'P' AND substr(content, 1, 10) = '[current] ' ORDER BY ${CREATED_AT_SORT_EXPR} DESC, id DESC`)
|
||||
.all()
|
||||
) as Array<{ content: string }>;
|
||||
) as Array<{ content: string; importance: string }>;
|
||||
|
||||
return rows
|
||||
.map((r) => String(r.content).replace(/^\[current\]\s*/, '').trim())
|
||||
.filter(Boolean);
|
||||
const seen = new Set<string>();
|
||||
const current: string[] = [];
|
||||
for (const row of rows) {
|
||||
const line = String(row.content).replace(/^\[current\]\s*/, '').trim();
|
||||
const colon = line.indexOf(':');
|
||||
if (colon <= 0) continue;
|
||||
const key = line.slice(0, colon).trim().replace(/\s+/g, ' ').toLowerCase();
|
||||
if (!key || seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
if (row.importance === 'deprecated') continue;
|
||||
current.push(line);
|
||||
}
|
||||
return current.reverse();
|
||||
}
|
||||
|
||||
258
packages/hive-mind-core/src/mind/transformers-model-load.ts
Normal file
258
packages/hive-mind-core/src/mind/transformers-model-load.ts
Normal file
@@ -0,0 +1,258 @@
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import Database from 'better-sqlite3';
|
||||
|
||||
const LOCK_WAIT_TIMEOUT_MS = 12 * 60 * 1_000;
|
||||
const LOCK_RETRY_MIN_MS = 35;
|
||||
const LOCK_RETRY_JITTER_MS = 30;
|
||||
const SAFE_HUGGING_FACE_MODEL_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*(?:\/[A-Za-z0-9][A-Za-z0-9._-]*)?$/;
|
||||
const CORRUPT_ONNX_ERROR = /^Load model from (.+\.onnx) failed:\s*Protobuf parsing failed\.?$/i;
|
||||
const LOCK_DIRECTORY = '.waggle-model-locks';
|
||||
|
||||
export interface TransformersModelLoadOptions<T> {
|
||||
cacheDir: string;
|
||||
model: string;
|
||||
load: (canonicalCacheDir: string) => Promise<T>;
|
||||
lockTimeoutMs?: number;
|
||||
onQuarantine?: (quarantineDir: string) => void | Promise<void>;
|
||||
}
|
||||
|
||||
function normalizeLockKey(value: string): string {
|
||||
return process.platform === 'win32' ? value.toLocaleLowerCase('en-US') : value;
|
||||
}
|
||||
|
||||
export function modelLoadLockPath(cacheDir: string, model: string): string {
|
||||
const modelHash = createHash('sha256')
|
||||
.update(normalizeLockKey(model))
|
||||
.digest('hex')
|
||||
.slice(0, 16);
|
||||
return path.join(path.resolve(cacheDir), LOCK_DIRECTORY, `${modelHash}.sqlite`);
|
||||
}
|
||||
|
||||
function isSqliteBusy(error: unknown): boolean {
|
||||
if (!(error instanceof Error) || !('code' in error)) return false;
|
||||
const code = String((error as Error & { code?: unknown }).code);
|
||||
return code === 'SQLITE_BUSY'
|
||||
|| code === 'SQLITE_BUSY_SNAPSHOT'
|
||||
|| code === 'SQLITE_LOCKED';
|
||||
}
|
||||
|
||||
async function acquireCrossProcessLock(
|
||||
database: Database.Database,
|
||||
timeoutMs: number,
|
||||
): Promise<void> {
|
||||
const deadline = performance.now() + timeoutMs;
|
||||
|
||||
for (;;) {
|
||||
try {
|
||||
database.exec('BEGIN IMMEDIATE');
|
||||
return;
|
||||
} catch (error) {
|
||||
if (!isSqliteBusy(error)) throw error;
|
||||
if (performance.now() >= deadline) {
|
||||
throw new Error(
|
||||
`Timed out waiting ${timeoutMs}ms for local model cache lock`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const retryMs = LOCK_RETRY_MIN_MS + Math.random() * LOCK_RETRY_JITTER_MS;
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, retryMs));
|
||||
}
|
||||
}
|
||||
|
||||
function isWithin(root: string, candidate: string): boolean {
|
||||
const relative = path.relative(root, candidate);
|
||||
return relative !== ''
|
||||
&& relative !== '..'
|
||||
&& !relative.startsWith(`..${path.sep}`)
|
||||
&& !path.isAbsolute(relative);
|
||||
}
|
||||
|
||||
function isWithinOrEqual(root: string, candidate: string): boolean {
|
||||
return root === candidate || isWithin(root, candidate);
|
||||
}
|
||||
|
||||
function assertUnlinkedPath(root: string, target: string): void {
|
||||
const relative = path.relative(root, target);
|
||||
if (relative === '' || relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
||||
throw new Error('Model cache path escapes its cache root');
|
||||
}
|
||||
|
||||
let current = root;
|
||||
for (const segment of relative.split(path.sep)) {
|
||||
current = path.join(current, segment);
|
||||
const stats = fs.lstatSync(current);
|
||||
if (stats.isSymbolicLink()) {
|
||||
throw new Error(`Model cache path crosses a filesystem link: ${current}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function validateQuarantinePaths(
|
||||
canonicalCacheDir: string,
|
||||
modelDir: string,
|
||||
reportedOnnxPath: string,
|
||||
): { modelDir: string; quarantineRoot: string } | null {
|
||||
if (!path.isAbsolute(reportedOnnxPath)) return null;
|
||||
if (!fs.existsSync(modelDir) || !fs.existsSync(reportedOnnxPath)) return null;
|
||||
|
||||
assertUnlinkedPath(canonicalCacheDir, modelDir);
|
||||
assertUnlinkedPath(canonicalCacheDir, reportedOnnxPath);
|
||||
|
||||
const quarantineRoot = path.dirname(modelDir);
|
||||
const modelStats = fs.lstatSync(modelDir);
|
||||
const reportStats = fs.lstatSync(reportedOnnxPath);
|
||||
const rootStats = fs.lstatSync(quarantineRoot);
|
||||
if (!modelStats.isDirectory() || !reportStats.isFile() || !rootStats.isDirectory()) return null;
|
||||
|
||||
const realCacheDir = fs.realpathSync.native(canonicalCacheDir);
|
||||
const realQuarantineRoot = fs.realpathSync.native(quarantineRoot);
|
||||
const realModelDir = fs.realpathSync.native(modelDir);
|
||||
const realOnnxPath = fs.realpathSync.native(reportedOnnxPath);
|
||||
if (!isWithinOrEqual(realCacheDir, realQuarantineRoot)
|
||||
|| !isWithin(realQuarantineRoot, realModelDir)
|
||||
|| path.dirname(realModelDir) !== realQuarantineRoot
|
||||
|| !isWithin(realModelDir, realOnnxPath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return { modelDir: realModelDir, quarantineRoot: realQuarantineRoot };
|
||||
}
|
||||
|
||||
function quarantineCorruptModel(
|
||||
canonicalCacheDir: string,
|
||||
model: string,
|
||||
reportedOnnxPath: string,
|
||||
): string | null {
|
||||
if (!SAFE_HUGGING_FACE_MODEL_ID.test(model)) return null;
|
||||
|
||||
const modelDir = path.join(canonicalCacheDir, ...model.split('/'));
|
||||
const firstValidation = validateQuarantinePaths(canonicalCacheDir, modelDir, reportedOnnxPath);
|
||||
if (!firstValidation) return null;
|
||||
|
||||
// Re-resolve immediately before the move so a changed link/path cannot redirect it.
|
||||
const finalValidation = validateQuarantinePaths(canonicalCacheDir, modelDir, reportedOnnxPath);
|
||||
if (!finalValidation) return null;
|
||||
|
||||
const quarantineDir = path.join(
|
||||
finalValidation.quarantineRoot,
|
||||
`${path.basename(finalValidation.modelDir)}.corrupt-${Date.now()}-${randomUUID()}`,
|
||||
);
|
||||
fs.renameSync(finalValidation.modelDir, quarantineDir);
|
||||
return quarantineDir;
|
||||
}
|
||||
|
||||
function reportedCorruptOnnxPath(error: unknown): string | null {
|
||||
if (!(error instanceof Error)) return null;
|
||||
return error.message.trim().match(CORRUPT_ONNX_ERROR)?.[1] ?? null;
|
||||
}
|
||||
|
||||
function notifyQuarantine(
|
||||
callback: TransformersModelLoadOptions<unknown>['onQuarantine'],
|
||||
quarantineDir: string,
|
||||
): void {
|
||||
try {
|
||||
const notification = callback?.(quarantineDir);
|
||||
if (notification) {
|
||||
void Promise.resolve(notification).catch(() => undefined);
|
||||
}
|
||||
} catch {
|
||||
// Notification is advisory and must not alter model recovery control flow.
|
||||
}
|
||||
}
|
||||
|
||||
function prepareLockDatabase(canonicalCacheDir: string, model: string): Database.Database {
|
||||
const lockPath = modelLoadLockPath(canonicalCacheDir, model);
|
||||
const lockDir = path.dirname(lockPath);
|
||||
fs.mkdirSync(lockDir, { recursive: true });
|
||||
|
||||
const lockDirStats = fs.lstatSync(lockDir);
|
||||
if (!lockDirStats.isDirectory() || lockDirStats.isSymbolicLink()) {
|
||||
throw new Error('Local model lock path is not a regular directory');
|
||||
}
|
||||
const realLockDir = fs.realpathSync.native(lockDir);
|
||||
if (!isWithin(canonicalCacheDir, realLockDir)) {
|
||||
throw new Error('Local model lock path escapes its cache root');
|
||||
}
|
||||
if (fs.existsSync(lockPath)) {
|
||||
const lockStats = fs.lstatSync(lockPath);
|
||||
if (!lockStats.isFile() || lockStats.isSymbolicLink()) {
|
||||
throw new Error('Local model lock database is not a regular file');
|
||||
}
|
||||
}
|
||||
|
||||
return new Database(lockPath, { timeout: 0 });
|
||||
}
|
||||
|
||||
async function runModelLoad<T>(options: TransformersModelLoadOptions<T>): Promise<T> {
|
||||
const timeoutMs = options.lockTimeoutMs ?? LOCK_WAIT_TIMEOUT_MS;
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
|
||||
throw new Error('Local model cache lock timeout must be a non-negative finite number');
|
||||
}
|
||||
|
||||
const requestedCacheDir = path.resolve(options.cacheDir);
|
||||
fs.mkdirSync(requestedCacheDir, { recursive: true });
|
||||
const canonicalCacheDir = fs.realpathSync.native(requestedCacheDir);
|
||||
const lockDatabase = prepareLockDatabase(canonicalCacheDir, options.model);
|
||||
let result!: T;
|
||||
let primaryError: unknown;
|
||||
let hasPrimaryError = false;
|
||||
|
||||
try {
|
||||
await acquireCrossProcessLock(lockDatabase, timeoutMs);
|
||||
|
||||
try {
|
||||
result = await options.load(canonicalCacheDir);
|
||||
} catch (firstError) {
|
||||
const reportedOnnxPath = reportedCorruptOnnxPath(firstError);
|
||||
if (!reportedOnnxPath) throw firstError;
|
||||
|
||||
let quarantineDir: string | null = null;
|
||||
try {
|
||||
quarantineDir = quarantineCorruptModel(canonicalCacheDir, options.model, reportedOnnxPath);
|
||||
} catch {
|
||||
// Preserve the original loader error if safe quarantine cannot complete.
|
||||
}
|
||||
if (!quarantineDir) throw firstError;
|
||||
|
||||
notifyQuarantine(options.onQuarantine, quarantineDir);
|
||||
result = await options.load(canonicalCacheDir);
|
||||
}
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
hasPrimaryError = true;
|
||||
}
|
||||
|
||||
let cleanupError: unknown;
|
||||
let hasCleanupError = false;
|
||||
if (lockDatabase.inTransaction) {
|
||||
try {
|
||||
lockDatabase.exec('ROLLBACK');
|
||||
} catch (error) {
|
||||
cleanupError = error;
|
||||
hasCleanupError = true;
|
||||
}
|
||||
}
|
||||
try {
|
||||
lockDatabase.close();
|
||||
} catch (error) {
|
||||
if (!hasCleanupError) cleanupError = error;
|
||||
hasCleanupError = true;
|
||||
}
|
||||
|
||||
if (hasPrimaryError) throw primaryError;
|
||||
if (hasCleanupError) throw cleanupError;
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Serializes one model/cache load across workers and processes. SQLite owns
|
||||
* the OS lock, so process termination releases it without PID/age heuristics.
|
||||
*/
|
||||
export function withTransformersModelLoad<T>(options: TransformersModelLoadOptions<T>): Promise<T> {
|
||||
return runModelLoad(options);
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { MindDB } from './mind/db.js';
|
||||
import { createCoreLogger } from './logger.js';
|
||||
@@ -61,8 +62,20 @@ export class MultiMindCache {
|
||||
this.cache.delete(workspaceId);
|
||||
}
|
||||
|
||||
const mindPath = this.getMindPath(workspaceId);
|
||||
if (!mindPath) return null;
|
||||
try {
|
||||
const mindPath = this.getMindPath(workspaceId);
|
||||
if (!mindPath) return null;
|
||||
if (mindPath === ':memory:') {
|
||||
if (this.cache.size >= this.maxOpen) this.evictLRU();
|
||||
const recheck = this.cache.get(workspaceId);
|
||||
if (recheck?.db.isOpen()) {
|
||||
recheck.lastAccessed = Date.now();
|
||||
return recheck.db;
|
||||
}
|
||||
const db = new MindDB(mindPath);
|
||||
this.cache.set(workspaceId, { db, lastAccessed: Date.now(), pins: carriedPins });
|
||||
return db;
|
||||
}
|
||||
|
||||
// Review Critical #2: path-traversal guard. Defense-in-depth against an
|
||||
// attacker-controlled workspaceId (e.g. from an LLM tool call with a misconfigured
|
||||
@@ -78,9 +91,6 @@ export class MultiMindCache {
|
||||
}
|
||||
}
|
||||
|
||||
// Review Major #5: re-check after evictLRU — a concurrent call may have just
|
||||
// inserted the same workspaceId between our initial .get() and here.
|
||||
try {
|
||||
if (this.cache.size >= this.maxOpen) {
|
||||
this.evictLRU();
|
||||
}
|
||||
@@ -89,7 +99,23 @@ export class MultiMindCache {
|
||||
recheck.lastAccessed = Date.now();
|
||||
return recheck.db;
|
||||
}
|
||||
const db = new MindDB(mindPath);
|
||||
const mindStat = fs.lstatSync(mindPath, { throwIfNoEntry: false });
|
||||
let canonicalMind: string;
|
||||
if (mindStat) {
|
||||
if (!mindStat.isFile() || mindStat.isSymbolicLink() || mindStat.nlink !== 1) return null;
|
||||
canonicalMind = fs.realpathSync.native(mindPath);
|
||||
} else {
|
||||
const canonicalParent = fs.realpathSync.native(path.dirname(mindPath));
|
||||
canonicalMind = path.join(canonicalParent, path.basename(mindPath));
|
||||
}
|
||||
if (this.allowedRoot) {
|
||||
const canonicalRoot = fs.realpathSync.native(this.allowedRoot);
|
||||
const relative = path.relative(canonicalRoot, canonicalMind);
|
||||
if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
const db = new MindDB(canonicalMind);
|
||||
this.cache.set(workspaceId, { db, lastAccessed: Date.now(), pins: carriedPins });
|
||||
return db;
|
||||
} catch (err) {
|
||||
@@ -130,7 +156,12 @@ export class MultiMindCache {
|
||||
*/
|
||||
release(workspaceId: string): void {
|
||||
const entry = this.cache.get(workspaceId);
|
||||
if (entry && entry.pins > 0) entry.pins -= 1;
|
||||
if (!entry || entry.pins === 0) return;
|
||||
|
||||
entry.pins -= 1;
|
||||
if (entry.pins === 0 && this.cache.size > this.maxOpen) {
|
||||
this.evictLRU();
|
||||
}
|
||||
}
|
||||
|
||||
has(workspaceId: string): boolean {
|
||||
|
||||
@@ -123,6 +123,13 @@ interface WorkspacesMeta {
|
||||
defaultWorkspace?: string | null;
|
||||
}
|
||||
|
||||
const WORKSPACE_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{0,199}$/;
|
||||
|
||||
function isContained(root: string, candidate: string): boolean {
|
||||
const relative = path.relative(root, candidate);
|
||||
return relative === '' || (!path.isAbsolute(relative) && relative !== '..' && !relative.startsWith(`..${path.sep}`));
|
||||
}
|
||||
|
||||
/**
|
||||
* WorkspaceManager manages workspace CRUD, groups, and directory structure.
|
||||
* Each workspace lives under {baseDir}/workspaces/{id}/ with:
|
||||
@@ -132,6 +139,7 @@ interface WorkspacesMeta {
|
||||
*/
|
||||
export class WorkspaceManager {
|
||||
private readonly workspacesDir: string;
|
||||
private readonly canonicalWorkspacesDir: string;
|
||||
private readonly metaPath: string;
|
||||
|
||||
constructor(private readonly baseDir: string) {
|
||||
@@ -141,6 +149,18 @@ export class WorkspaceManager {
|
||||
if (!fs.existsSync(this.workspacesDir)) {
|
||||
fs.mkdirSync(this.workspacesDir, { recursive: true });
|
||||
}
|
||||
const rootStat = fs.lstatSync(this.workspacesDir);
|
||||
const canonicalBase = fs.realpathSync.native(baseDir);
|
||||
const canonicalRoot = fs.realpathSync.native(this.workspacesDir);
|
||||
if (
|
||||
rootStat.isSymbolicLink()
|
||||
|| !rootStat.isDirectory()
|
||||
|| canonicalRoot === canonicalBase
|
||||
|| !isContained(canonicalBase, canonicalRoot)
|
||||
) {
|
||||
throw new Error('Workspace root must be a regular directory inside the data directory');
|
||||
}
|
||||
this.canonicalWorkspacesDir = canonicalRoot;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -164,8 +184,19 @@ export class WorkspaceManager {
|
||||
*/
|
||||
// Reverse-ported from OSS hive-mind (oss-drift triage R4, 2026-06-11).
|
||||
ensure(id: string, options: Partial<CreateWorkspaceOptions> = {}): WorkspaceConfig {
|
||||
this.assertWorkspaceId(id);
|
||||
const existing = this.get(id);
|
||||
if (existing) return existing;
|
||||
const workspacePath = path.join(this.resolveWorkspaceRoot(), id);
|
||||
const workspaceStat = fs.lstatSync(workspacePath, { throwIfNoEntry: false });
|
||||
if (workspaceStat) {
|
||||
if (id !== 'default' || !this.isEmptyLegacyWorkspaceDirectory(workspacePath, workspaceStat)) {
|
||||
throw new Error(`Workspace path already exists but is not a valid workspace: ${id}`);
|
||||
}
|
||||
const sessionsPath = path.join(workspacePath, 'sessions');
|
||||
if (fs.lstatSync(sessionsPath, { throwIfNoEntry: false })) fs.rmdirSync(sessionsPath);
|
||||
fs.rmdirSync(workspacePath);
|
||||
}
|
||||
|
||||
return this.createWithId(id, {
|
||||
...options,
|
||||
@@ -178,13 +209,19 @@ export class WorkspaceManager {
|
||||
* Shared create path: write directory structure + config for an exact id.
|
||||
*/
|
||||
private createWithId(id: string, options: CreateWorkspaceOptions): WorkspaceConfig {
|
||||
const wsDir = path.join(this.workspacesDir, id);
|
||||
this.assertWorkspaceId(id);
|
||||
const canonicalRoot = this.resolveWorkspaceRoot();
|
||||
const wsDir = path.join(canonicalRoot, id);
|
||||
|
||||
fs.mkdirSync(wsDir, { recursive: true });
|
||||
fs.mkdirSync(path.join(wsDir, 'sessions'), { recursive: true });
|
||||
fs.mkdirSync(wsDir);
|
||||
const canonicalWorkspace = fs.realpathSync.native(wsDir);
|
||||
if (!isContained(canonicalRoot, canonicalWorkspace)) {
|
||||
throw new Error(`Workspace path escapes workspace root: ${id}`);
|
||||
}
|
||||
fs.mkdirSync(path.join(canonicalWorkspace, 'sessions'));
|
||||
|
||||
// Touch workspace.mind — MindDB will init schema when first opened
|
||||
fs.writeFileSync(path.join(wsDir, 'workspace.mind'), '');
|
||||
fs.writeFileSync(path.join(canonicalWorkspace, 'workspace.mind'), '', { flag: 'wx' });
|
||||
|
||||
const config: WorkspaceConfig = {
|
||||
id,
|
||||
@@ -214,9 +251,9 @@ export class WorkspaceManager {
|
||||
};
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(wsDir, 'workspace.json'),
|
||||
path.join(canonicalWorkspace, 'workspace.json'),
|
||||
JSON.stringify(config, null, 2),
|
||||
'utf-8'
|
||||
{ encoding: 'utf-8', flag: 'wx' }
|
||||
);
|
||||
|
||||
return config;
|
||||
@@ -226,18 +263,14 @@ export class WorkspaceManager {
|
||||
* List all workspaces by reading workspace.json from each subdirectory.
|
||||
*/
|
||||
list(): WorkspaceConfig[] {
|
||||
if (!fs.existsSync(this.workspacesDir)) return [];
|
||||
|
||||
const entries = fs.readdirSync(this.workspacesDir, { withFileTypes: true });
|
||||
const root = this.resolveWorkspaceRoot();
|
||||
const entries = fs.readdirSync(root, { withFileTypes: true });
|
||||
const configs: WorkspaceConfig[] = [];
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
const configPath = path.join(this.workspacesDir, entry.name, 'workspace.json');
|
||||
if (fs.existsSync(configPath)) {
|
||||
const raw = fs.readFileSync(configPath, 'utf-8');
|
||||
configs.push(JSON.parse(raw) as WorkspaceConfig);
|
||||
}
|
||||
if (!entry.isDirectory() || !WORKSPACE_ID.test(entry.name)) continue;
|
||||
const config = this.get(entry.name);
|
||||
if (config) configs.push(config);
|
||||
}
|
||||
|
||||
return configs;
|
||||
@@ -262,11 +295,22 @@ export class WorkspaceManager {
|
||||
* Get a workspace by ID. Returns null if not found.
|
||||
*/
|
||||
get(id: string): WorkspaceConfig | null {
|
||||
const configPath = path.join(this.workspacesDir, id, 'workspace.json');
|
||||
if (!fs.existsSync(configPath)) return null;
|
||||
if (!WORKSPACE_ID.test(id)) return null;
|
||||
|
||||
const raw = fs.readFileSync(configPath, 'utf-8');
|
||||
return JSON.parse(raw) as WorkspaceConfig;
|
||||
try {
|
||||
const workspaceDir = this.resolveWorkspaceDir(id);
|
||||
if (!workspaceDir) return null;
|
||||
const configPath = path.join(workspaceDir, 'workspace.json');
|
||||
const configStat = fs.lstatSync(configPath, { throwIfNoEntry: false });
|
||||
if (!configStat?.isFile() || configStat.isSymbolicLink() || configStat.nlink !== 1) return null;
|
||||
const canonicalConfig = fs.realpathSync.native(configPath);
|
||||
if (!isContained(workspaceDir, canonicalConfig)) return null;
|
||||
|
||||
const config = JSON.parse(fs.readFileSync(canonicalConfig, 'utf-8')) as WorkspaceConfig;
|
||||
return config.id === id ? config : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -293,10 +337,11 @@ export class WorkspaceManager {
|
||||
* Delete a workspace by removing its entire directory.
|
||||
*/
|
||||
delete(id: string): void {
|
||||
const wsDir = path.join(this.workspacesDir, id);
|
||||
if (fs.existsSync(wsDir)) {
|
||||
fs.rmSync(wsDir, { recursive: true, force: true });
|
||||
}
|
||||
this.assertWorkspaceId(id);
|
||||
if (!this.get(id)) return;
|
||||
const workspaceDir = this.resolveWorkspaceDir(id);
|
||||
if (!workspaceDir) return;
|
||||
fs.rmSync(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -318,7 +363,85 @@ export class WorkspaceManager {
|
||||
* Get the path to a workspace's .mind file.
|
||||
*/
|
||||
getMindPath(id: string): string {
|
||||
return path.join(this.workspacesDir, id, 'workspace.mind');
|
||||
this.assertWorkspaceId(id);
|
||||
const lexicalMindPath = path.join(this.workspacesDir, id, 'workspace.mind');
|
||||
const workspaceDir = this.resolveWorkspaceDir(id);
|
||||
if (!workspaceDir) throw new Error(`Workspace not found: ${id}`);
|
||||
|
||||
const configPath = path.join(workspaceDir, 'workspace.json');
|
||||
const configStat = fs.lstatSync(configPath, { throwIfNoEntry: false });
|
||||
if (!configStat?.isFile() || configStat.isSymbolicLink() || configStat.nlink !== 1) {
|
||||
throw new Error(`Workspace config not found: ${id}`);
|
||||
}
|
||||
const canonicalConfig = fs.realpathSync.native(configPath);
|
||||
if (!isContained(workspaceDir, canonicalConfig)) {
|
||||
throw new Error(`Workspace config escapes workspace directory: ${id}`);
|
||||
}
|
||||
const config = JSON.parse(fs.readFileSync(canonicalConfig, 'utf-8')) as WorkspaceConfig;
|
||||
if (config.id !== id) throw new Error(`Workspace config id mismatch: ${id}`);
|
||||
|
||||
const mindPath = path.join(workspaceDir, 'workspace.mind');
|
||||
const mindStat = fs.lstatSync(mindPath, { throwIfNoEntry: false });
|
||||
if (!mindStat) return lexicalMindPath;
|
||||
if (!mindStat?.isFile() || mindStat.isSymbolicLink() || mindStat.nlink !== 1) {
|
||||
throw new Error(`Workspace mind is not a regular file: ${id}`);
|
||||
}
|
||||
const canonicalMind = fs.realpathSync.native(mindPath);
|
||||
if (!isContained(workspaceDir, canonicalMind)) {
|
||||
throw new Error(`Workspace mind escapes workspace directory: ${id}`);
|
||||
}
|
||||
return lexicalMindPath;
|
||||
}
|
||||
|
||||
private assertWorkspaceId(id: string): void {
|
||||
if (!WORKSPACE_ID.test(id)) throw new Error(`Invalid workspace id: ${id}`);
|
||||
}
|
||||
|
||||
private isEmptyLegacyWorkspaceDirectory(workspacePath: string, stat: fs.Stats): boolean {
|
||||
if (stat.isSymbolicLink() || !stat.isDirectory()) return false;
|
||||
const canonicalRoot = this.resolveWorkspaceRoot();
|
||||
const canonicalWorkspace = fs.realpathSync.native(workspacePath);
|
||||
if (!isContained(canonicalRoot, canonicalWorkspace)) return false;
|
||||
const entries = fs.readdirSync(workspacePath, { withFileTypes: true });
|
||||
if (entries.length === 0) return true;
|
||||
if (entries.length !== 1 || entries[0]?.name !== 'sessions' || !entries[0].isDirectory()) return false;
|
||||
const sessionsPath = path.join(workspacePath, 'sessions');
|
||||
const sessionsStat = fs.lstatSync(sessionsPath);
|
||||
if (sessionsStat.isSymbolicLink()) return false;
|
||||
const canonicalSessions = fs.realpathSync.native(sessionsPath);
|
||||
return isContained(canonicalWorkspace, canonicalSessions) && fs.readdirSync(sessionsPath).length === 0;
|
||||
}
|
||||
|
||||
private resolveWorkspaceDir(id: string): string | null {
|
||||
this.assertWorkspaceId(id);
|
||||
const lexicalRoot = this.resolveWorkspaceRoot();
|
||||
const lexicalWorkspace = path.resolve(lexicalRoot, id);
|
||||
if (!isContained(lexicalRoot, lexicalWorkspace)) {
|
||||
throw new Error(`Workspace path escapes workspace root: ${id}`);
|
||||
}
|
||||
const workspaceStat = fs.lstatSync(lexicalWorkspace, { throwIfNoEntry: false });
|
||||
if (!workspaceStat) return null;
|
||||
if (workspaceStat.isSymbolicLink() || !workspaceStat.isDirectory()) {
|
||||
throw new Error(`Workspace path is not a regular directory: ${id}`);
|
||||
}
|
||||
|
||||
const canonicalWorkspace = fs.realpathSync.native(lexicalWorkspace);
|
||||
if (!isContained(this.canonicalWorkspacesDir, canonicalWorkspace)) {
|
||||
throw new Error(`Workspace path escapes workspace root: ${id}`);
|
||||
}
|
||||
return canonicalWorkspace;
|
||||
}
|
||||
|
||||
private resolveWorkspaceRoot(): string {
|
||||
const rootStat = fs.lstatSync(this.workspacesDir, { throwIfNoEntry: false });
|
||||
if (!rootStat?.isDirectory() || rootStat.isSymbolicLink()) {
|
||||
throw new Error('Workspace root is not a regular directory');
|
||||
}
|
||||
const canonicalRoot = fs.realpathSync.native(this.workspacesDir);
|
||||
if (canonicalRoot !== this.canonicalWorkspacesDir) {
|
||||
throw new Error('Workspace root changed after initialization');
|
||||
}
|
||||
return canonicalRoot;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -381,7 +504,7 @@ export class WorkspaceManager {
|
||||
}
|
||||
|
||||
private workspaceExists(id: string): boolean {
|
||||
return fs.existsSync(path.join(this.workspacesDir, id));
|
||||
return fs.existsSync(path.join(this.resolveWorkspaceRoot(), id));
|
||||
}
|
||||
|
||||
private loadMeta(): WorkspacesMeta {
|
||||
|
||||
Reference in New Issue
Block a user