moving
This commit is contained in:
Oleg Maslov
751
packages/hive-mind-core/src/memory-ingress-guard.ts
Normal file
751
packages/hive-mind-core/src/memory-ingress-guard.ts
Normal file
@@ -0,0 +1,751 @@
|
||||
import { scanForInjection, type ScanResult } from './injection-scanner.js';
|
||||
|
||||
export interface ExternalMemoryIngressInput {
|
||||
title?: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface ExternalMemoryProjectionInput {
|
||||
content: string;
|
||||
messages?: unknown;
|
||||
parseMethod?: unknown;
|
||||
maxChars?: number;
|
||||
}
|
||||
|
||||
export type ExternalMemoryIngressDecision =
|
||||
| { action: 'allow'; scan: ScanResult }
|
||||
| { action: 'block'; reason: 'prompt_injection'; scan: ScanResult };
|
||||
|
||||
// RawArchive supports one million characters per item. Keep the same explicit
|
||||
// public-ingress budget, checked before concatenation, scanning, or normalization.
|
||||
const MAX_EXTERNAL_MEMORY_INGRESS_CHARS = 1_000_000;
|
||||
|
||||
const NAMED_HTML_ENTITIES: Readonly<Record<string, string>> = Object.freeze({
|
||||
af: '\u2061',
|
||||
amp: '&',
|
||||
applyfunction: '\u2061',
|
||||
apos: "'",
|
||||
colon: ':',
|
||||
emsp: ' ',
|
||||
ensp: ' ',
|
||||
gt: '>',
|
||||
hairsp: ' ',
|
||||
ic: '\u2063',
|
||||
invisiblecomma: '\u2063',
|
||||
invisibletimes: '\u2062',
|
||||
it: '\u2062',
|
||||
lrm: '\u200e',
|
||||
lt: '<',
|
||||
negativemediumspace: '\u200b',
|
||||
negativethickspace: '\u200b',
|
||||
negativethinspace: '\u200b',
|
||||
negativeverythinspace: '\u200b',
|
||||
newline: '\n',
|
||||
nbsp: ' ',
|
||||
nobreak: '\u2060',
|
||||
quot: '"',
|
||||
rlm: '\u200f',
|
||||
shy: '\u00ad',
|
||||
tab: '\t',
|
||||
thinsp: ' ',
|
||||
zwj: '',
|
||||
zwnj: '',
|
||||
zwsp: '',
|
||||
zerowidthspace: '\u200b',
|
||||
});
|
||||
|
||||
type CanonicalMemoryMessage = {
|
||||
role: 'user' | 'assistant' | 'system';
|
||||
text: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Return only the attacker-controlled text represented by a stored adapter
|
||||
* projection. Role prefixes may be omitted only when plain canonical messages
|
||||
* exactly reproduce the full content and did not come from universal raw text.
|
||||
*/
|
||||
export function projectExternalMemoryContent(input: ExternalMemoryProjectionInput): string {
|
||||
let cappedContent = '';
|
||||
try {
|
||||
const content = typeof input.content === 'string' ? input.content : '';
|
||||
const maxChars = input.maxChars;
|
||||
cappedContent = maxChars === undefined
|
||||
|| !Number.isSafeInteger(maxChars)
|
||||
|| maxChars < 0
|
||||
? content
|
||||
: content.slice(0, maxChars);
|
||||
if (input.parseMethod === 'universal-text'
|
||||
|| !Array.isArray(input.messages)
|
||||
|| input.messages.length === 0) {
|
||||
return cappedContent;
|
||||
}
|
||||
|
||||
const messages: CanonicalMemoryMessage[] = [];
|
||||
for (const candidate of input.messages) {
|
||||
if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) {
|
||||
return cappedContent;
|
||||
}
|
||||
const prototype = Object.getPrototypeOf(candidate);
|
||||
if (prototype !== Object.prototype && prototype !== null) return cappedContent;
|
||||
const roleDescriptor = Object.getOwnPropertyDescriptor(candidate, 'role');
|
||||
const textDescriptor = Object.getOwnPropertyDescriptor(candidate, 'text');
|
||||
if (!roleDescriptor || !('value' in roleDescriptor)
|
||||
|| !textDescriptor || !('value' in textDescriptor)) {
|
||||
return cappedContent;
|
||||
}
|
||||
const role = roleDescriptor.value as unknown;
|
||||
const text = textDescriptor.value as unknown;
|
||||
if ((role !== 'user' && role !== 'assistant' && role !== 'system')
|
||||
|| typeof text !== 'string') {
|
||||
return cappedContent;
|
||||
}
|
||||
messages.push({ role, text });
|
||||
}
|
||||
|
||||
const serialized = messages
|
||||
.map(message => `${message.role}: ${message.text}`)
|
||||
.join('\n\n');
|
||||
if (serialized !== content) return cappedContent;
|
||||
|
||||
const parts: string[] = [];
|
||||
let cursor = 0;
|
||||
let offset = 0;
|
||||
for (const [index, message] of messages.entries()) {
|
||||
if (index > 0) offset += 2;
|
||||
const prefixStart = offset;
|
||||
const prefixEnd = prefixStart + `${message.role}: `.length;
|
||||
if (prefixStart >= cappedContent.length) break;
|
||||
if (message.role === 'system') return cappedContent;
|
||||
parts.push(cappedContent.slice(cursor, prefixStart));
|
||||
cursor = Math.min(prefixEnd, cappedContent.length);
|
||||
offset = prefixEnd + message.text.length;
|
||||
}
|
||||
parts.push(cappedContent.slice(cursor));
|
||||
return parts.join('');
|
||||
} catch {
|
||||
return cappedContent;
|
||||
}
|
||||
}
|
||||
|
||||
function decodeHtmlEntities(value: string): string {
|
||||
if (!value.includes('&')) return value;
|
||||
return value
|
||||
.replace(/&(?:amp;){2,}/gi, '&')
|
||||
.replace(/&#(?:x([0-9a-f]{1,6})|([0-9]{1,7}));?/gi, (match, hex: string, decimal: string) => {
|
||||
const codePoint = Number.parseInt(hex ?? decimal, hex ? 16 : 10);
|
||||
if (!Number.isInteger(codePoint)
|
||||
|| codePoint <= 0
|
||||
|| codePoint > 0x10ffff
|
||||
|| (codePoint >= 0xd800 && codePoint <= 0xdfff)) {
|
||||
return match;
|
||||
}
|
||||
return String.fromCodePoint(codePoint);
|
||||
})
|
||||
.replace(/&([a-z][a-z0-9]+);/gi, (match, name: string) =>
|
||||
NAMED_HTML_ENTITIES[name.toLowerCase()] ?? match)
|
||||
.replace(
|
||||
/&(amp|apos|colon|emsp|ensp|gt|hairsp|lt|newline|nbsp|quot|tab|thinsp|zwj|zwnj|zwsp)(?=[^a-z0-9;]|$)/gi,
|
||||
(_match, name: string) => NAMED_HTML_ENTITIES[name.toLowerCase()],
|
||||
);
|
||||
}
|
||||
|
||||
function decodePercentEncoding(value: string): string {
|
||||
if (!/[+%]/.test(value)) return value;
|
||||
const withSpaces = value.replace(/\+/g, ' ');
|
||||
return withSpaces.replace(/(?:%[0-9a-f]{2})+/gi, (run) => {
|
||||
try {
|
||||
return decodeURIComponent(run);
|
||||
} catch {
|
||||
const bytes = Uint8Array.from(
|
||||
run.match(/[0-9a-f]{2}/gi) ?? [],
|
||||
hex => Number.parseInt(hex, 16),
|
||||
);
|
||||
return new TextDecoder('utf-8', { fatal: false }).decode(bytes);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function decodeUnicodeEscapes(value: string): string {
|
||||
if (!/\\u/i.test(value)) return value;
|
||||
return value.replace(
|
||||
/\\u(?:\{([0-9a-f]{1,6})\}|([0-9a-f]{4}))/gi,
|
||||
(match, braced: string | undefined, fixed: string | undefined) => {
|
||||
const codePoint = Number.parseInt(braced ?? fixed ?? '', 16);
|
||||
if (!Number.isInteger(codePoint) || codePoint > 0x10ffff) return match;
|
||||
if (braced !== undefined) {
|
||||
if (codePoint >= 0xd800 && codePoint <= 0xdfff) return match;
|
||||
return String.fromCodePoint(codePoint);
|
||||
}
|
||||
return String.fromCharCode(codePoint);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function decodeHexEscapes(value: string): string {
|
||||
if (!/\\x/i.test(value)) return value;
|
||||
return value.replace(/\\x([0-9a-f]{2})/gi, (_match, hex: string) =>
|
||||
String.fromCharCode(Number.parseInt(hex, 16)));
|
||||
}
|
||||
|
||||
const MIXED_SCRIPT_CONFUSABLES: Readonly<Record<string, string>> = Object.freeze({
|
||||
'\u0391': 'A',
|
||||
'\u0392': 'B',
|
||||
'\u0395': 'E',
|
||||
'\u0396': 'Z',
|
||||
'\u0397': 'H',
|
||||
'\u0399': 'I',
|
||||
'\u039a': 'K',
|
||||
'\u039c': 'M',
|
||||
'\u039d': 'N',
|
||||
'\u039f': 'O',
|
||||
'\u03a1': 'P',
|
||||
'\u03a4': 'T',
|
||||
'\u03a5': 'Y',
|
||||
'\u03a7': 'X',
|
||||
'\u03b1': 'a',
|
||||
'\u03b5': 'e',
|
||||
'\u03b9': 'i',
|
||||
'\u03bf': 'o',
|
||||
'\u03c1': 'p',
|
||||
'\u03c7': 'x',
|
||||
'\u03f2': 'c',
|
||||
'\u03f9': 'C',
|
||||
'\u0405': 'S',
|
||||
'\u0406': 'I',
|
||||
'\u0408': 'J',
|
||||
'\u0410': 'A',
|
||||
'\u0412': 'B',
|
||||
'\u0415': 'E',
|
||||
'\u041a': 'K',
|
||||
'\u041c': 'M',
|
||||
'\u041d': 'H',
|
||||
'\u041e': 'O',
|
||||
'\u0420': 'P',
|
||||
'\u0421': 'C',
|
||||
'\u0422': 'T',
|
||||
'\u0425': 'X',
|
||||
'\u0430': 'a',
|
||||
'\u0435': 'e',
|
||||
'\u043e': 'o',
|
||||
'\u0440': 'p',
|
||||
'\u0441': 'c',
|
||||
'\u0443': 'y',
|
||||
'\u0445': 'x',
|
||||
'\u0455': 's',
|
||||
'\u0456': 'i',
|
||||
'\u0458': 'j',
|
||||
});
|
||||
|
||||
const MIXED_SCRIPT_CONFUSABLE_PATTERN = /[\u0391\u0392\u0395\u0396\u0397\u0399\u039a\u039c\u039d\u039f\u03a1\u03a4\u03a5\u03a7\u03b1\u03b5\u03b9\u03bf\u03c1\u03c7\u03f2\u03f9\u0405\u0406\u0408\u0410\u0412\u0415\u041a\u041c\u041d\u041e\u0420\u0421\u0422\u0425\u0430\u0435\u043e\u0440\u0441\u0443\u0445\u0455\u0456\u0458]/;
|
||||
const MIXED_SCRIPT_CONFUSABLE_REPLACE_PATTERN = new RegExp(
|
||||
MIXED_SCRIPT_CONFUSABLE_PATTERN.source,
|
||||
'g',
|
||||
);
|
||||
|
||||
function projectMixedScriptConfusables(value: string): string | undefined {
|
||||
if (!MIXED_SCRIPT_CONFUSABLE_PATTERN.test(value)) return undefined;
|
||||
let changed = false;
|
||||
const projected = value.replace(/[\p{L}\p{M}]+/gu, (token) => {
|
||||
if (!/[A-Za-z]/.test(token) || !MIXED_SCRIPT_CONFUSABLE_PATTERN.test(token)) return token;
|
||||
changed = true;
|
||||
return token.replace(
|
||||
MIXED_SCRIPT_CONFUSABLE_REPLACE_PATTERN,
|
||||
char => MIXED_SCRIPT_CONFUSABLES[char] ?? char,
|
||||
);
|
||||
});
|
||||
return changed ? projected : undefined;
|
||||
}
|
||||
|
||||
const BASE64_CANDIDATE_PATTERN = /(?:^|[^A-Za-z0-9+/_=-])([A-Za-z0-9+/_-]{24,}={0,2})(?=$|[^A-Za-z0-9+/_=-])/g;
|
||||
const MAX_BASE64_CANDIDATES = 16;
|
||||
const MAX_BASE64_CANDIDATE_CHARS = 262_144;
|
||||
const MAX_BASE64_TOTAL_CHARS = 524_288;
|
||||
const MAX_BASE64_DEPTH = 4;
|
||||
|
||||
type Base64Candidate = {
|
||||
value: string;
|
||||
start: number;
|
||||
end: number;
|
||||
};
|
||||
|
||||
function collectBase64Candidates(source: string, allowImplicitWrapped = false): {
|
||||
candidates: Base64Candidate[];
|
||||
complete: boolean;
|
||||
} {
|
||||
const wrappedBlocks: Base64Candidate[] = [];
|
||||
|
||||
const trimmed = source.trim();
|
||||
const wrappedChunks = trimmed.split(/[ \t\r\n]+/);
|
||||
const wrapWidth = wrappedChunks[0]?.length ?? 0;
|
||||
if (allowImplicitWrapped
|
||||
&& wrappedChunks.length > 1
|
||||
&& wrapWidth >= 4
|
||||
&& wrapWidth <= 76
|
||||
&& wrapWidth % 4 === 0
|
||||
&& wrappedChunks.every(chunk => /^[A-Za-z0-9+/_-]+={0,2}$/.test(chunk))
|
||||
&& wrappedChunks.slice(0, -1).every(chunk => chunk.length === wrapWidth && !chunk.includes('='))
|
||||
&& wrappedChunks.at(-1)!.length <= wrapWidth) {
|
||||
const candidate = wrappedChunks.join('');
|
||||
if (candidate.length >= 24) {
|
||||
const start = source.length - source.trimStart().length;
|
||||
wrappedBlocks.push({ value: candidate, start, end: start + trimmed.length });
|
||||
}
|
||||
}
|
||||
|
||||
const directivePattern = /(?:\bdecode\b[^\r\n:]{0,160}\bbase64\b|\bbase64\b[^\r\n:]{0,160}\bdecode\b)[^\r\n:]{0,160}:/gi;
|
||||
for (const directive of source.matchAll(directivePattern)) {
|
||||
const tailStart = (directive.index ?? 0) + directive[0].length;
|
||||
const tail = source.slice(tailStart);
|
||||
const wrapped = tail.match(/^[ \t\r\n]*([A-Za-z0-9+/_=-]+(?:[ \t\r\n]+[A-Za-z0-9+/_=-]+)*)/);
|
||||
if (!wrapped) continue;
|
||||
const captured = wrapped[1];
|
||||
const start = tailStart + wrapped[0].indexOf(captured);
|
||||
let candidate = '';
|
||||
let end = start;
|
||||
for (const chunk of captured.matchAll(/[A-Za-z0-9+/_=-]+/g)) {
|
||||
candidate += chunk[0];
|
||||
end = start + (chunk.index ?? 0) + chunk[0].length;
|
||||
if (chunk[0].includes('=')) break;
|
||||
}
|
||||
if (candidate.length >= 24) wrappedBlocks.push({ value: candidate, start, end });
|
||||
}
|
||||
|
||||
const candidates: Base64Candidate[] = [];
|
||||
const contiguousPattern = new RegExp(BASE64_CANDIDATE_PATTERN.source, 'g');
|
||||
for (const match of source.matchAll(contiguousPattern)) {
|
||||
const value = match[1];
|
||||
const start = (match.index ?? 0) + match[0].length - value.length;
|
||||
const end = start + value.length;
|
||||
if (wrappedBlocks.some(block => start >= block.start && end <= block.end)) continue;
|
||||
candidates.push({ value, start, end });
|
||||
}
|
||||
candidates.push(...wrappedBlocks);
|
||||
candidates.sort((left, right) => left.start - right.start || left.end - right.end);
|
||||
return { candidates, complete: true };
|
||||
}
|
||||
|
||||
function decodeBase64Text(candidate: string): string | undefined {
|
||||
let normalized = candidate.replace(/-/g, '+').replace(/_/g, '/');
|
||||
const remainder = normalized.length % 4;
|
||||
if (remainder === 1) return undefined;
|
||||
if (remainder > 0) normalized += '='.repeat(4 - remainder);
|
||||
|
||||
try {
|
||||
const binary = atob(normalized);
|
||||
const bytes = Uint8Array.from(binary, char => char.charCodeAt(0));
|
||||
const decoded = new TextDecoder('utf-8').decode(bytes);
|
||||
if (!decoded) return undefined;
|
||||
|
||||
let printable = 0;
|
||||
let total = 0;
|
||||
for (const char of decoded) {
|
||||
total++;
|
||||
const codePoint = char.codePointAt(0) ?? 0;
|
||||
const isPrintable = char === '\n' || char === '\r' || char === '\t'
|
||||
|| (codePoint >= 0x20 && codePoint !== 0x7f);
|
||||
if (codePoint !== 0xfffd && isPrintable) {
|
||||
printable++;
|
||||
}
|
||||
}
|
||||
return total > 0 && printable / total >= 0.85 ? decoded : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function addProjection(projections: Set<string>, value: string): void {
|
||||
if (projections.has(value)) return;
|
||||
projections.add(value);
|
||||
const confusable = projectMixedScriptConfusables(value);
|
||||
if (confusable !== undefined) projections.add(confusable);
|
||||
}
|
||||
|
||||
const FORMAT_CHARACTER_PATTERN = /\p{Cf}/u;
|
||||
const DEFAULT_IGNORABLE_PATTERN = /\p{Default_Ignorable_Code_Point}/u;
|
||||
|
||||
function isHiddenSeparator(char: string): boolean {
|
||||
const codePoint = char.codePointAt(0) ?? 0;
|
||||
if (codePoint <= 0x9f) {
|
||||
return codePoint <= 0x08
|
||||
|| codePoint === 0x0b
|
||||
|| codePoint === 0x0c
|
||||
|| (codePoint >= 0x0e && codePoint <= 0x1f)
|
||||
|| codePoint >= 0x7f;
|
||||
}
|
||||
if (codePoint >= 0xd800 && codePoint <= 0xdfff) return true;
|
||||
return FORMAT_CHARACTER_PATTERN.test(char) || DEFAULT_IGNORABLE_PATTERN.test(char);
|
||||
}
|
||||
|
||||
function replaceHiddenSeparators(value: string, replacement: string): string {
|
||||
let containsHidden = false;
|
||||
for (const char of value) {
|
||||
if (isHiddenSeparator(char)) {
|
||||
containsHidden = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!containsHidden) return value;
|
||||
|
||||
let projected = '';
|
||||
for (const char of value) projected += isHiddenSeparator(char) ? replacement : char;
|
||||
return projected;
|
||||
}
|
||||
|
||||
function projectDelimitedWords(value: string): string | undefined {
|
||||
const projected = value.replace(
|
||||
/(\p{L})([\p{P}\p{S}\p{White_Space}]+)(?=\p{L})/gu,
|
||||
(match, letter: string, separators: string) =>
|
||||
/[\p{P}\p{S}]/u.test(separators) ? `${letter} ` : match,
|
||||
);
|
||||
return projected === value ? undefined : projected;
|
||||
}
|
||||
|
||||
type HtmlTagBoundary =
|
||||
| { kind: 'close'; index: number }
|
||||
| { kind: 'nested'; index: number }
|
||||
| { kind: 'eof'; index: number };
|
||||
|
||||
function findHtmlTagBoundary(value: string, start: number): HtmlTagBoundary {
|
||||
let quote: '"' | "'" | undefined;
|
||||
for (let index = start; index < value.length; index++) {
|
||||
const char = value[index];
|
||||
if (char === '<') return { kind: 'nested', index };
|
||||
if (quote) {
|
||||
if (char === quote) quote = undefined;
|
||||
} else if (char === '"' || char === "'") {
|
||||
quote = char;
|
||||
} else if (char === '>') {
|
||||
return { kind: 'close', index };
|
||||
}
|
||||
}
|
||||
return { kind: 'eof', index: value.length };
|
||||
}
|
||||
|
||||
function normalizeHtmlToken(value: string): string {
|
||||
return value.replace(/[:_-]+/g, ' ');
|
||||
}
|
||||
|
||||
function extractHtmlAttributeTokens(value: string, start: number, end: number): string {
|
||||
const tokens: string[] = [];
|
||||
let index = start;
|
||||
while (index < end) {
|
||||
while (index < end && /[\s/]/.test(value[index])) index++;
|
||||
const nameStart = index;
|
||||
while (index < end && !/[\s=/>]/.test(value[index])) index++;
|
||||
if (index === nameStart) {
|
||||
index++;
|
||||
continue;
|
||||
}
|
||||
const name = normalizeHtmlToken(value.slice(nameStart, index));
|
||||
while (index < end && /\s/.test(value[index])) index++;
|
||||
if (value[index] !== '=') {
|
||||
if (name) tokens.push(name);
|
||||
continue;
|
||||
}
|
||||
index++;
|
||||
while (index < end && /\s/.test(value[index])) index++;
|
||||
|
||||
const quote = value[index] === '"' || value[index] === "'"
|
||||
? value[index]
|
||||
: undefined;
|
||||
if (quote) index++;
|
||||
const valueStart = index;
|
||||
if (quote) {
|
||||
while (index < end && value[index] !== quote) index++;
|
||||
} else {
|
||||
while (index < end && !/[\s>]/.test(value[index])) index++;
|
||||
}
|
||||
if (index > valueStart) tokens.push(normalizeHtmlToken(value.slice(valueStart, index)));
|
||||
if (quote && index < end) index++;
|
||||
}
|
||||
return tokens.join(' ');
|
||||
}
|
||||
|
||||
function stripHtmlMarkup(value: string): {
|
||||
rendered: string;
|
||||
tagNames: string;
|
||||
attributes: string;
|
||||
lexical: string;
|
||||
} {
|
||||
if (!value.includes('<')) {
|
||||
return { rendered: value, tagNames: value, attributes: value, lexical: value };
|
||||
}
|
||||
|
||||
const rendered: string[] = [];
|
||||
const tagNames: string[] = [];
|
||||
const attributes: string[] = [];
|
||||
const lexical: string[] = [];
|
||||
for (let index = 0; index < value.length;) {
|
||||
if (value.startsWith('<!--', index)) {
|
||||
const commentEnd = value.indexOf('-->', index + 4);
|
||||
if (commentEnd === -1) {
|
||||
const visibleTail = value.slice(index + 4);
|
||||
rendered.push(visibleTail);
|
||||
tagNames.push(visibleTail);
|
||||
attributes.push(visibleTail);
|
||||
lexical.push(visibleTail);
|
||||
break;
|
||||
}
|
||||
const commentText = value.slice(index + 4, commentEnd);
|
||||
if (commentText) {
|
||||
tagNames.push(' ', commentText, ' ');
|
||||
attributes.push(' ', commentText, ' ');
|
||||
lexical.push(' ', commentText, ' ');
|
||||
}
|
||||
index = commentEnd + 3;
|
||||
continue;
|
||||
}
|
||||
if (value.startsWith('-->', index)) {
|
||||
index += 3;
|
||||
continue;
|
||||
}
|
||||
if (value[index] === '<' && /[!/A-Za-z?]/.test(value[index + 1] ?? '')) {
|
||||
let tagNameEnd = index + 1;
|
||||
if (value[tagNameEnd] === '/') tagNameEnd++;
|
||||
const tagNameStart = tagNameEnd;
|
||||
while (/[A-Za-z0-9:!_-]/.test(value[tagNameEnd] ?? '')) tagNameEnd++;
|
||||
const tagName = normalizeHtmlToken(value.slice(tagNameStart, tagNameEnd));
|
||||
const boundary = findHtmlTagBoundary(value, tagNameEnd);
|
||||
if (boundary.kind === 'close') {
|
||||
const attributeTokens = extractHtmlAttributeTokens(
|
||||
value,
|
||||
tagNameEnd,
|
||||
boundary.index,
|
||||
);
|
||||
if (tagName) tagNames.push(' ', tagName, ' ');
|
||||
if (attributeTokens) attributes.push(' ', attributeTokens, ' ');
|
||||
if (tagName || attributeTokens) {
|
||||
lexical.push(' ', tagName, ' ', attributeTokens, ' ');
|
||||
}
|
||||
index = boundary.index + 1;
|
||||
continue;
|
||||
}
|
||||
const visibleTail = value.slice(tagNameEnd, boundary.index);
|
||||
rendered.push(visibleTail);
|
||||
if (tagName) tagNames.push(' ', tagName, ' ');
|
||||
tagNames.push(visibleTail);
|
||||
attributes.push(visibleTail);
|
||||
if (tagName) lexical.push(' ', tagName, ' ');
|
||||
lexical.push(visibleTail);
|
||||
if (boundary.kind === 'eof') break;
|
||||
index = boundary.index;
|
||||
continue;
|
||||
}
|
||||
rendered.push(value[index]);
|
||||
tagNames.push(value[index]);
|
||||
attributes.push(value[index]);
|
||||
lexical.push(value[index]);
|
||||
index++;
|
||||
}
|
||||
return {
|
||||
rendered: rendered.join(''),
|
||||
tagNames: tagNames.join(''),
|
||||
attributes: attributes.join(''),
|
||||
lexical: lexical.join(''),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizedIngressProjections(value: string): {
|
||||
projections: string[];
|
||||
complete: boolean;
|
||||
};
|
||||
function normalizedIngressProjections(value: string, includeBase64: boolean): {
|
||||
projections: string[];
|
||||
complete: boolean;
|
||||
};
|
||||
function normalizedIngressProjections(value: string, includeBase64 = true): {
|
||||
projections: string[];
|
||||
complete: boolean;
|
||||
} {
|
||||
const maxPasses = 64;
|
||||
const maxWork = MAX_EXTERNAL_MEMORY_INGRESS_CHARS + 1;
|
||||
const originalProjection = value.normalize('NFKC');
|
||||
const normalizationStages = new Set([originalProjection]);
|
||||
let decodedProjection = originalProjection;
|
||||
let complete = false;
|
||||
let work = 0;
|
||||
for (let pass = 0; pass < maxPasses; pass++) {
|
||||
work += decodedProjection.length;
|
||||
if (work > maxWork) break;
|
||||
const decodedText = decodeUnicodeEscapes(decodeHexEscapes(
|
||||
decodeHtmlEntities(decodePercentEncoding(decodedProjection)),
|
||||
));
|
||||
const decoded = decodedText === decodedProjection
|
||||
? decodedProjection
|
||||
: decodedText.normalize('NFKC');
|
||||
if (decoded === decodedProjection) {
|
||||
complete = true;
|
||||
break;
|
||||
}
|
||||
normalizationStages.add(decoded);
|
||||
decodedProjection = decoded;
|
||||
}
|
||||
if (!complete) return { projections: [decodedProjection], complete: false };
|
||||
|
||||
const htmlProjections = stripHtmlMarkup(decodedProjection);
|
||||
const projections = new Set<string>();
|
||||
for (const projection of normalizationStages) addProjection(projections, projection);
|
||||
for (const projection of new Set([
|
||||
htmlProjections.rendered,
|
||||
htmlProjections.tagNames,
|
||||
htmlProjections.attributes,
|
||||
htmlProjections.lexical,
|
||||
])) {
|
||||
let unformatted = projection;
|
||||
if (unformatted.includes('[')) {
|
||||
unformatted = unformatted
|
||||
.replace(/!\[([^\]]*)\]\([^)]*\)/g, '$1')
|
||||
.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
|
||||
.replace(/\[([^\]]+)\]\[[^\]]*\]/g, '$1');
|
||||
}
|
||||
if (/[*_~`]/.test(unformatted)) unformatted = unformatted.replace(/[*_~`]/g, '');
|
||||
const compact = replaceHiddenSeparators(unformatted, '');
|
||||
addProjection(projections, compact);
|
||||
if (compact !== unformatted) {
|
||||
addProjection(projections, replaceHiddenSeparators(unformatted, ' '));
|
||||
}
|
||||
}
|
||||
|
||||
if (!includeBase64) return { projections: [...projections], complete: true };
|
||||
|
||||
const candidateLineages = new Map<number, Map<string, number[]>>();
|
||||
let nextCandidateLineage = 1;
|
||||
const resolveCandidateLineage = (
|
||||
parentLineage: number,
|
||||
candidate: string,
|
||||
ordinal: number,
|
||||
): { lineage: number; created: boolean } => {
|
||||
let candidates = candidateLineages.get(parentLineage);
|
||||
if (!candidates) {
|
||||
candidates = new Map();
|
||||
candidateLineages.set(parentLineage, candidates);
|
||||
}
|
||||
let lineages = candidates.get(candidate);
|
||||
if (!lineages) {
|
||||
lineages = [];
|
||||
candidates.set(candidate, lineages);
|
||||
}
|
||||
const existing = lineages[ordinal];
|
||||
if (existing !== undefined) return { lineage: existing, created: false };
|
||||
const lineage = nextCandidateLineage++;
|
||||
lineages[ordinal] = lineage;
|
||||
return { lineage, created: true };
|
||||
};
|
||||
let decodedCandidateCount = 0;
|
||||
let decodedCandidateChars = 0;
|
||||
let base64Sources = [...projections].map(value => ({ value, lineage: 0 }));
|
||||
let reachedDepthLimit = true;
|
||||
for (let depth = 0; depth < MAX_BASE64_DEPTH; depth++) {
|
||||
const decodedValues: Array<{ value: string; lineage: number }> = [];
|
||||
for (const source of base64Sources) {
|
||||
const collected = collectBase64Candidates(source.value, depth > 0);
|
||||
if (!collected.complete) return { projections: [...projections], complete: false };
|
||||
const occurrenceCounts = new Map<string, number>();
|
||||
for (const occurrence of collected.candidates) {
|
||||
const candidate = occurrence.value;
|
||||
const ordinal = occurrenceCounts.get(candidate) ?? 0;
|
||||
occurrenceCounts.set(candidate, ordinal + 1);
|
||||
const occurrenceLineage = resolveCandidateLineage(
|
||||
source.lineage,
|
||||
candidate,
|
||||
ordinal,
|
||||
);
|
||||
if (!occurrenceLineage.created) continue;
|
||||
if (candidate.length > MAX_BASE64_CANDIDATE_CHARS) {
|
||||
return { projections: [...projections], complete: false };
|
||||
}
|
||||
const decoded = decodeBase64Text(candidate);
|
||||
if (decoded === undefined) continue;
|
||||
decodedCandidateCount++;
|
||||
decodedCandidateChars += candidate.length;
|
||||
if (decodedCandidateCount > MAX_BASE64_CANDIDATES
|
||||
|| decodedCandidateChars > MAX_BASE64_TOTAL_CHARS) {
|
||||
return { projections: [...projections], complete: false };
|
||||
}
|
||||
decodedValues.push({ value: decoded, lineage: occurrenceLineage.lineage });
|
||||
}
|
||||
}
|
||||
if (decodedValues.length === 0) {
|
||||
reachedDepthLimit = false;
|
||||
break;
|
||||
}
|
||||
|
||||
const nextSources: Array<{ value: string; lineage: number }> = [];
|
||||
for (const decoded of decodedValues) {
|
||||
const nested = normalizedIngressProjections(decoded.value, false);
|
||||
if (!nested.complete) return { projections: [...projections], complete: false };
|
||||
for (const projection of nested.projections) {
|
||||
addProjection(projections, projection);
|
||||
nextSources.push({ value: projection, lineage: decoded.lineage });
|
||||
}
|
||||
}
|
||||
base64Sources = nextSources;
|
||||
}
|
||||
|
||||
if (reachedDepthLimit && base64Sources.length > 0) {
|
||||
for (const source of base64Sources) {
|
||||
const collected = collectBase64Candidates(source.value, true);
|
||||
if (!collected.complete) return { projections: [...projections], complete: false };
|
||||
const occurrenceCounts = new Map<string, number>();
|
||||
for (const occurrence of collected.candidates) {
|
||||
const candidate = occurrence.value;
|
||||
const ordinal = occurrenceCounts.get(candidate) ?? 0;
|
||||
occurrenceCounts.set(candidate, ordinal + 1);
|
||||
const occurrenceSeen = candidateLineages
|
||||
.get(source.lineage)
|
||||
?.get(candidate)?.[ordinal] !== undefined;
|
||||
if (!occurrenceSeen
|
||||
&& decodeBase64Text(candidate) !== undefined) {
|
||||
return { projections: [...projections], complete: false };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { projections: [...projections], complete: true };
|
||||
}
|
||||
|
||||
/** Evaluate untrusted content before it can enter persistent memory. */
|
||||
export function evaluateExternalMemoryIngress(
|
||||
input: ExternalMemoryIngressInput,
|
||||
): ExternalMemoryIngressDecision {
|
||||
const title = input.title ?? '';
|
||||
if (title.length > MAX_EXTERNAL_MEMORY_INGRESS_CHARS
|
||||
|| input.content.length > MAX_EXTERNAL_MEMORY_INGRESS_CHARS - title.length) {
|
||||
return {
|
||||
action: 'block',
|
||||
reason: 'prompt_injection',
|
||||
scan: { safe: false, score: 0.6, flags: ['normalization_limit'] },
|
||||
};
|
||||
}
|
||||
|
||||
const projection = `${title}\n${input.content}`;
|
||||
let scan = scanForInjection(projection, 'tool_output');
|
||||
if (scan.safe) {
|
||||
const normalizedIngress = normalizedIngressProjections(projection);
|
||||
for (const normalized of normalizedIngress.projections) {
|
||||
if (normalized !== projection) {
|
||||
const normalizedScan = scanForInjection(normalized, 'tool_output');
|
||||
if (!normalizedScan.safe) {
|
||||
scan = normalizedScan;
|
||||
break;
|
||||
}
|
||||
}
|
||||
const delimited = projectDelimitedWords(normalized);
|
||||
if (delimited !== undefined) {
|
||||
const delimitedScan = scanForInjection(delimited, 'tool_output');
|
||||
if (!delimitedScan.safe) {
|
||||
scan = delimitedScan;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (scan.safe && !normalizedIngress.complete) {
|
||||
scan = { safe: false, score: 0.6, flags: ['normalization_limit'] };
|
||||
}
|
||||
}
|
||||
|
||||
return scan.safe
|
||||
? { action: 'allow', scan }
|
||||
: { action: 'block', reason: 'prompt_injection', scan };
|
||||
}
|
||||
Reference in New Issue
Block a user