import { scanForInjection, type ScanResult } from './injection-scanner.js'; export interface ExternalMemoryIngressInput { title?: string; content: string; } export interface ExternalMemoryProjectionInput { content: string; messages?: unknown; parseMethod?: unknown; maxChars?: number; } export type ExternalMemoryIngressDecision = | { action: 'allow'; scan: ScanResult } | { action: 'block'; reason: 'prompt_injection'; scan: ScanResult }; // RawArchive supports one million characters per item. Keep the same explicit // public-ingress budget, checked before concatenation, scanning, or normalization. const MAX_EXTERNAL_MEMORY_INGRESS_CHARS = 1_000_000; const NAMED_HTML_ENTITIES: Readonly> = Object.freeze({ af: '\u2061', amp: '&', applyfunction: '\u2061', apos: "'", colon: ':', emsp: ' ', ensp: ' ', gt: '>', hairsp: ' ', ic: '\u2063', invisiblecomma: '\u2063', invisibletimes: '\u2062', it: '\u2062', lrm: '\u200e', lt: '<', negativemediumspace: '\u200b', negativethickspace: '\u200b', negativethinspace: '\u200b', negativeverythinspace: '\u200b', newline: '\n', nbsp: ' ', nobreak: '\u2060', quot: '"', rlm: '\u200f', shy: '\u00ad', tab: '\t', thinsp: ' ', zwj: '', zwnj: '', zwsp: '', zerowidthspace: '\u200b', }); type CanonicalMemoryMessage = { role: 'user' | 'assistant' | 'system'; text: string; }; /** * Return only the attacker-controlled text represented by a stored adapter * projection. Role prefixes may be omitted only when plain canonical messages * exactly reproduce the full content and did not come from universal raw text. */ export function projectExternalMemoryContent(input: ExternalMemoryProjectionInput): string { let cappedContent = ''; try { const content = typeof input.content === 'string' ? input.content : ''; const maxChars = input.maxChars; cappedContent = maxChars === undefined || !Number.isSafeInteger(maxChars) || maxChars < 0 ? content : content.slice(0, maxChars); if (input.parseMethod === 'universal-text' || !Array.isArray(input.messages) || input.messages.length === 0) { return cappedContent; } const messages: CanonicalMemoryMessage[] = []; for (const candidate of input.messages) { if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) { return cappedContent; } const prototype = Object.getPrototypeOf(candidate); if (prototype !== Object.prototype && prototype !== null) return cappedContent; const roleDescriptor = Object.getOwnPropertyDescriptor(candidate, 'role'); const textDescriptor = Object.getOwnPropertyDescriptor(candidate, 'text'); if (!roleDescriptor || !('value' in roleDescriptor) || !textDescriptor || !('value' in textDescriptor)) { return cappedContent; } const role = roleDescriptor.value as unknown; const text = textDescriptor.value as unknown; if ((role !== 'user' && role !== 'assistant' && role !== 'system') || typeof text !== 'string') { return cappedContent; } messages.push({ role, text }); } const serialized = messages .map(message => `${message.role}: ${message.text}`) .join('\n\n'); if (serialized !== content) return cappedContent; const parts: string[] = []; let cursor = 0; let offset = 0; for (const [index, message] of messages.entries()) { if (index > 0) offset += 2; const prefixStart = offset; const prefixEnd = prefixStart + `${message.role}: `.length; if (prefixStart >= cappedContent.length) break; if (message.role === 'system') return cappedContent; parts.push(cappedContent.slice(cursor, prefixStart)); cursor = Math.min(prefixEnd, cappedContent.length); offset = prefixEnd + message.text.length; } parts.push(cappedContent.slice(cursor)); return parts.join(''); } catch { return cappedContent; } } function decodeHtmlEntities(value: string): string { if (!value.includes('&')) return value; return value .replace(/&(?:amp;){2,}/gi, '&') .replace(/&#(?:x([0-9a-f]{1,6})|([0-9]{1,7}));?/gi, (match, hex: string, decimal: string) => { const codePoint = Number.parseInt(hex ?? decimal, hex ? 16 : 10); if (!Number.isInteger(codePoint) || codePoint <= 0 || codePoint > 0x10ffff || (codePoint >= 0xd800 && codePoint <= 0xdfff)) { return match; } return String.fromCodePoint(codePoint); }) .replace(/&([a-z][a-z0-9]+);/gi, (match, name: string) => NAMED_HTML_ENTITIES[name.toLowerCase()] ?? match) .replace( /&(amp|apos|colon|emsp|ensp|gt|hairsp|lt|newline|nbsp|quot|tab|thinsp|zwj|zwnj|zwsp)(?=[^a-z0-9;]|$)/gi, (_match, name: string) => NAMED_HTML_ENTITIES[name.toLowerCase()], ); } function decodePercentEncoding(value: string): string { if (!/[+%]/.test(value)) return value; const withSpaces = value.replace(/\+/g, ' '); return withSpaces.replace(/(?:%[0-9a-f]{2})+/gi, (run) => { try { return decodeURIComponent(run); } catch { const bytes = Uint8Array.from( run.match(/[0-9a-f]{2}/gi) ?? [], hex => Number.parseInt(hex, 16), ); return new TextDecoder('utf-8', { fatal: false }).decode(bytes); } }); } function decodeUnicodeEscapes(value: string): string { if (!/\\u/i.test(value)) return value; return value.replace( /\\u(?:\{([0-9a-f]{1,6})\}|([0-9a-f]{4}))/gi, (match, braced: string | undefined, fixed: string | undefined) => { const codePoint = Number.parseInt(braced ?? fixed ?? '', 16); if (!Number.isInteger(codePoint) || codePoint > 0x10ffff) return match; if (braced !== undefined) { if (codePoint >= 0xd800 && codePoint <= 0xdfff) return match; return String.fromCodePoint(codePoint); } return String.fromCharCode(codePoint); }, ); } function decodeHexEscapes(value: string): string { if (!/\\x/i.test(value)) return value; return value.replace(/\\x([0-9a-f]{2})/gi, (_match, hex: string) => String.fromCharCode(Number.parseInt(hex, 16))); } const MIXED_SCRIPT_CONFUSABLES: Readonly> = Object.freeze({ '\u0391': 'A', '\u0392': 'B', '\u0395': 'E', '\u0396': 'Z', '\u0397': 'H', '\u0399': 'I', '\u039a': 'K', '\u039c': 'M', '\u039d': 'N', '\u039f': 'O', '\u03a1': 'P', '\u03a4': 'T', '\u03a5': 'Y', '\u03a7': 'X', '\u03b1': 'a', '\u03b5': 'e', '\u03b9': 'i', '\u03bf': 'o', '\u03c1': 'p', '\u03c7': 'x', '\u03f2': 'c', '\u03f9': 'C', '\u0405': 'S', '\u0406': 'I', '\u0408': 'J', '\u0410': 'A', '\u0412': 'B', '\u0415': 'E', '\u041a': 'K', '\u041c': 'M', '\u041d': 'H', '\u041e': 'O', '\u0420': 'P', '\u0421': 'C', '\u0422': 'T', '\u0425': 'X', '\u0430': 'a', '\u0435': 'e', '\u043e': 'o', '\u0440': 'p', '\u0441': 'c', '\u0443': 'y', '\u0445': 'x', '\u0455': 's', '\u0456': 'i', '\u0458': 'j', }); const MIXED_SCRIPT_CONFUSABLE_PATTERN = /[\u0391\u0392\u0395\u0396\u0397\u0399\u039a\u039c\u039d\u039f\u03a1\u03a4\u03a5\u03a7\u03b1\u03b5\u03b9\u03bf\u03c1\u03c7\u03f2\u03f9\u0405\u0406\u0408\u0410\u0412\u0415\u041a\u041c\u041d\u041e\u0420\u0421\u0422\u0425\u0430\u0435\u043e\u0440\u0441\u0443\u0445\u0455\u0456\u0458]/; const MIXED_SCRIPT_CONFUSABLE_REPLACE_PATTERN = new RegExp( MIXED_SCRIPT_CONFUSABLE_PATTERN.source, 'g', ); function projectMixedScriptConfusables(value: string): string | undefined { if (!MIXED_SCRIPT_CONFUSABLE_PATTERN.test(value)) return undefined; let changed = false; const projected = value.replace(/[\p{L}\p{M}]+/gu, (token) => { if (!/[A-Za-z]/.test(token) || !MIXED_SCRIPT_CONFUSABLE_PATTERN.test(token)) return token; changed = true; return token.replace( MIXED_SCRIPT_CONFUSABLE_REPLACE_PATTERN, char => MIXED_SCRIPT_CONFUSABLES[char] ?? char, ); }); return changed ? projected : undefined; } const BASE64_CANDIDATE_PATTERN = /(?:^|[^A-Za-z0-9+/_=-])([A-Za-z0-9+/_-]{24,}={0,2})(?=$|[^A-Za-z0-9+/_=-])/g; const MAX_BASE64_CANDIDATES = 16; const MAX_BASE64_CANDIDATE_CHARS = 262_144; const MAX_BASE64_TOTAL_CHARS = 524_288; const MAX_BASE64_DEPTH = 4; type Base64Candidate = { value: string; start: number; end: number; }; function collectBase64Candidates(source: string, allowImplicitWrapped = false): { candidates: Base64Candidate[]; complete: boolean; } { const wrappedBlocks: Base64Candidate[] = []; const trimmed = source.trim(); const wrappedChunks = trimmed.split(/[ \t\r\n]+/); const wrapWidth = wrappedChunks[0]?.length ?? 0; if (allowImplicitWrapped && wrappedChunks.length > 1 && wrapWidth >= 4 && wrapWidth <= 76 && wrapWidth % 4 === 0 && wrappedChunks.every(chunk => /^[A-Za-z0-9+/_-]+={0,2}$/.test(chunk)) && wrappedChunks.slice(0, -1).every(chunk => chunk.length === wrapWidth && !chunk.includes('=')) && wrappedChunks.at(-1)!.length <= wrapWidth) { const candidate = wrappedChunks.join(''); if (candidate.length >= 24) { const start = source.length - source.trimStart().length; wrappedBlocks.push({ value: candidate, start, end: start + trimmed.length }); } } const directivePattern = /(?:\bdecode\b[^\r\n:]{0,160}\bbase64\b|\bbase64\b[^\r\n:]{0,160}\bdecode\b)[^\r\n:]{0,160}:/gi; for (const directive of source.matchAll(directivePattern)) { const tailStart = (directive.index ?? 0) + directive[0].length; const tail = source.slice(tailStart); const wrapped = tail.match(/^[ \t\r\n]*([A-Za-z0-9+/_=-]+(?:[ \t\r\n]+[A-Za-z0-9+/_=-]+)*)/); if (!wrapped) continue; const captured = wrapped[1]; const start = tailStart + wrapped[0].indexOf(captured); let candidate = ''; let end = start; for (const chunk of captured.matchAll(/[A-Za-z0-9+/_=-]+/g)) { candidate += chunk[0]; end = start + (chunk.index ?? 0) + chunk[0].length; if (chunk[0].includes('=')) break; } if (candidate.length >= 24) wrappedBlocks.push({ value: candidate, start, end }); } const candidates: Base64Candidate[] = []; const contiguousPattern = new RegExp(BASE64_CANDIDATE_PATTERN.source, 'g'); for (const match of source.matchAll(contiguousPattern)) { const value = match[1]; const start = (match.index ?? 0) + match[0].length - value.length; const end = start + value.length; if (wrappedBlocks.some(block => start >= block.start && end <= block.end)) continue; candidates.push({ value, start, end }); } candidates.push(...wrappedBlocks); candidates.sort((left, right) => left.start - right.start || left.end - right.end); return { candidates, complete: true }; } function decodeBase64Text(candidate: string): string | undefined { let normalized = candidate.replace(/-/g, '+').replace(/_/g, '/'); const remainder = normalized.length % 4; if (remainder === 1) return undefined; if (remainder > 0) normalized += '='.repeat(4 - remainder); try { const binary = atob(normalized); const bytes = Uint8Array.from(binary, char => char.charCodeAt(0)); const decoded = new TextDecoder('utf-8').decode(bytes); if (!decoded) return undefined; let printable = 0; let total = 0; for (const char of decoded) { total++; const codePoint = char.codePointAt(0) ?? 0; const isPrintable = char === '\n' || char === '\r' || char === '\t' || (codePoint >= 0x20 && codePoint !== 0x7f); if (codePoint !== 0xfffd && isPrintable) { printable++; } } return total > 0 && printable / total >= 0.85 ? decoded : undefined; } catch { return undefined; } } function addProjection(projections: Set, value: string): void { if (projections.has(value)) return; projections.add(value); const confusable = projectMixedScriptConfusables(value); if (confusable !== undefined) projections.add(confusable); } const FORMAT_CHARACTER_PATTERN = /\p{Cf}/u; const DEFAULT_IGNORABLE_PATTERN = /\p{Default_Ignorable_Code_Point}/u; function isHiddenSeparator(char: string): boolean { const codePoint = char.codePointAt(0) ?? 0; if (codePoint <= 0x9f) { return codePoint <= 0x08 || codePoint === 0x0b || codePoint === 0x0c || (codePoint >= 0x0e && codePoint <= 0x1f) || codePoint >= 0x7f; } if (codePoint >= 0xd800 && codePoint <= 0xdfff) return true; return FORMAT_CHARACTER_PATTERN.test(char) || DEFAULT_IGNORABLE_PATTERN.test(char); } function replaceHiddenSeparators(value: string, replacement: string): string { let containsHidden = false; for (const char of value) { if (isHiddenSeparator(char)) { containsHidden = true; break; } } if (!containsHidden) return value; let projected = ''; for (const char of value) projected += isHiddenSeparator(char) ? replacement : char; return projected; } function projectDelimitedWords(value: string): string | undefined { const projected = value.replace( /(\p{L})([\p{P}\p{S}\p{White_Space}]+)(?=\p{L})/gu, (match, letter: string, separators: string) => /[\p{P}\p{S}]/u.test(separators) ? `${letter} ` : match, ); return projected === value ? undefined : projected; } type HtmlTagBoundary = | { kind: 'close'; index: number } | { kind: 'nested'; index: number } | { kind: 'eof'; index: number }; function findHtmlTagBoundary(value: string, start: number): HtmlTagBoundary { let quote: '"' | "'" | undefined; for (let index = start; index < value.length; index++) { const char = value[index]; if (char === '<') return { kind: 'nested', index }; if (quote) { if (char === quote) quote = undefined; } else if (char === '"' || char === "'") { quote = char; } else if (char === '>') { return { kind: 'close', index }; } } return { kind: 'eof', index: value.length }; } function normalizeHtmlToken(value: string): string { return value.replace(/[:_-]+/g, ' '); } function extractHtmlAttributeTokens(value: string, start: number, end: number): string { const tokens: string[] = []; let index = start; while (index < end) { while (index < end && /[\s/]/.test(value[index])) index++; const nameStart = index; while (index < end && !/[\s=/>]/.test(value[index])) index++; if (index === nameStart) { index++; continue; } const name = normalizeHtmlToken(value.slice(nameStart, index)); while (index < end && /\s/.test(value[index])) index++; if (value[index] !== '=') { if (name) tokens.push(name); continue; } index++; while (index < end && /\s/.test(value[index])) index++; const quote = value[index] === '"' || value[index] === "'" ? value[index] : undefined; if (quote) index++; const valueStart = index; if (quote) { while (index < end && value[index] !== quote) index++; } else { while (index < end && !/[\s>]/.test(value[index])) index++; } if (index > valueStart) tokens.push(normalizeHtmlToken(value.slice(valueStart, index))); if (quote && index < end) index++; } return tokens.join(' '); } function stripHtmlMarkup(value: string): { rendered: string; tagNames: string; attributes: string; lexical: string; } { if (!value.includes('<')) { return { rendered: value, tagNames: value, attributes: value, lexical: value }; } const rendered: string[] = []; const tagNames: string[] = []; const attributes: string[] = []; const lexical: string[] = []; for (let index = 0; index < value.length;) { if (value.startsWith('', index + 4); if (commentEnd === -1) { const visibleTail = value.slice(index + 4); rendered.push(visibleTail); tagNames.push(visibleTail); attributes.push(visibleTail); lexical.push(visibleTail); break; } const commentText = value.slice(index + 4, commentEnd); if (commentText) { tagNames.push(' ', commentText, ' '); attributes.push(' ', commentText, ' '); lexical.push(' ', commentText, ' '); } index = commentEnd + 3; continue; } if (value.startsWith('-->', index)) { index += 3; continue; } if (value[index] === '<' && /[!/A-Za-z?]/.test(value[index + 1] ?? '')) { let tagNameEnd = index + 1; if (value[tagNameEnd] === '/') tagNameEnd++; const tagNameStart = tagNameEnd; while (/[A-Za-z0-9:!_-]/.test(value[tagNameEnd] ?? '')) tagNameEnd++; const tagName = normalizeHtmlToken(value.slice(tagNameStart, tagNameEnd)); const boundary = findHtmlTagBoundary(value, tagNameEnd); if (boundary.kind === 'close') { const attributeTokens = extractHtmlAttributeTokens( value, tagNameEnd, boundary.index, ); if (tagName) tagNames.push(' ', tagName, ' '); if (attributeTokens) attributes.push(' ', attributeTokens, ' '); if (tagName || attributeTokens) { lexical.push(' ', tagName, ' ', attributeTokens, ' '); } index = boundary.index + 1; continue; } const visibleTail = value.slice(tagNameEnd, boundary.index); rendered.push(visibleTail); if (tagName) tagNames.push(' ', tagName, ' '); tagNames.push(visibleTail); attributes.push(visibleTail); if (tagName) lexical.push(' ', tagName, ' '); lexical.push(visibleTail); if (boundary.kind === 'eof') break; index = boundary.index; continue; } rendered.push(value[index]); tagNames.push(value[index]); attributes.push(value[index]); lexical.push(value[index]); index++; } return { rendered: rendered.join(''), tagNames: tagNames.join(''), attributes: attributes.join(''), lexical: lexical.join(''), }; } function normalizedIngressProjections(value: string): { projections: string[]; complete: boolean; }; function normalizedIngressProjections(value: string, includeBase64: boolean): { projections: string[]; complete: boolean; }; function normalizedIngressProjections(value: string, includeBase64 = true): { projections: string[]; complete: boolean; } { const maxPasses = 64; const maxWork = MAX_EXTERNAL_MEMORY_INGRESS_CHARS + 1; const originalProjection = value.normalize('NFKC'); const normalizationStages = new Set([originalProjection]); let decodedProjection = originalProjection; let complete = false; let work = 0; for (let pass = 0; pass < maxPasses; pass++) { work += decodedProjection.length; if (work > maxWork) break; const decodedText = decodeUnicodeEscapes(decodeHexEscapes( decodeHtmlEntities(decodePercentEncoding(decodedProjection)), )); const decoded = decodedText === decodedProjection ? decodedProjection : decodedText.normalize('NFKC'); if (decoded === decodedProjection) { complete = true; break; } normalizationStages.add(decoded); decodedProjection = decoded; } if (!complete) return { projections: [decodedProjection], complete: false }; const htmlProjections = stripHtmlMarkup(decodedProjection); const projections = new Set(); for (const projection of normalizationStages) addProjection(projections, projection); for (const projection of new Set([ htmlProjections.rendered, htmlProjections.tagNames, htmlProjections.attributes, htmlProjections.lexical, ])) { let unformatted = projection; if (unformatted.includes('[')) { unformatted = unformatted .replace(/!\[([^\]]*)\]\([^)]*\)/g, '$1') .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1') .replace(/\[([^\]]+)\]\[[^\]]*\]/g, '$1'); } if (/[*_~`]/.test(unformatted)) unformatted = unformatted.replace(/[*_~`]/g, ''); const compact = replaceHiddenSeparators(unformatted, ''); addProjection(projections, compact); if (compact !== unformatted) { addProjection(projections, replaceHiddenSeparators(unformatted, ' ')); } } if (!includeBase64) return { projections: [...projections], complete: true }; const candidateLineages = new Map>(); let nextCandidateLineage = 1; const resolveCandidateLineage = ( parentLineage: number, candidate: string, ordinal: number, ): { lineage: number; created: boolean } => { let candidates = candidateLineages.get(parentLineage); if (!candidates) { candidates = new Map(); candidateLineages.set(parentLineage, candidates); } let lineages = candidates.get(candidate); if (!lineages) { lineages = []; candidates.set(candidate, lineages); } const existing = lineages[ordinal]; if (existing !== undefined) return { lineage: existing, created: false }; const lineage = nextCandidateLineage++; lineages[ordinal] = lineage; return { lineage, created: true }; }; let decodedCandidateCount = 0; let decodedCandidateChars = 0; let base64Sources = [...projections].map(value => ({ value, lineage: 0 })); let reachedDepthLimit = true; for (let depth = 0; depth < MAX_BASE64_DEPTH; depth++) { const decodedValues: Array<{ value: string; lineage: number }> = []; for (const source of base64Sources) { const collected = collectBase64Candidates(source.value, depth > 0); if (!collected.complete) return { projections: [...projections], complete: false }; const occurrenceCounts = new Map(); for (const occurrence of collected.candidates) { const candidate = occurrence.value; const ordinal = occurrenceCounts.get(candidate) ?? 0; occurrenceCounts.set(candidate, ordinal + 1); const occurrenceLineage = resolveCandidateLineage( source.lineage, candidate, ordinal, ); if (!occurrenceLineage.created) continue; if (candidate.length > MAX_BASE64_CANDIDATE_CHARS) { return { projections: [...projections], complete: false }; } const decoded = decodeBase64Text(candidate); if (decoded === undefined) continue; decodedCandidateCount++; decodedCandidateChars += candidate.length; if (decodedCandidateCount > MAX_BASE64_CANDIDATES || decodedCandidateChars > MAX_BASE64_TOTAL_CHARS) { return { projections: [...projections], complete: false }; } decodedValues.push({ value: decoded, lineage: occurrenceLineage.lineage }); } } if (decodedValues.length === 0) { reachedDepthLimit = false; break; } const nextSources: Array<{ value: string; lineage: number }> = []; for (const decoded of decodedValues) { const nested = normalizedIngressProjections(decoded.value, false); if (!nested.complete) return { projections: [...projections], complete: false }; for (const projection of nested.projections) { addProjection(projections, projection); nextSources.push({ value: projection, lineage: decoded.lineage }); } } base64Sources = nextSources; } if (reachedDepthLimit && base64Sources.length > 0) { for (const source of base64Sources) { const collected = collectBase64Candidates(source.value, true); if (!collected.complete) return { projections: [...projections], complete: false }; const occurrenceCounts = new Map(); for (const occurrence of collected.candidates) { const candidate = occurrence.value; const ordinal = occurrenceCounts.get(candidate) ?? 0; occurrenceCounts.set(candidate, ordinal + 1); const occurrenceSeen = candidateLineages .get(source.lineage) ?.get(candidate)?.[ordinal] !== undefined; if (!occurrenceSeen && decodeBase64Text(candidate) !== undefined) { return { projections: [...projections], complete: false }; } } } } return { projections: [...projections], complete: true }; } /** Evaluate untrusted content before it can enter persistent memory. */ export function evaluateExternalMemoryIngress( input: ExternalMemoryIngressInput, ): ExternalMemoryIngressDecision { const title = input.title ?? ''; if (title.length > MAX_EXTERNAL_MEMORY_INGRESS_CHARS || input.content.length > MAX_EXTERNAL_MEMORY_INGRESS_CHARS - title.length) { return { action: 'block', reason: 'prompt_injection', scan: { safe: false, score: 0.6, flags: ['normalization_limit'] }, }; } const projection = `${title}\n${input.content}`; let scan = scanForInjection(projection, 'tool_output'); if (scan.safe) { const normalizedIngress = normalizedIngressProjections(projection); for (const normalized of normalizedIngress.projections) { if (normalized !== projection) { const normalizedScan = scanForInjection(normalized, 'tool_output'); if (!normalizedScan.safe) { scan = normalizedScan; break; } } const delimited = projectDelimitedWords(normalized); if (delimited !== undefined) { const delimitedScan = scanForInjection(delimited, 'tool_output'); if (!delimitedScan.safe) { scan = delimitedScan; break; } } } if (scan.safe && !normalizedIngress.complete) { scan = { safe: false, score: 0.6, flags: ['normalization_limit'] }; } } return scan.safe ? { action: 'allow', scan } : { action: 'block', reason: 'prompt_injection', scan }; }