moving
This commit is contained in:
@@ -20,8 +20,8 @@
|
||||
*/
|
||||
|
||||
import type { LLMCallFn } from './pipeline.js';
|
||||
import { scanForInjection } from '../injection-scanner.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
import { evaluateExternalMemoryIngress } from '../memory-ingress-guard.js';
|
||||
import { isNoiseName, normalizeEntityName } from '../mind/entity-normalizer.js';
|
||||
import type { KnowledgeGraph } from '../mind/knowledge.js';
|
||||
|
||||
@@ -138,6 +138,7 @@ function unwrapFencedBlock(text: string): string {
|
||||
*/
|
||||
function parseJsonlOutput(raw: string, validFrameIds: ReadonlySet<number>): KgEntity[] {
|
||||
const entities: KgEntity[] = [];
|
||||
const seenEntityFrames = new Set<string>();
|
||||
const cleaned = unwrapFencedBlock(raw);
|
||||
|
||||
for (const line of cleaned.split('\n')) {
|
||||
@@ -146,13 +147,16 @@ function parseJsonlOutput(raw: string, validFrameIds: ReadonlySet<number>): KgEn
|
||||
|
||||
let parsed: Record<string, unknown>;
|
||||
try {
|
||||
parsed = JSON.parse(trimmed) as Record<string, unknown>;
|
||||
const value: unknown = JSON.parse(trimmed);
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) continue;
|
||||
parsed = value as Record<string, unknown>;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
const frameId = Number(parsed.frame_id);
|
||||
if (!Number.isFinite(frameId) || !validFrameIds.has(frameId)) continue;
|
||||
const frameId = parsed.frame_id;
|
||||
if (typeof frameId !== 'number' || !Number.isSafeInteger(frameId) ||
|
||||
!validFrameIds.has(frameId)) continue;
|
||||
|
||||
const name = typeof parsed.name === 'string' ? parsed.name.trim() : '';
|
||||
if (name.length < 2) continue;
|
||||
@@ -163,12 +167,17 @@ function parseJsonlOutput(raw: string, validFrameIds: ReadonlySet<number>): KgEn
|
||||
const rawType = typeof parsed.type === 'string' ? parsed.type.toLowerCase().trim() : '';
|
||||
if (!(KG_ENTITY_TYPES as readonly string[]).includes(rawType)) continue;
|
||||
|
||||
const scan = scanForInjection(name, 'tool_output');
|
||||
if (!scan.safe) {
|
||||
log.warn('dropping extracted entity name with injection payload', { flags: scan.flags.join(',') });
|
||||
const ingress = evaluateExternalMemoryIngress({ content: name });
|
||||
if (ingress.action !== 'allow') {
|
||||
log.warn('dropping extracted entity name with injection payload', {
|
||||
flags: ingress.scan.flags.join(','),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const entityFrameKey = JSON.stringify([frameId, normalizeEntityName(name)]);
|
||||
if (seenEntityFrames.has(entityFrameKey)) continue;
|
||||
seenEntityFrames.add(entityFrameKey);
|
||||
entities.push({ frameId, name, type: rawType as KgEntityType });
|
||||
}
|
||||
|
||||
@@ -214,7 +223,14 @@ export interface WriteKgEntitiesResult {
|
||||
|
||||
function safeParseProps(raw: string | undefined | null): Record<string, unknown> {
|
||||
if (!raw) return {};
|
||||
try { return JSON.parse(raw) as Record<string, unknown>; } catch { return {}; }
|
||||
try {
|
||||
const value: unknown = JSON.parse(raw);
|
||||
return value && typeof value === 'object' && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -231,37 +247,54 @@ export function writeKgEntities(
|
||||
kg: KnowledgeGraph,
|
||||
extraction: KgEntityExtraction,
|
||||
): WriteKgEntitiesResult {
|
||||
const result: WriteKgEntitiesResult = { created: 0, updated: 0 };
|
||||
return kg.runInTransaction(() => {
|
||||
const result: WriteKgEntitiesResult = { created: 0, updated: 0 };
|
||||
|
||||
for (const entity of extraction.entities) {
|
||||
// Defense at the write seam (mirrors the cognify CLI): callers other than
|
||||
// extractKgEntities may not have noise-filtered.
|
||||
if (isNoiseName(entity.name)) continue;
|
||||
if (normalizeEntityName(entity.name).length < 3) continue;
|
||||
for (const entity of extraction.entities) {
|
||||
if (!entity || typeof entity !== 'object') continue;
|
||||
if (!Number.isSafeInteger(entity.frameId) || entity.frameId <= 0) continue;
|
||||
const name = typeof entity.name === 'string' ? entity.name.trim() : '';
|
||||
if (isNoiseName(name)) continue;
|
||||
if (normalizeEntityName(name).length < 3) continue;
|
||||
if (!(KG_ENTITY_TYPES as readonly string[]).includes(entity.type)) continue;
|
||||
if (evaluateExternalMemoryIngress({ content: name }).action !== 'allow') continue;
|
||||
|
||||
const existing = kg.findEntityByName(entity.name);
|
||||
if (existing) {
|
||||
const existingProps = safeParseProps(existing.properties);
|
||||
const seenCount = Number(existingProps.seen_count ?? 1) + 1;
|
||||
kg.updateEntity(existing.id, {
|
||||
properties: { ...existingProps, seen_count: seenCount },
|
||||
});
|
||||
kg.linkEntityToFrame(existing.id, entity.frameId);
|
||||
result.updated++;
|
||||
} else {
|
||||
try {
|
||||
const created = kg.createEntity(entity.type, entity.name, { seen_count: 1, source: 'cognify-llm' });
|
||||
kg.linkEntityToFrame(created.id, entity.frameId);
|
||||
result.created++;
|
||||
} catch (e: unknown) {
|
||||
// Ontology validation may reject — skip this entity, never abort the pass.
|
||||
log.warn('createEntity rejected extracted entity', {
|
||||
name: entity.name,
|
||||
error: e instanceof Error ? e.message : String(e),
|
||||
const existing = kg.findEntityByName(name);
|
||||
if (existing) {
|
||||
if (!kg.linkEntityToFrameStrict(existing.id, entity.frameId)) continue;
|
||||
const existingProps = safeParseProps(existing.properties);
|
||||
const previousSeenCount = Number(existingProps.seen_count ?? 1);
|
||||
const seenCount = (Number.isFinite(previousSeenCount) && previousSeenCount >= 0
|
||||
? previousSeenCount
|
||||
: 1) + 1;
|
||||
kg.updateEntity(existing.id, {
|
||||
properties: { ...existingProps, seen_count: seenCount },
|
||||
});
|
||||
result.updated++;
|
||||
} else {
|
||||
let created: { id: number };
|
||||
try {
|
||||
created = kg.createEntity(entity.type, name, {
|
||||
seen_count: 1,
|
||||
source: 'cognify-llm',
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message.startsWith('Validation failed:')) {
|
||||
log.warn('createEntity rejected extracted entity', {
|
||||
name,
|
||||
error: error.message,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (!kg.linkEntityToFrameStrict(created.id, entity.frameId)) {
|
||||
throw new Error(`KG writer failed to link entity ${created.id} to frame ${entity.frameId}`);
|
||||
}
|
||||
result.created++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -16,7 +16,10 @@ import type {
|
||||
} from './types.js';
|
||||
import { CLASSIFY_PROMPT, EXTRACT_PROMPT, SYNTHESIZE_PROMPT } from './prompts.js';
|
||||
import { dedup } from './dedup.js';
|
||||
import { scanForInjection } from '../injection-scanner.js';
|
||||
import {
|
||||
evaluateExternalMemoryIngress,
|
||||
projectExternalMemoryContent,
|
||||
} from '../memory-ingress-guard.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
|
||||
const log = createCoreLogger('harvest-pipeline');
|
||||
@@ -102,26 +105,34 @@ export class HarvestPipeline {
|
||||
const errors: string[] = [];
|
||||
log.info('harvest pipeline starting', { source, itemCount: items.length, batchSize: this.batchSize, concurrency: this.concurrency });
|
||||
|
||||
// Pass 0: Injection scan — drop any item whose title or content carries a
|
||||
// prompt-injection payload (role_override / prompt_extraction / instruction_injection).
|
||||
// Pass 0: Injection scan — drop any item whose untrusted title or message
|
||||
// text carries a prompt-injection payload. Structured conversation adapters
|
||||
// synthesize item.content with trusted `user:` / `assistant:` labels; scan
|
||||
// their original message text instead so those labels are not mistaken for
|
||||
// attacker-supplied authority markers. Unstructured items still scan their
|
||||
// complete content. The exact-serialization check prevents a partial
|
||||
// messages projection from hiding extra attacker-controlled content.
|
||||
// Harvest ingests UNTRUSTED external exports (ChatGPT/Claude/Gemini JSON dumps,
|
||||
// Perplexity shares, URL fetches). A hostile file must not flow through to the
|
||||
// LLM passes or into memory frames.
|
||||
const originalCount = items.length;
|
||||
items = items.filter((item) => {
|
||||
// Scan title + first 4KB of content — enough to catch payloads hidden in either field.
|
||||
// Using 'tool_output' context since imports are external data, weighted like tool output.
|
||||
const probe = `${item.title ?? ''}\n${(item.content ?? '').slice(0, 4000)}`;
|
||||
const scan = scanForInjection(probe, 'tool_output');
|
||||
if (!scan.safe) {
|
||||
const reason = scan.flags.join(',');
|
||||
const untrustedContent = projectExternalMemoryContent({
|
||||
content: item.content ?? '',
|
||||
messages: item.messages,
|
||||
parseMethod: item.metadata?.parseMethod,
|
||||
});
|
||||
const decision = evaluateExternalMemoryIngress({
|
||||
title: item.title,
|
||||
content: untrustedContent,
|
||||
});
|
||||
if (decision.action === 'block') {
|
||||
log.warn('dropping harvest item with injection payload', {
|
||||
itemId: item.id,
|
||||
title: item.title?.slice(0, 80),
|
||||
flags: scan.flags,
|
||||
score: scan.score,
|
||||
itemId: String(item.id).slice(0, 80),
|
||||
flags: decision.scan.flags,
|
||||
score: decision.scan.score,
|
||||
});
|
||||
errors.push(`Blocked item "${item.title?.slice(0, 40) ?? item.id}" — injection detected (${reason})`);
|
||||
errors.push('Blocked imported item due to unsafe content.');
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
import type { FrameStore } from '../mind/frames.js';
|
||||
import type { UniversalImportItem } from './types.js';
|
||||
import { HARVEST_FRAME_CONTENT_CAP } from './types.js';
|
||||
import { scanForInjection } from '../injection-scanner.js';
|
||||
import { evaluateExternalMemoryIngress } from '../memory-ingress-guard.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
|
||||
const log = createCoreLogger('raw-turns');
|
||||
@@ -44,6 +44,7 @@ export const MIND_RAWTURN_PREFIX = '[mind-rawturn';
|
||||
/** Hard per-conversation cap — backstop against pathological exports.
|
||||
* LoCoMo conversations run ~600 turns; 2000 leaves generous headroom. */
|
||||
export const MAX_TURNS_PER_ITEM = 2000;
|
||||
const MAX_INJECTION_DROP_LOGS = 8;
|
||||
|
||||
/** Env kill switch (checked by CALLERS, mirrored here for the recall lane). */
|
||||
export const RAWDETAIL_KILL_SWITCH = 'WAGGLE_RAWDETAIL';
|
||||
@@ -127,31 +128,38 @@ export function writeRawTurnFrames(
|
||||
const itemTs = isIsoTimestamp(item.timestamp) ? item.timestamp : undefined;
|
||||
|
||||
let turn = 0;
|
||||
let inspected = 0;
|
||||
let suppressedInjectionLogs = 0;
|
||||
for (const msg of messages) {
|
||||
if (msg.role !== 'user' && msg.role !== 'assistant') continue;
|
||||
if (inspected >= MAX_TURNS_PER_ITEM) {
|
||||
result.capped = true;
|
||||
break;
|
||||
}
|
||||
inspected++;
|
||||
const text = (msg.text ?? '').trim();
|
||||
if (text.length === 0) {
|
||||
result.skippedEmpty++;
|
||||
continue;
|
||||
}
|
||||
if (turn >= MAX_TURNS_PER_ITEM) {
|
||||
result.capped = true;
|
||||
break;
|
||||
}
|
||||
// Scan first 4KB — same probe budget as the harvest pipeline's Pass 0.
|
||||
const scan = scanForInjection(text.slice(0, 4000), 'tool_output');
|
||||
if (!scan.safe) {
|
||||
const storedText = text.slice(0, HARVEST_FRAME_CONTENT_CAP);
|
||||
const decision = evaluateExternalMemoryIngress({ content: storedText });
|
||||
if (decision.action === 'block') {
|
||||
result.injectionDropped++;
|
||||
log.warn('dropping raw turn with injection payload', {
|
||||
conv: convKey, turn, flags: scan.flags.join(','),
|
||||
});
|
||||
if (result.injectionDropped <= MAX_INJECTION_DROP_LOGS) {
|
||||
log.warn('dropping raw turn with injection payload', {
|
||||
conv: convKey, turn, flags: decision.scan.flags,
|
||||
});
|
||||
} else {
|
||||
suppressedInjectionLogs++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const speaker = sanitizeToken(msg.role, 24);
|
||||
const createdAt = isIsoTimestamp(msg.timestamp) ? msg.timestamp : itemTs;
|
||||
frames.createIFrame(
|
||||
gopId,
|
||||
`${rawTurnHeader(convKey, turn, speaker)}\n${text.slice(0, HARVEST_FRAME_CONTENT_CAP)}`,
|
||||
`${rawTurnHeader(convKey, turn, speaker)}\n${storedText}`,
|
||||
'normal',
|
||||
'import',
|
||||
createdAt,
|
||||
@@ -160,9 +168,14 @@ export function writeRawTurnFrames(
|
||||
turn++;
|
||||
}
|
||||
|
||||
if (suppressedInjectionLogs > 0) {
|
||||
log.warn('additional raw-turn injection warnings suppressed', {
|
||||
conv: convKey, suppressed: suppressedInjectionLogs,
|
||||
});
|
||||
}
|
||||
if (result.capped) {
|
||||
log.warn('raw-turn storage capped — conversation exceeds MAX_TURNS_PER_ITEM', {
|
||||
conv: convKey, stored: result.written, totalMessages: messages.length,
|
||||
conv: convKey, inspected, stored: result.written, totalMessages: messages.length,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
|
||||
@@ -7,8 +7,8 @@
|
||||
* internal services on the cloud/TEAMS deploy (sidecar binds 0.0.0.0) and can
|
||||
* exfiltrate instance-metadata IAM credentials.
|
||||
*
|
||||
* This module is a self-contained, dependency-free (node builtins only) copy of
|
||||
* the guard spec shared with `packages/agent/src/url-egress-guard.ts`. It lives
|
||||
* This module is a self-contained copy of the guard spec shared with
|
||||
* `packages/agent/src/url-egress-guard.ts`. It lives
|
||||
* here — rather than importing from @waggle/agent — because hive-mind-core is
|
||||
* OSS-mirrored and must not depend on Waggle-proprietary packages. Keep the two
|
||||
* implementations in sync; they share one spec.
|
||||
@@ -19,7 +19,8 @@
|
||||
*/
|
||||
|
||||
import { lookup as dnsLookup } from 'node:dns/promises';
|
||||
import { isIP } from 'node:net';
|
||||
import { isIP, type LookupFunction } from 'node:net';
|
||||
import { Agent } from 'undici';
|
||||
|
||||
export type AddressClass =
|
||||
| 'public'
|
||||
@@ -85,6 +86,7 @@ function classifyIpv4(ip: string): AddressClass {
|
||||
if (a >= 240) return 'reserved'; // 240.0.0.0/4 + 255.255.255.255
|
||||
if (a === 192 && b === 0 && c === 0) return 'reserved'; // 192.0.0.0/24
|
||||
if (a === 192 && b === 0 && c === 2) return 'reserved'; // TEST-NET-1
|
||||
if (a === 192 && b === 88 && c === 99) return 'reserved'; // Deprecated 6to4 relay anycast
|
||||
if (a === 198 && (b === 18 || b === 19)) return 'reserved'; // 198.18.0.0/15
|
||||
if (a === 198 && b === 51 && c === 100) return 'reserved'; // TEST-NET-2
|
||||
if (a === 203 && b === 0 && c === 113) return 'reserved'; // TEST-NET-3
|
||||
@@ -142,11 +144,20 @@ function classifyIpv6(ip: string): AddressClass {
|
||||
}
|
||||
|
||||
if ((h[0] & 0xffc0) === 0xfe80) return 'link-local'; // fe80::/10
|
||||
if ((h[0] & 0xffc0) === 0xfec0) return 'reserved'; // fec0::/10 deprecated site-local
|
||||
if ((h[0] & 0xfe00) === 0xfc00) return 'unique-local'; // fc00::/7 (ULA)
|
||||
if ((h[0] & 0xff00) === 0xff00) return 'multicast'; // ff00::/8
|
||||
if (h[0] === 0x2001 && h[1] === 0x0db8) return 'reserved'; // 2001:db8::/32 docs
|
||||
if (h[0] === 0x0064 && h[1] === 0xff9b) return 'reserved'; // 64:ff9b::/96 NAT64
|
||||
if (
|
||||
h[0] === 0x0064 && h[1] === 0xff9b
|
||||
&& ((h[2] === 0 && h[3] === 0 && h[4] === 0 && h[5] === 0) || h[2] === 1)
|
||||
) return 'reserved'; // 64:ff9b::/96 and 64:ff9b:1::/48 translation prefixes
|
||||
if (h[0] === 0x0100 && h[1] === 0 && h[2] === 0 && h[3] === 0) return 'reserved'; // 100::/64 discard
|
||||
if (h[0] === 0x0100 && h[1] === 0 && h[2] === 0 && h[3] === 1) return 'reserved'; // 100:0:0:1::/64 dummy
|
||||
if (h[0] === 0x2001 && h[1] === 2 && h[2] === 0) return 'reserved'; // 2001:2::/48 benchmark
|
||||
if (h[0] === 0x2002) return 'reserved'; // 2002::/16 deprecated 6to4
|
||||
if (h[0] === 0x3fff && (h[1] & 0xf000) === 0) return 'reserved'; // 3fff::/20 docs
|
||||
if (h[0] === 0x5f00) return 'reserved'; // 5f00::/16 SRv6 SIDs
|
||||
return 'public';
|
||||
}
|
||||
|
||||
@@ -169,6 +180,120 @@ async function defaultLookup(hostname: string): Promise<ResolvedAddress[]> {
|
||||
return results.map((r) => ({ address: r.address, family: r.family }));
|
||||
}
|
||||
|
||||
async function resolveHostname(
|
||||
hostname: string,
|
||||
rawUrl: string,
|
||||
lookupFn: LookupFn,
|
||||
): Promise<ResolvedAddress[]> {
|
||||
try {
|
||||
const addresses = await lookupFn(hostname);
|
||||
if (!addresses || addresses.length === 0) {
|
||||
throw new EgressBlockedError(
|
||||
`DNS resolution returned no addresses for "${hostname}"`,
|
||||
rawUrl,
|
||||
);
|
||||
}
|
||||
return addresses;
|
||||
} catch (err) {
|
||||
if (err instanceof EgressBlockedError) throw err;
|
||||
const detail = err instanceof Error ? err.message : String(err);
|
||||
throw new EgressBlockedError(
|
||||
`DNS resolution failed for "${hostname}": ${detail}`,
|
||||
rawUrl,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function validateResolvedAddresses(
|
||||
addresses: ResolvedAddress[],
|
||||
hostname: string,
|
||||
rawUrl: string,
|
||||
allowLocal: boolean,
|
||||
): void {
|
||||
for (const { address } of addresses) {
|
||||
const cls = classifyAddress(address);
|
||||
if (!isAllowed(cls, allowLocal)) {
|
||||
throw new EgressBlockedError(
|
||||
`Blocked egress to ${cls} address ${address} (host "${hostname}")`,
|
||||
rawUrl,
|
||||
cls,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function createGuardedLookup(
|
||||
allowLocal: boolean,
|
||||
lookupFn: LookupFn,
|
||||
): LookupFunction {
|
||||
return (hostname, options, callback) => {
|
||||
void resolveHostname(hostname, hostname, lookupFn)
|
||||
.then((addresses) => {
|
||||
validateResolvedAddresses(addresses, hostname, hostname, allowLocal);
|
||||
|
||||
const requestedFamily = options.family === 4 || options.family === 'IPv4'
|
||||
? 4
|
||||
: options.family === 6 || options.family === 'IPv6'
|
||||
? 6
|
||||
: 0;
|
||||
const candidates = requestedFamily === 0
|
||||
? addresses
|
||||
: addresses.filter(({ family }) => family === requestedFamily);
|
||||
if (candidates.length === 0) {
|
||||
throw new EgressBlockedError(
|
||||
`DNS resolution returned no IPv${requestedFamily} addresses for "${hostname}"`,
|
||||
hostname,
|
||||
);
|
||||
}
|
||||
|
||||
if (options.all) {
|
||||
callback(null, candidates);
|
||||
} else {
|
||||
const selected = candidates[0];
|
||||
callback(null, selected.address, selected.family);
|
||||
}
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
callback(err as NodeJS.ErrnoException, '');
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
function createGuardedAgent(allowLocal: boolean, lookupFn: LookupFn): Agent {
|
||||
return new Agent({
|
||||
autoSelectFamily: true,
|
||||
connect: { lookup: createGuardedLookup(allowLocal, lookupFn) },
|
||||
});
|
||||
}
|
||||
|
||||
const defaultGuardedAgents = new Map<boolean, Agent>();
|
||||
|
||||
function getDefaultGuardedAgent(allowLocal: boolean): Agent {
|
||||
const existing = defaultGuardedAgents.get(allowLocal);
|
||||
if (existing) return existing;
|
||||
const agent = createGuardedAgent(allowLocal, defaultLookup);
|
||||
defaultGuardedAgents.set(allowLocal, agent);
|
||||
return agent;
|
||||
}
|
||||
|
||||
function findEgressBlockedError(
|
||||
error: unknown,
|
||||
seen = new Set<unknown>(),
|
||||
): EgressBlockedError | null {
|
||||
if (error instanceof EgressBlockedError) return error;
|
||||
if (typeof error !== 'object' || error === null || seen.has(error)) return null;
|
||||
seen.add(error);
|
||||
|
||||
if (error instanceof AggregateError) {
|
||||
for (const nested of error.errors) {
|
||||
const blocked = findEgressBlockedError(nested, seen);
|
||||
if (blocked) return blocked;
|
||||
}
|
||||
}
|
||||
|
||||
return findEgressBlockedError((error as { cause?: unknown }).cause, seen);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that `rawUrl` is an http(s) URL whose host resolves only to public
|
||||
* addresses. Throws {@link EgressBlockedError} otherwise. Returns parsed URL.
|
||||
@@ -191,6 +316,10 @@ export async function assertUrlAllowed(
|
||||
);
|
||||
}
|
||||
|
||||
if (parsed.username || parsed.password) {
|
||||
throw new EgressBlockedError('Blocked URL credentials', rawUrl);
|
||||
}
|
||||
|
||||
// url.hostname keeps the surrounding brackets on an IPv6 literal ("[::1]"),
|
||||
// which isIP() does not recognize — strip them so the literal is classified
|
||||
// directly (loopback/private/link-local/…) instead of falling through to a DNS
|
||||
@@ -204,34 +333,11 @@ export async function assertUrlAllowed(
|
||||
addresses = [{ address: hostname, family: literalFamily }];
|
||||
} else {
|
||||
const lookupFn = options.lookup ?? defaultLookup;
|
||||
try {
|
||||
addresses = await lookupFn(hostname);
|
||||
} catch (err) {
|
||||
const detail = err instanceof Error ? err.message : String(err);
|
||||
throw new EgressBlockedError(
|
||||
`DNS resolution failed for "${hostname}": ${detail}`,
|
||||
rawUrl,
|
||||
);
|
||||
}
|
||||
if (!addresses || addresses.length === 0) {
|
||||
throw new EgressBlockedError(
|
||||
`DNS resolution returned no addresses for "${hostname}"`,
|
||||
rawUrl,
|
||||
);
|
||||
}
|
||||
addresses = await resolveHostname(hostname, rawUrl, lookupFn);
|
||||
}
|
||||
|
||||
const allowLocal = options.allowLocal ?? false;
|
||||
for (const { address } of addresses) {
|
||||
const cls = classifyAddress(address);
|
||||
if (!isAllowed(cls, allowLocal)) {
|
||||
throw new EgressBlockedError(
|
||||
`Blocked egress to ${cls} address ${address} (host "${hostname}")`,
|
||||
rawUrl,
|
||||
cls,
|
||||
);
|
||||
}
|
||||
}
|
||||
validateResolvedAddresses(addresses, hostname, rawUrl, allowLocal);
|
||||
|
||||
return parsed;
|
||||
}
|
||||
@@ -239,29 +345,119 @@ export async function assertUrlAllowed(
|
||||
export interface SafeFetchOptions extends EgressGuardOptions {
|
||||
/** Maximum redirect hops to follow (default 5). */
|
||||
maxRedirects?: number;
|
||||
/** Injectable fetch (tests). Defaults to globalThis.fetch. */
|
||||
fetchImpl?: typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
type FetchWithDispatcher = (
|
||||
input: string | URL | Request,
|
||||
init: RequestInit & { dispatcher: Agent },
|
||||
) => Promise<Response>;
|
||||
|
||||
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
||||
const CROSS_ORIGIN_SECRET_HEADERS = [
|
||||
'authorization',
|
||||
'proxy-authorization',
|
||||
'cookie',
|
||||
'cookie2',
|
||||
'x-api-key',
|
||||
'api-key',
|
||||
] as const;
|
||||
const REQUEST_BODY_HEADERS = [
|
||||
'content-encoding',
|
||||
'content-language',
|
||||
'content-length',
|
||||
'content-location',
|
||||
'content-type',
|
||||
] as const;
|
||||
|
||||
function isNonReplayableBody(body: BodyInit): boolean {
|
||||
const candidate = body as unknown as {
|
||||
getReader?: unknown;
|
||||
pipe?: unknown;
|
||||
[Symbol.asyncIterator]?: unknown;
|
||||
};
|
||||
return typeof candidate.getReader === 'function'
|
||||
|| typeof candidate.pipe === 'function'
|
||||
|| typeof candidate[Symbol.asyncIterator] === 'function';
|
||||
}
|
||||
|
||||
function redirectRequestInit(
|
||||
init: RequestInit,
|
||||
status: number,
|
||||
fromUrl: URL,
|
||||
toUrl: URL,
|
||||
): RequestInit {
|
||||
const next = { ...init };
|
||||
const method = (next.method ?? 'GET').toUpperCase();
|
||||
const rewriteToGet = ((status === 301 || status === 302) && method === 'POST')
|
||||
|| (status === 303 && method !== 'GET' && method !== 'HEAD');
|
||||
const headersToDelete = new Set<string>(['host']);
|
||||
|
||||
if (rewriteToGet) {
|
||||
next.method = 'GET';
|
||||
delete next.body;
|
||||
for (const name of REQUEST_BODY_HEADERS) headersToDelete.add(name);
|
||||
} else if (next.body !== undefined && next.body !== null && isNonReplayableBody(next.body)) {
|
||||
throw new TypeError('Cannot replay a streamed request body across a redirect');
|
||||
}
|
||||
|
||||
if (fromUrl.origin !== toUrl.origin) {
|
||||
for (const name of CROSS_ORIGIN_SECRET_HEADERS) headersToDelete.add(name);
|
||||
}
|
||||
|
||||
const headers = new Headers(next.headers);
|
||||
for (const name of headersToDelete) headers.delete(name);
|
||||
next.headers = headers;
|
||||
return next;
|
||||
}
|
||||
|
||||
/**
|
||||
* SSRF-safe fetch. Validates before the request and re-validates every redirect
|
||||
* hop (`redirect: 'manual'`). Caller-supplied `redirect` in `init` is ignored.
|
||||
* hop (`redirect: 'manual'`). Native fetch is mandatory; proxy transports need
|
||||
* an equivalent pinned connector rather than a global dispatcher override.
|
||||
* Caller-supplied `redirect` in `init` is ignored.
|
||||
*/
|
||||
export async function safeFetch(
|
||||
rawUrl: string,
|
||||
init: RequestInit = {},
|
||||
options: SafeFetchOptions = {},
|
||||
): Promise<Response> {
|
||||
if ('fetchImpl' in options) {
|
||||
throw new TypeError('safeFetch fetchImpl injection is not supported; socket pinning requires native fetch');
|
||||
}
|
||||
const maxRedirects = options.maxRedirects ?? 5;
|
||||
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
||||
|
||||
let currentUrl = rawUrl;
|
||||
let currentInit = { ...init };
|
||||
for (let hop = 0; hop <= maxRedirects; hop++) {
|
||||
await assertUrlAllowed(currentUrl, options);
|
||||
const allowLocal = options.allowLocal ?? false;
|
||||
const temporaryAgent = options.lookup !== undefined;
|
||||
const dispatcher = temporaryAgent
|
||||
? createGuardedAgent(allowLocal, options.lookup!)
|
||||
: getDefaultGuardedAgent(allowLocal);
|
||||
|
||||
const response = await fetchImpl(currentUrl, { ...init, redirect: 'manual' });
|
||||
let response: Response;
|
||||
try {
|
||||
response = await (globalThis.fetch as unknown as FetchWithDispatcher)(currentUrl, {
|
||||
...currentInit,
|
||||
redirect: 'manual',
|
||||
dispatcher,
|
||||
});
|
||||
} catch (err) {
|
||||
if (temporaryAgent) {
|
||||
await dispatcher.close().catch(() => undefined);
|
||||
}
|
||||
const blocked = findEgressBlockedError(err);
|
||||
if (blocked) {
|
||||
throw new EgressBlockedError(blocked.message, currentUrl, blocked.addressClass);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
const isRedirect = response.status >= 300 && response.status < 400;
|
||||
if (temporaryAgent) {
|
||||
void dispatcher.close().catch(() => undefined);
|
||||
}
|
||||
|
||||
const isRedirect = REDIRECT_STATUSES.has(response.status);
|
||||
const location = isRedirect ? response.headers.get('location') : null;
|
||||
if (!location) {
|
||||
return response;
|
||||
@@ -273,16 +469,22 @@ export async function safeFetch(
|
||||
/* best-effort; ignore */
|
||||
}
|
||||
|
||||
let nextUrl: string;
|
||||
let nextUrl: URL;
|
||||
try {
|
||||
nextUrl = new URL(location, currentUrl).toString();
|
||||
nextUrl = new URL(location, currentUrl);
|
||||
} catch {
|
||||
throw new EgressBlockedError(
|
||||
`Invalid redirect target "${location}"`,
|
||||
currentUrl,
|
||||
);
|
||||
}
|
||||
currentUrl = nextUrl;
|
||||
currentInit = redirectRequestInit(
|
||||
currentInit,
|
||||
response.status,
|
||||
new URL(currentUrl),
|
||||
nextUrl,
|
||||
);
|
||||
currentUrl = nextUrl.toString();
|
||||
}
|
||||
|
||||
throw new EgressBlockedError(
|
||||
|
||||
Reference in New Issue
Block a user