This commit is contained in:
Oleg Maslov
2026-09-02 10:14:22 +02:00
parent 0c3e2ead3b
commit b20b138fe4
771 changed files with 161561 additions and 9027 deletions

View File

@@ -18,14 +18,15 @@
* them as literals, so they fall through to `dns.lookup`, whose getaddrinfo
* backend returns the canonical dotted form we then classify.
*
* Dependency-free (node builtins only). A structurally identical guard lives at
* `packages/hive-mind-core/src/harvest/url-egress-guard.ts` for the OSS-mirrored
* harvest adapter (which must not import from @waggle/agent). Keep the two in
* sync — they share this spec.
* Socket pinning uses an Undici dispatcher whose connector consumes the same
* address records this guard validates. The OSS-mirrored harvest adapter keeps
* a separate implementation because it cannot import from @waggle/agent. Keep
* the two in sync — they share this spec.
*/
import { lookup as dnsLookup } from 'node:dns/promises';
import { isIP } from 'node:net';
import { isIP, type LookupFunction } from 'node:net';
import { Agent } from 'undici';
export type AddressClass =
| 'public'
@@ -103,6 +104,7 @@ function classifyIpv4(ip: string): AddressClass {
// Documentation / benchmark / protocol-assignment blocks — non-routable.
if (a === 192 && b === 0 && c === 0) return 'reserved'; // 192.0.0.0/24
if (a === 192 && b === 0 && c === 2) return 'reserved'; // TEST-NET-1
if (a === 192 && b === 88 && c === 99) return 'reserved'; // Deprecated 6to4 relay anycast
if (a === 198 && (b === 18 || b === 19)) return 'reserved'; // 198.18.0.0/15
if (a === 198 && b === 51 && c === 100) return 'reserved'; // TEST-NET-2
if (a === 203 && b === 0 && c === 113) return 'reserved'; // TEST-NET-3
@@ -170,11 +172,20 @@ function classifyIpv6(ip: string): AddressClass {
}
if ((h[0] & 0xffc0) === 0xfe80) return 'link-local'; // fe80::/10
if ((h[0] & 0xffc0) === 0xfec0) return 'reserved'; // fec0::/10 deprecated site-local
if ((h[0] & 0xfe00) === 0xfc00) return 'unique-local'; // fc00::/7 (ULA)
if ((h[0] & 0xff00) === 0xff00) return 'multicast'; // ff00::/8
if (h[0] === 0x2001 && h[1] === 0x0db8) return 'reserved'; // 2001:db8::/32 docs
if (h[0] === 0x0064 && h[1] === 0xff9b) return 'reserved'; // 64:ff9b::/96 NAT64
if (
h[0] === 0x0064 && h[1] === 0xff9b
&& ((h[2] === 0 && h[3] === 0 && h[4] === 0 && h[5] === 0) || h[2] === 1)
) return 'reserved'; // 64:ff9b::/96 and 64:ff9b:1::/48 translation prefixes
if (h[0] === 0x0100 && h[1] === 0 && h[2] === 0 && h[3] === 0) return 'reserved'; // 100::/64 discard
if (h[0] === 0x0100 && h[1] === 0 && h[2] === 0 && h[3] === 1) return 'reserved'; // 100:0:0:1::/64 dummy
if (h[0] === 0x2001 && h[1] === 2 && h[2] === 0) return 'reserved'; // 2001:2::/48 benchmark
if (h[0] === 0x2002) return 'reserved'; // 2002::/16 deprecated 6to4
if (h[0] === 0x3fff && (h[1] & 0xf000) === 0) return 'reserved'; // 3fff::/20 docs
if (h[0] === 0x5f00) return 'reserved'; // 5f00::/16 SRv6 SIDs
return 'public';
}
@@ -201,6 +212,125 @@ async function defaultLookup(hostname: string): Promise<ResolvedAddress[]> {
return results.map((r) => ({ address: r.address, family: r.family }));
}
async function resolveHostname(
hostname: string,
rawUrl: string,
lookupFn: LookupFn,
): Promise<ResolvedAddress[]> {
try {
const addresses = await lookupFn(hostname);
if (!addresses || addresses.length === 0) {
throw new EgressBlockedError(
`DNS resolution returned no addresses for "${hostname}"`,
rawUrl,
);
}
return addresses;
} catch (err) {
if (err instanceof EgressBlockedError) throw err;
const detail = err instanceof Error ? err.message : String(err);
throw new EgressBlockedError(
`DNS resolution failed for "${hostname}": ${detail}`,
rawUrl,
);
}
}
function validateResolvedAddresses(
addresses: ResolvedAddress[],
hostname: string,
rawUrl: string,
allowLocal: boolean,
): void {
for (const { address } of addresses) {
const cls = classifyAddress(address);
if (!isAllowed(cls, allowLocal)) {
throw new EgressBlockedError(
`Blocked egress to ${cls} address ${address} (host "${hostname}")`,
rawUrl,
cls,
);
}
}
}
/**
* Resolve, validate, and return the exact same addresses to the socket layer.
* This removes the DNS validation/connect race: net/tls never performs a third
* lookup after the records have passed the egress policy.
*/
function createGuardedLookup(
allowLocal: boolean,
lookupFn: LookupFn,
): LookupFunction {
return (hostname, options, callback) => {
void resolveHostname(hostname, hostname, lookupFn)
.then((addresses) => {
validateResolvedAddresses(addresses, hostname, hostname, allowLocal);
const requestedFamily = options.family === 4 || options.family === 'IPv4'
? 4
: options.family === 6 || options.family === 'IPv6'
? 6
: 0;
const candidates = requestedFamily === 0
? addresses
: addresses.filter(({ family }) => family === requestedFamily);
if (candidates.length === 0) {
throw new EgressBlockedError(
`DNS resolution returned no IPv${requestedFamily} addresses for "${hostname}"`,
hostname,
);
}
if (options.all) {
callback(null, candidates);
} else {
const selected = candidates[0];
callback(null, selected.address, selected.family);
}
})
.catch((err: unknown) => {
callback(err as NodeJS.ErrnoException, '');
});
};
}
function createGuardedAgent(allowLocal: boolean, lookupFn: LookupFn): Agent {
return new Agent({
autoSelectFamily: true,
connect: { lookup: createGuardedLookup(allowLocal, lookupFn) },
});
}
const defaultGuardedAgents = new Map<boolean, Agent>();
function getDefaultGuardedAgent(allowLocal: boolean): Agent {
const existing = defaultGuardedAgents.get(allowLocal);
if (existing) return existing;
const agent = createGuardedAgent(allowLocal, defaultLookup);
defaultGuardedAgents.set(allowLocal, agent);
return agent;
}
function findEgressBlockedError(
error: unknown,
seen = new Set<unknown>(),
): EgressBlockedError | null {
if (error instanceof EgressBlockedError) return error;
if (typeof error !== 'object' || error === null || seen.has(error)) return null;
seen.add(error);
if (error instanceof AggregateError) {
for (const nested of error.errors) {
const blocked = findEgressBlockedError(nested, seen);
if (blocked) return blocked;
}
}
return findEgressBlockedError((error as { cause?: unknown }).cause, seen);
}
/**
* Validate that `rawUrl` is an http(s) URL whose host resolves only to
* fetchable public addresses. Throws {@link EgressBlockedError} otherwise.
@@ -224,6 +354,10 @@ export async function assertUrlAllowed(
);
}
if (parsed.username || parsed.password) {
throw new EgressBlockedError('Blocked URL credentials', rawUrl);
}
// url.hostname keeps the surrounding brackets on an IPv6 literal ("[::1]"),
// which isIP() does not recognize — strip them so the literal is classified
// directly (loopback/private/link-local/…) instead of falling through to a DNS
@@ -238,34 +372,11 @@ export async function assertUrlAllowed(
addresses = [{ address: hostname, family: literalFamily }];
} else {
const lookupFn = options.lookup ?? defaultLookup;
try {
addresses = await lookupFn(hostname);
} catch (err) {
const detail = err instanceof Error ? err.message : String(err);
throw new EgressBlockedError(
`DNS resolution failed for "${hostname}": ${detail}`,
rawUrl,
);
}
if (!addresses || addresses.length === 0) {
throw new EgressBlockedError(
`DNS resolution returned no addresses for "${hostname}"`,
rawUrl,
);
}
addresses = await resolveHostname(hostname, rawUrl, lookupFn);
}
const allowLocal = options.allowLocal ?? false;
for (const { address } of addresses) {
const cls = classifyAddress(address);
if (!isAllowed(cls, allowLocal)) {
throw new EgressBlockedError(
`Blocked egress to ${cls} address ${address} (host "${hostname}")`,
rawUrl,
cls,
);
}
}
validateResolvedAddresses(addresses, hostname, rawUrl, allowLocal);
return parsed;
}
@@ -273,14 +384,78 @@ export async function assertUrlAllowed(
export interface SafeFetchOptions extends EgressGuardOptions {
/** Maximum redirect hops to follow (default 5). */
maxRedirects?: number;
/** Injectable fetch (tests). Defaults to globalThis.fetch. */
fetchImpl?: typeof globalThis.fetch;
}
type FetchWithDispatcher = (
input: string | URL | Request,
init: RequestInit & { dispatcher: Agent },
) => Promise<Response>;
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
const CROSS_ORIGIN_SECRET_HEADERS = [
'authorization',
'proxy-authorization',
'cookie',
'cookie2',
'x-api-key',
'api-key',
] as const;
const REQUEST_BODY_HEADERS = [
'content-encoding',
'content-language',
'content-length',
'content-location',
'content-type',
] as const;
function isNonReplayableBody(body: BodyInit): boolean {
const candidate = body as unknown as {
getReader?: unknown;
pipe?: unknown;
[Symbol.asyncIterator]?: unknown;
};
return typeof candidate.getReader === 'function'
|| typeof candidate.pipe === 'function'
|| typeof candidate[Symbol.asyncIterator] === 'function';
}
/** Apply Fetch's method/body policy and prevent credential forwarding. */
function redirectRequestInit(
init: RequestInit,
status: number,
fromUrl: URL,
toUrl: URL,
): RequestInit {
const next = { ...init };
const method = (next.method ?? 'GET').toUpperCase();
const rewriteToGet = ((status === 301 || status === 302) && method === 'POST')
|| (status === 303 && method !== 'GET' && method !== 'HEAD');
const headersToDelete = new Set<string>(['host']);
if (rewriteToGet) {
next.method = 'GET';
delete next.body;
for (const name of REQUEST_BODY_HEADERS) headersToDelete.add(name);
} else if (next.body !== undefined && next.body !== null && isNonReplayableBody(next.body)) {
throw new TypeError('Cannot replay a streamed request body across a redirect');
}
if (fromUrl.origin !== toUrl.origin) {
for (const name of CROSS_ORIGIN_SECRET_HEADERS) headersToDelete.add(name);
}
const headers = new Headers(next.headers);
for (const name of headersToDelete) headers.delete(name);
next.headers = headers;
return next;
}
/**
* SSRF-safe fetch. Validates the target before the request and re-validates
* every redirect hop (`redirect: 'manual'`) so a public URL cannot redirect
* into a private/link-local address. Caller-supplied `redirect` in `init` is
* into a private/link-local address. Native fetch is mandatory; proxy transports
* need an equivalent pinned connector rather than a global dispatcher override.
* Caller-supplied `redirect` in `init` is
* ignored — this helper owns redirect handling.
*/
export async function safeFetch(
@@ -288,16 +463,46 @@ export async function safeFetch(
init: RequestInit = {},
options: SafeFetchOptions = {},
): Promise<Response> {
if ('fetchImpl' in options) {
throw new TypeError('safeFetch fetchImpl injection is not supported; socket pinning requires native fetch');
}
const maxRedirects = options.maxRedirects ?? 5;
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
let currentUrl = rawUrl;
let currentInit = { ...init };
for (let hop = 0; hop <= maxRedirects; hop++) {
await assertUrlAllowed(currentUrl, options);
const allowLocal = options.allowLocal ?? false;
const temporaryAgent = options.lookup !== undefined;
const dispatcher = temporaryAgent
? createGuardedAgent(allowLocal, options.lookup!)
: getDefaultGuardedAgent(allowLocal);
const response = await fetchImpl(currentUrl, { ...init, redirect: 'manual' });
let response: Response;
try {
response = await (globalThis.fetch as unknown as FetchWithDispatcher)(currentUrl, {
...currentInit,
redirect: 'manual',
dispatcher,
});
} catch (err) {
if (temporaryAgent) {
await dispatcher.close().catch(() => undefined);
}
const blocked = findEgressBlockedError(err);
if (blocked) {
throw new EgressBlockedError(blocked.message, currentUrl, blocked.addressClass);
}
throw err;
}
const isRedirect = response.status >= 300 && response.status < 400;
// A custom resolver gets an isolated Agent so tests and one-off policies
// cannot contaminate pooled connections. close() is graceful: it waits for
// the returned response body without blocking or aborting the caller.
if (temporaryAgent) {
void dispatcher.close().catch(() => undefined);
}
const isRedirect = REDIRECT_STATUSES.has(response.status);
const location = isRedirect ? response.headers.get('location') : null;
if (!location) {
return response;
@@ -310,16 +515,22 @@ export async function safeFetch(
/* best-effort; ignore */
}
let nextUrl: string;
let nextUrl: URL;
try {
nextUrl = new URL(location, currentUrl).toString();
nextUrl = new URL(location, currentUrl);
} catch {
throw new EgressBlockedError(
`Invalid redirect target "${location}"`,
currentUrl,
);
}
currentUrl = nextUrl;
currentInit = redirectRequestInit(
currentInit,
response.status,
new URL(currentUrl),
nextUrl,
);
currentUrl = nextUrl.toString();
}
throw new EgressBlockedError(