/** * URL Source Adapter — fetches web pages and extracts readable content. * * Uses built-in fetch + basic HTML stripping. * Splits content by sections if headings are present. */ import { randomUUID } from 'node:crypto'; import type { SourceAdapter, UniversalImportItem } from './types.js'; import { safeFetch, allowLocalFromEnv } from './url-egress-guard.js'; /** Strip HTML tags and decode common entities. Returns plain text. */ function stripHtml(html: string): string { // Remove script and style blocks entirely let text = html.replace(//gi, ''); text = text.replace(//gi, ''); text = text.replace(//gi, ''); text = text.replace(//gi, ''); text = text.replace(//gi, ''); // Convert headings to markdown-style text = text.replace(/]*>([\s\S]*?)<\/h1>/gi, '\n# $1\n'); text = text.replace(/]*>([\s\S]*?)<\/h2>/gi, '\n## $1\n'); text = text.replace(/]*>([\s\S]*?)<\/h3>/gi, '\n### $1\n'); // Convert paragraphs and breaks to newlines text = text.replace(/<\/p>/gi, '\n\n'); text = text.replace(//gi, '\n'); text = text.replace(/]*>/gi, '\n- '); // Strip remaining tags text = text.replace(/<[^>]+>/g, ''); // Decode common HTML entities text = text.replace(/&/g, '&'); text = text.replace(/</g, '<'); text = text.replace(/>/g, '>'); text = text.replace(/"/g, '"'); text = text.replace(/'/g, "'"); text = text.replace(/ /g, ' '); // Collapse excessive whitespace text = text.replace(/[ \t]+/g, ' '); text = text.replace(/\n{3,}/g, '\n\n'); return text.trim(); } /** Extract from HTML. */ function extractTitle(html: string): string | undefined { const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i); return match ? match[1].trim().replace(/\s+/g, ' ') : undefined; } /** Extract meta description from HTML. */ function extractDescription(html: string): string | undefined { const match = html.match(/<meta[^>]*name=["']description["'][^>]*content=["']([\s\S]*?)["'][^>]*>/i) ?? html.match(/<meta[^>]*content=["']([\s\S]*?)["'][^>]*name=["']description["'][^>]*>/i); return match ? match[1].trim() : undefined; } export class UrlAdapter implements SourceAdapter { readonly sourceType = 'url' as const; readonly displayName = 'Web URL'; parse(input: unknown): UniversalImportItem[] { // Synchronous parse — for pre-fetched HTML content if (typeof input !== 'string') return []; // If input looks like HTML, parse it directly if (input.includes('<html') || input.includes('<body') || input.includes('<div')) { return this.parseHtml(input, undefined); } // If input is a URL, return empty — caller should use fetchAndParse() if (input.startsWith('http://') || input.startsWith('https://')) { return []; } return []; } /** Fetch a URL and parse its content. Async because of network I/O. * Routed through the SSRF egress guard — the target and every redirect hop * must resolve to a public address (loopback allowed only when * WAGGLE_ALLOW_LOCAL_FETCH is set). Blocked targets throw EgressBlockedError, * surfaced to the MCP ingest caller as an error result. */ async fetchAndParse(url: string): Promise<UniversalImportItem[]> { const response = await safeFetch( url, { headers: { 'User-Agent': 'Waggle-Memory/1.0 (knowledge harvester)', 'Accept': 'text/html,application/xhtml+xml,text/plain', }, signal: AbortSignal.timeout(15_000), }, { allowLocal: allowLocalFromEnv() }, ); if (!response.ok) { throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`); } const contentType = response.headers.get('content-type') ?? ''; const body = await response.text(); if (contentType.includes('text/html') || contentType.includes('xhtml')) { return this.parseHtml(body, url); } // Plain text or other — treat as plaintext return [{ id: randomUUID(), source: 'url', type: 'document', title: url, content: body.slice(0, 4000), timestamp: new Date().toISOString(), metadata: { sourceUrl: url, contentType: 'article' }, }]; } private parseHtml(html: string, sourceUrl: string | undefined): UniversalImportItem[] { const title = extractTitle(html) ?? sourceUrl ?? 'Web page'; const description = extractDescription(html); const plainText = stripHtml(html); if (!plainText || plainText.length < 50) return []; // For short pages, return as single item if (plainText.length <= 4000) { return [{ id: randomUUID(), source: 'url', type: 'document', title, content: plainText, timestamp: new Date().toISOString(), metadata: { ...(sourceUrl && { sourceUrl }), ...(description && { description }), contentType: 'article', }, }]; } // For longer pages, split by headings const sections = plainText.split(/\n(?=#{1,3}\s)/); const items: UniversalImportItem[] = []; for (const section of sections) { const trimmed = section.trim(); if (trimmed.length < 30) continue; const headingMatch = trimmed.match(/^#{1,3}\s+(.+)/); const sectionTitle = headingMatch ? headingMatch[1].trim() : title; items.push({ id: randomUUID(), source: 'url', type: 'document', title: sectionTitle === title ? title : `${title} — ${sectionTitle}`, content: trimmed.slice(0, 4000), timestamp: new Date().toISOString(), metadata: { ...(sourceUrl && { sourceUrl }), contentType: 'article', }, }); } return items.length > 0 ? items : [{ id: randomUUID(), source: 'url', type: 'document', title, content: plainText.slice(0, 4000), timestamp: new Date().toISOString(), metadata: { ...(sourceUrl && { sourceUrl }), contentType: 'article', }, }]; } }