feat(FN-3758): align web fetch test coverage and documentation
Aligns web fetch tool test coverage across multiple test suites and updates the agents documentation, with a patch changeset for `@runfusion/fusion`. Fusion-Task-Id: FN-3758
This commit is contained in:
Fusion
248
packages/engine/src/web-fetch.ts
Normal file
248
packages/engine/src/web-fetch.ts
Normal file
@@ -0,0 +1,248 @@
|
||||
import { lookup } from "node:dns/promises";
|
||||
import { isIP } from "node:net";
|
||||
|
||||
export const dnsResolver = {
|
||||
lookup,
|
||||
};
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_MAX_BYTES = 500 * 1024;
|
||||
const DEFAULT_USER_AGENT = "FusionWebFetch/1.0";
|
||||
|
||||
export interface WebFetchOptions {
|
||||
timeoutMs?: number;
|
||||
maxBytes?: number;
|
||||
userAgent?: string;
|
||||
allowPrivateHosts?: boolean;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface WebFetchResult {
|
||||
url: string;
|
||||
finalUrl: string;
|
||||
status: number;
|
||||
contentType: string;
|
||||
mimeType: string;
|
||||
title?: string;
|
||||
description?: string;
|
||||
content: string;
|
||||
truncated: boolean;
|
||||
bytesRead: number;
|
||||
}
|
||||
|
||||
export type WebFetchErrorCode =
|
||||
| "invalid-url"
|
||||
| "blocked-host"
|
||||
| "blocked-scheme"
|
||||
| "timeout"
|
||||
| "too-large"
|
||||
| "unsupported-mime"
|
||||
| "http-error"
|
||||
| "network-error";
|
||||
|
||||
export class WebFetchError extends Error {
|
||||
readonly code: WebFetchErrorCode;
|
||||
|
||||
constructor(code: WebFetchErrorCode, message: string, options?: { cause?: unknown }) {
|
||||
super(message, options);
|
||||
this.name = "WebFetchError";
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
export async function assertSafeUrl(url: string, allowPrivateHosts = false): Promise<void> {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch (error) {
|
||||
throw new WebFetchError("invalid-url", "Invalid URL", { cause: error });
|
||||
}
|
||||
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||
throw new WebFetchError("blocked-scheme", `Blocked URL scheme: ${parsed.protocol}`);
|
||||
}
|
||||
|
||||
if (allowPrivateHosts) {
|
||||
return;
|
||||
}
|
||||
|
||||
const host = normalizeHost(parsed.hostname);
|
||||
if (isIpBlocked(host)) {
|
||||
throw new WebFetchError("blocked-host", `Blocked private host: ${host}`);
|
||||
}
|
||||
|
||||
if (isIP(host) === 0) {
|
||||
try {
|
||||
const resolved = await dnsResolver.lookup(host, { all: true });
|
||||
if (resolved.some((entry) => isIpBlocked(entry.address))) {
|
||||
throw new WebFetchError("blocked-host", `Blocked private host: ${host}`);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof WebFetchError) {
|
||||
throw error;
|
||||
}
|
||||
throw new WebFetchError("network-error", `DNS lookup failed for ${host}`, { cause: error });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchWebContent(url: string, options: WebFetchOptions = {}): Promise<WebFetchResult> {
|
||||
const timeoutMs = Number(options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||
const maxBytes = Number(options.maxBytes ?? DEFAULT_MAX_BYTES);
|
||||
|
||||
await assertSafeUrl(url, options.allowPrivateHosts ?? false);
|
||||
|
||||
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
||||
const requestSignal = options.signal ? AbortSignal.any([options.signal, timeoutSignal]) : timeoutSignal;
|
||||
|
||||
try {
|
||||
if (requestSignal.aborted) {
|
||||
throw new DOMException("Aborted", "AbortError");
|
||||
}
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
redirect: "follow",
|
||||
headers: {
|
||||
"User-Agent": options.userAgent ?? DEFAULT_USER_AGENT,
|
||||
},
|
||||
signal: requestSignal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new WebFetchError("http-error", `fetch failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const contentType = response.headers.get("content-type") ?? "application/octet-stream";
|
||||
const mimeType = contentType.split(";")[0].trim().toLowerCase();
|
||||
const raw = await response.text();
|
||||
|
||||
let title: string | undefined;
|
||||
let description: string | undefined;
|
||||
let content: string;
|
||||
|
||||
if (mimeType.includes("text/html")) {
|
||||
const extracted = extractHtml(raw);
|
||||
title = extracted.title;
|
||||
description = extracted.description;
|
||||
content = extracted.content;
|
||||
} else if (mimeType.includes("application/json") || looksLikeJson(raw)) {
|
||||
content = JSON.stringify(JSON.parse(raw), null, 2);
|
||||
} else if (mimeType.includes("text/") || mimeType.includes("markdown")) {
|
||||
content = raw;
|
||||
} else {
|
||||
throw new WebFetchError("unsupported-mime", `unsupported mime type: ${mimeType}`);
|
||||
}
|
||||
|
||||
const bytesRead = content.length;
|
||||
if (bytesRead > maxBytes) {
|
||||
return {
|
||||
url,
|
||||
finalUrl: response.url || url,
|
||||
status: response.status,
|
||||
contentType,
|
||||
mimeType,
|
||||
title,
|
||||
description,
|
||||
content: content.slice(0, maxBytes),
|
||||
truncated: true,
|
||||
bytesRead,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
url,
|
||||
finalUrl: response.url || url,
|
||||
status: response.status,
|
||||
contentType,
|
||||
mimeType,
|
||||
title,
|
||||
description,
|
||||
content,
|
||||
truncated: false,
|
||||
bytesRead,
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof WebFetchError) {
|
||||
throw error;
|
||||
}
|
||||
if (error instanceof DOMException && error.name === "AbortError") {
|
||||
const timedOut = timeoutSignal.aborted && !(options.signal?.aborted ?? false);
|
||||
throw new WebFetchError(timedOut ? "timeout" : "network-error", timedOut ? "Fetch timed out" : "Fetch aborted", { cause: error });
|
||||
}
|
||||
if (error instanceof Error && error.name === "TimeoutError") {
|
||||
throw new WebFetchError("timeout", error.message, { cause: error });
|
||||
}
|
||||
throw new WebFetchError("network-error", error instanceof Error ? error.message : "fetch failed", { cause: error });
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeHost(host: string): string {
|
||||
if (host.startsWith("[") && host.endsWith("]")) {
|
||||
return host.slice(1, -1);
|
||||
}
|
||||
return host;
|
||||
}
|
||||
|
||||
function extractHtml(html: string): { title?: string; description?: string; content: string } {
|
||||
const title = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.trim();
|
||||
const description = html.match(/<meta[^>]*name=["']description["'][^>]*content=["']([^"']+)["'][^>]*>/i)?.[1]?.trim();
|
||||
const stripped = html
|
||||
.replace(/<script[\s\S]*?<\/script>/gi, " ")
|
||||
.replace(/<style[\s\S]*?<\/style>/gi, " ")
|
||||
.replace(/<(nav|footer|header)[\s\S]*?<\/\1>/gi, " ");
|
||||
const main = stripped.match(/<(main|article)[^>]*>([\s\S]*?)<\/\1>/i)?.[2] ?? stripped.match(/<body[^>]*>([\s\S]*?)<\/body>/i)?.[1] ?? stripped;
|
||||
const content = main.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
|
||||
return { title, description, content };
|
||||
}
|
||||
|
||||
function looksLikeJson(value: string): boolean {
|
||||
const trimmed = value.trim();
|
||||
return trimmed.startsWith("{") || trimmed.startsWith("[");
|
||||
}
|
||||
|
||||
function isIpBlocked(address: string): boolean {
|
||||
const version = isIP(address);
|
||||
if (version === 4) {
|
||||
const normalized = normalizeMappedIpv4(address);
|
||||
if (normalized) {
|
||||
return isIpv4Blocked(normalized);
|
||||
}
|
||||
return isIpv4Blocked(address);
|
||||
}
|
||||
|
||||
if (version === 6) {
|
||||
const normalized = address.toLowerCase();
|
||||
if (normalized === "::1") return true;
|
||||
if (normalized.startsWith("fc") || normalized.startsWith("fd")) return true;
|
||||
if (normalized.startsWith("fe8") || normalized.startsWith("fe9") || normalized.startsWith("fea") || normalized.startsWith("feb")) return true;
|
||||
const mapped = normalizeMappedIpv4(normalized);
|
||||
if (mapped) {
|
||||
return isIpv4Blocked(mapped);
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function normalizeMappedIpv4(address: string): string | null {
|
||||
const lower = address.toLowerCase();
|
||||
if (!lower.startsWith("::ffff:")) {
|
||||
return null;
|
||||
}
|
||||
const candidate = lower.slice(7);
|
||||
return isIP(candidate) === 4 ? candidate : null;
|
||||
}
|
||||
|
||||
function isIpv4Blocked(address: string): boolean {
|
||||
const parts = address.split(".").map((part) => Number(part));
|
||||
const [a, b] = parts;
|
||||
if (a === 0) return true;
|
||||
if (a === 10) return true;
|
||||
if (a === 127) return true;
|
||||
if (a === 169 && b === 254) return true;
|
||||
if (a === 172 && b >= 16 && b <= 31) return true;
|
||||
if (a === 192 && b === 168) return true;
|
||||
if (a === 100 && b >= 64 && b <= 127) return true;
|
||||
return false;
|
||||
}
|
||||
Reference in New Issue
Block a user