feat(FN-3758): align web fetch test coverage and documentation

Aligns web fetch tool test coverage across multiple test suites and updates the agents documentation, with a patch changeset for `@runfusion/fusion`.

Fusion-Task-Id: FN-3758

.changeset/fn-3758-web-fetch-tool.md

@@ -0,0 +1,5 @@
+---
+"@runfusion/fusion": minor
+---
+
+Add native `fn_web_fetch` tool for lightweight URL fetching from agent/chat sessions, with SSRF guard, timeout, and size caps. Use the `agent-browser` skill for JS-rendered pages.

AGENTS.md

@@ -254,6 +254,15 @@ The pi extension provides tools and a `/fn` command for interacting with fn from
 
 The extension has no skills — tool descriptions give the LLM everything it needs.
 
+### WebFetch tool (`fn_web_fetch`)
+
+Use `fn_web_fetch` for lightweight URL reads from agent/chat sessions. It performs an HTTP GET, follows redirects, extracts readable text (including HTML→text and JSON pretty-print), and returns bounded content.
+
+- Default limits: `timeoutMs=30000` and `maxBytes=512000` (500 KB)
+- Security: blocks private/loopback/link-local hosts (including DNS-resolved private addresses) unless explicitly overridden in internal/test contexts
+- Scope: read-only fetch (no JS rendering, no auth flows, no POST/cookie workflows)
+- Use `agent-browser` skill when pages require JavaScript execution, interactive navigation, or richer browser behavior
+
 ## Agent Spawning (`spawn_agent` tool)
 
 The executor agent can spawn child agents that run in parallel. Each spawned agent:

packages/cli/skill/fusion/SKILL.md

@@ -33,7 +33,7 @@ Mission → Milestone → Slice → Feature → Task
 - **Agent tools** — `fn_agent_stop`, `fn_agent_start`, `fn_list_agents`, `fn_delegate_task`, `fn_agent_show`, `fn_agent_org_chart`
 - **Skills tools** — `fn_skills_search`, `fn_skills_install`
 - **Insight tools** — `fn_insight_list`, `fn_insight_show`, `fn_insight_run_list`, `fn_insight_run_show`
-- **Other tools** — `fn_research_run`, `fn_research_list`, `fn_research_get`, `fn_research_cancel`, `fn_research_retry`
+- **Other tools** — `fn_web_fetch`, `fn_research_run`, `fn_research_list`, `fn_research_get`, `fn_research_cancel`, `fn_research_retry`
 <!-- END: tool-categories -->
 - **Dashboard** — Use `/fn` command to start/stop the dashboard
 

packages/cli/skill/fusion/references/engine-tools.md

@@ -18,6 +18,7 @@ These tools are **not** part of the user-invokable extension surface. They are i
 | `fn_memory_search` | triage, executor, heartbeat | Search project memory plus per-agent layered memory snippets | `query` (string), `limit?` (number) |
 | `fn_memory_get` | triage, executor, heartbeat | Read a bounded memory file window (including bounded per-agent layered paths) | `path` (string), `startLine?` (number), `lineCount?` (number) |
 | `fn_memory_append` | executor, heartbeat (when writable backend enabled) | Append memory notes with explicit scope: `scope="agent"` for private operating context, `scope="project"` for workspace-wide durable knowledge | `scope?` (`project` \| `agent`), `layer` (`long-term` \| `daily`), `content` (string) |
+| `fn_web_fetch` | executor, step-session, heartbeat | Lightweight HTTP fetch with HTML→text extraction, timeout/size caps, and SSRF guard (no JS rendering) | `url` (string), `prompt?` (string), `timeoutMs?` (number), `maxBytes?` (number) |
 | `fn_research_run` | triage, executor | Start a bounded research run (optionally wait for completion) and return structured findings metadata | `query` (string), `wait_for_completion?` (boolean), `max_wait_ms?` (number) |
 | `fn_research_list` | triage, executor | List recent research runs with status/summary metadata | `status?` (`pending` \| `running` \| `completed` \| `failed` \| `cancelled`), `limit?` (number) |
 | `fn_research_get` | triage, executor | Read one research run's structured findings/citations payload | `id` (string) |

packages/cli/skill/fusion/references/extension-tools.md

@@ -364,6 +364,17 @@ Show a single insight-generation run by ID.
 
 ## Other Tools
 
+### fn_web_fetch
+
+Lightweight URL fetch (no JS rendering). Use agent-browser skill for JS-heavy pages. URL to fetch (http/https) Optional extraction hint for downstream summarization Timeout in milliseconds (default: 30000) Max bytes to return (default: 512000)
+
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `url` | string | ✓ | URL to fetch (http/https) |
+| `prompt` | string | — | Optional extraction hint for downstream summarization |
+| `timeoutMs` | number | — | Timeout in milliseconds (default: 30000) |
+| `maxBytes` | number | — | Max bytes to return (default: 512000) |
+
 ### fn_research_run
 
 Start a bounded research run and optionally wait for findings.

packages/cli/skill/fusion/references/fusion-capabilities.md

@@ -29,6 +29,7 @@ All skill/extension tool invocations in this catalog use the public `fn_*` names
 | `fn_task_import_github_issue` | Import a specific GitHub issue as a Fusion task. Fetches the issue by number and creates a single task in the planning column with the issue title and body. |
 | `fn_task_browse_github_issues` | List open GitHub issues from a repository to browse before importing. Returns issue numbers, titles, and URLs for selection. Use with fn_task_import_github_issue to import specific issues by number. |
 | `fn_task_plan` | Create a task via AI-guided planning mode — interactive conversation to refine your idea into a well-specified task. |
+| `fn_web_fetch` | Lightweight URL fetch (no JS rendering). Use agent-browser skill for JS-heavy pages. URL to fetch (http/https) Optional extraction hint for downstream summarization Timeout in milliseconds (default: 30000) Max bytes to return (default: 512000) |
 | `fn_research_run` | Start a bounded research run and optionally wait for findings. |
 | `fn_research_list` | List recent research runs. |
 | `fn_research_get` | Get one research run and structured findings. |

packages/cli/src/__tests__/extension-web-fetch.test.ts

@@ -0,0 +1,43 @@
+import { describe, expect, it, vi } from "vitest";
+
+const fetchWebContentMock = vi.hoisted(() => vi.fn());
+
+vi.mock("@fusion/engine", () => ({
+  fetchWebContent: fetchWebContentMock,
+}));
+
+import kbExtension from "../extension.js";
+
+describe("extension fn_web_fetch", () => {
+  it("registers and executes fn_web_fetch", async () => {
+    const tools = new Map<string, any>();
+    const api = {
+      registerTool(def: any) {
+        tools.set(def.name, def);
+      },
+      registerCommand: vi.fn(),
+      registerShortcut: vi.fn(),
+      registerFlag: vi.fn(),
+      on: vi.fn(),
+    } as any;
+
+    fetchWebContentMock.mockResolvedValue({
+      finalUrl: "https://example.com/final",
+      status: 200,
+      contentType: "text/plain",
+      title: "Example",
+      content: "hello world",
+      truncated: false,
+      bytesRead: 11,
+    });
+
+    kbExtension(api);
+    const tool = tools.get("fn_web_fetch");
+    expect(tool).toBeTruthy();
+
+    const result = await tool.execute("id", { url: "https://example.com" }, undefined, undefined, { cwd: process.cwd() });
+    expect(fetchWebContentMock).toHaveBeenCalledWith("https://example.com", { timeoutMs: undefined, maxBytes: undefined });
+    expect(result.content[0].text).toContain("https://example.com/final");
+    expect(result.details.status).toBe(200);
+  });
+});

packages/cli/src/extension.ts

@@ -25,6 +25,7 @@ import {
   isGhAvailable,
   runGhJsonAsync,
 } from "@fusion/core/gh-cli";
+import { fetchWebContent } from "@fusion/engine";
 import { resolve, basename, extname, join } from "node:path";
 import { readFile } from "node:fs/promises";
 import { existsSync } from "node:fs";
@@ -1363,6 +1364,54 @@ export default function kbExtension(pi: ExtensionAPI) {
     },
   });
 
+  pi.registerTool({
+    name: "fn_web_fetch",
+    label: "fn: Web Fetch",
+    description: "Lightweight URL fetch (no JS rendering). Use agent-browser skill for JS-heavy pages.",
+    parameters: Type.Object({
+      url: Type.String({ description: "URL to fetch (http/https)" }),
+      prompt: Type.Optional(Type.String({ description: "Optional extraction hint for downstream summarization" })),
+      timeoutMs: Type.Optional(Type.Number({ description: "Timeout in milliseconds (default: 30000)" })),
+      maxBytes: Type.Optional(Type.Number({ description: "Max bytes to return (default: 512000)" })),
+    }),
+    promptSnippet: "Fetch and extract readable text from a webpage URL",
+    promptGuidelines: [
+      "Use for lightweight GET requests where JS rendering is not required.",
+      "For JS-rendered pages or complex browsing flows, use agent-browser skill instead.",
+    ],
+    async execute(_toolCallId, params) {
+      const result = await fetchWebContent(params.url, {
+        timeoutMs: params.timeoutMs,
+        maxBytes: params.maxBytes,
+      });
+      return {
+        content: [{
+          type: "text",
+          text: [
+            `URL: ${result.finalUrl}`,
+            `Status: ${result.status}`,
+            `Content-Type: ${result.contentType}`,
+            params.prompt ? `Prompt: ${params.prompt}` : undefined,
+            result.title ? `Title: ${result.title}` : undefined,
+            "",
+            result.content,
+            result.truncated ? "\n[truncated to maxBytes]" : "",
+          ].filter(Boolean).join("\n"),
+        }],
+        details: {
+          finalUrl: result.finalUrl,
+          status: result.status,
+          contentType: result.contentType,
+          title: result.title,
+          truncated: result.truncated,
+          bytesRead: result.bytesRead,
+          promptSnippet: params.prompt ? params.prompt.slice(0, 200) : undefined,
+          promptGuidelines: "Lightweight fetch only; for JS-rendered pages, use agent-browser skill.",
+        },
+      };
+    },
+  });
+
   // ── Research Tools ──────────────────────────────────────────────
 
   pi.registerTool({

packages/engine/src/__tests__/agent-tools-web-fetch.test.ts

@@ -0,0 +1,44 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { createWebFetchTool } from "../agent-tools.js";
+
+describe("createWebFetchTool", () => {
+  const originalFetch = global.fetch;
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+    global.fetch = originalFetch;
+  });
+
+  it("returns fetched content with metadata", async () => {
+    global.fetch = vi.fn().mockResolvedValue({
+      ok: true,
+      status: 200,
+      url: "https://example.com/final",
+      headers: new Headers({ "content-type": "text/plain" }),
+      text: async () => "hello world",
+    } as Response);
+
+    const tool = createWebFetchTool();
+    const result = await tool.execute("id", { url: "https://example.com" } as any, undefined, undefined, {} as any);
+    const text = result.content[0] && "text" in result.content[0] ? result.content[0].text : "";
+
+    expect((result as any).isError).toBeUndefined();
+    expect(text).toContain("URL: https://example.com/final");
+    expect(text).toContain("Status: 200");
+    expect(text).toContain("hello world");
+  });
+
+  it("returns blocked-host error", async () => {
+    const tool = createWebFetchTool();
+    const result = await tool.execute("id", { url: "http://127.0.0.1" } as any, undefined, undefined, {} as any);
+    const text = result.content[0] && "text" in result.content[0] ? result.content[0].text : "";
+
+    expect((result as any).isError).toBe(true);
+    expect(text).toContain("blocked-host");
+  });
+
+  it("requires url parameter", async () => {
+    const tool = createWebFetchTool();
+    await expect(tool.execute("id", {} as any, undefined, undefined, {} as any)).resolves.toMatchObject({ isError: true });
+  });
+});

packages/engine/src/__tests__/heartbeat-executor.test.ts

@@ -1920,8 +1920,8 @@ describe("executeHeartbeat", () => {
       expect(callArgs.systemPrompt).toContain("fn_task_document_write");
       expect(callArgs.tools).toBe("coding");
       // Tools: fn_task_create, fn_task_log, fn_task_document_write, fn_task_document_read, fn_list_agents, fn_delegate_task,
-      // fn_get_agent_config, fn_update_agent_config, fn_read_evaluations, fn_update_identity, fn_memory_search, fn_memory_get, fn_memory_append, fn_heartbeat_done
-      expect(callArgs.customTools).toHaveLength(14);
+      // fn_get_agent_config, fn_update_agent_config, fn_read_evaluations, fn_update_identity, fn_web_fetch, fn_memory_search, fn_memory_get, fn_memory_append, fn_heartbeat_done
+      expect(callArgs.customTools).toHaveLength(15);
       expect(callArgs.customTools![0]!.name).toBe("fn_task_create");
       expect(callArgs.customTools![1]!.name).toBe("fn_task_log");
       expect(callArgs.customTools![2]!.name).toBe("fn_task_document_write");
@@ -1932,11 +1932,12 @@ describe("executeHeartbeat", () => {
       expect(callArgs.customTools![7]!.name).toBe("fn_update_agent_config");
       expect(callArgs.customTools![8]!.name).toBe("fn_read_evaluations");
       expect(callArgs.customTools![9]!.name).toBe("fn_update_identity");
-      expect(callArgs.customTools![10]!.name).toBe("fn_memory_search");
-      expect(callArgs.customTools![11]!.name).toBe("fn_memory_get");
-      expect(callArgs.customTools![12]!.name).toBe("fn_memory_append");
+      expect(callArgs.customTools![10]!.name).toBe("fn_web_fetch");
+      expect(callArgs.customTools![11]!.name).toBe("fn_memory_search");
+      expect(callArgs.customTools![12]!.name).toBe("fn_memory_get");
+      expect(callArgs.customTools![13]!.name).toBe("fn_memory_append");
       // fn_heartbeat_done is last (terminal tool)
-      expect(callArgs.customTools![13]!.name).toBe("fn_heartbeat_done");
+      expect(callArgs.customTools![14]!.name).toBe("fn_heartbeat_done");
     });
 
     it("loads workspace memory into system prompt and identity snapshot when inline memory is empty", async () => {

packages/engine/src/__tests__/web-fetch.test.ts

@@ -0,0 +1,96 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { dnsResolver, fetchWebContent, WebFetchError } from "../web-fetch.js";
+
+describe("web-fetch", () => {
+  const originalFetch = global.fetch;
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+    global.fetch = originalFetch;
+  });
+
+  it("extracts html content", async () => {
+    global.fetch = vi.fn().mockResolvedValue({
+      ok: true,
+      status: 200,
+      url: "https://example.com/final",
+      headers: new Headers({ "content-type": "text/html" }),
+      text: async () => "<html><head><title>Hello</title><meta name='description' content='Desc'></head><body><main><h1>Title</h1><p>Body</p></main></body></html>",
+    } as Response);
+
+    const result = await fetchWebContent("https://example.com");
+    expect(result.content).toContain("Title Body");
+    expect(result.title).toBe("Hello");
+    expect(result.description).toBe("Desc");
+    expect(result.finalUrl).toBe("https://example.com/final");
+  });
+
+  it("pretty prints json", async () => {
+    global.fetch = vi.fn().mockResolvedValue({
+      ok: true,
+      status: 200,
+      url: "https://example.com",
+      headers: new Headers({ "content-type": "application/json" }),
+      text: async () => '{"a":1}',
+    } as Response);
+
+    const result = await fetchWebContent("https://example.com");
+    expect(result.content).toContain('"a": 1');
+  });
+
+  it("passes through plain text", async () => {
+    global.fetch = vi.fn().mockResolvedValue({
+      ok: true,
+      status: 200,
+      url: "https://example.com",
+      headers: new Headers({ "content-type": "text/plain" }),
+      text: async () => "hello world",
+    } as Response);
+
+    const result = await fetchWebContent("https://example.com");
+    expect(result.content).toBe("hello world");
+  });
+
+  it("maps timeout", async () => {
+    global.fetch = vi.fn().mockImplementation(async (_url, init: RequestInit) => {
+      await new Promise((_, reject) => init.signal?.addEventListener("abort", () => reject(new DOMException("Aborted", "AbortError"))));
+      return { ok: true, status: 200, url: "https://example.com", headers: new Headers(), text: async () => "" } as Response;
+    });
+    await expect(fetchWebContent("https://example.com", { timeoutMs: 1 })).rejects.toMatchObject({ code: "timeout" });
+  });
+
+  it("truncates content to max bytes", async () => {
+    global.fetch = vi.fn().mockResolvedValue({
+      ok: true,
+      status: 200,
+      url: "https://example.com",
+      headers: new Headers({ "content-type": "text/plain" }),
+      text: async () => "x".repeat(100),
+    } as Response);
+
+    const result = await fetchWebContent("https://example.com", { maxBytes: 10 });
+    expect(result.content).toHaveLength(10);
+    expect(result.truncated).toBe(true);
+    expect(result.bytesRead).toBe(100);
+  });
+
+  it("maps non-ok response to http-error", async () => {
+    global.fetch = vi.fn().mockResolvedValue({ ok: false, status: 404 } as Response);
+    await expect(fetchWebContent("https://example.com")).rejects.toMatchObject({ code: "http-error" });
+  });
+
+  it.each(["file:///tmp/test", "ftp://example.com", "data:text/plain,hello"])("blocks unsupported scheme %s", async (url) => {
+    await expect(fetchWebContent(url)).rejects.toMatchObject({ code: "blocked-scheme" });
+  });
+
+  it.each(["http://127.0.0.1", "http://10.0.0.5", "http://[::1]", "http://169.254.0.1"])("blocks private literal host %s", async (url) => {
+    await expect(fetchWebContent(url)).rejects.toMatchObject({ code: "blocked-host" });
+  });
+
+  it("blocks dns-resolved private host", async () => {
+    vi.spyOn(dnsResolver, "lookup").mockResolvedValue([{ address: "10.0.0.1", family: 4 }] as unknown as Awaited<ReturnType<typeof dnsResolver.lookup>>);
+    const pending = fetchWebContent("https://internal.example.com");
+    await expect(pending).rejects.toBeInstanceOf(WebFetchError);
+    await expect(pending).rejects.toMatchObject({ code: "blocked-host" });
+  });
+});

packages/engine/src/agent-heartbeat.ts

@@ -22,7 +22,7 @@ import { ApprovalRequestStore, buildExecutionMemoryInstructions, isEphemeralAgen
 import type { ToolDefinition } from "@mariozechner/pi-coding-agent";
 import { Type, type Static } from "@mariozechner/pi-ai";
 import { createHash } from "node:crypto";
-import { createTaskCreateTool, createTaskLogToolWithContext, createTaskDocumentWriteTool, createTaskDocumentReadTool, createListAgentsTool, createDelegateTaskTool, createGetAgentConfigTool, createUpdateAgentConfigTool, createSendMessageTool, createReadMessagesTool, createMemoryTools, createReadEvaluationsTool, createUpdateIdentityTool, createReflectOnPerformanceTool, readAgentMemoryWorkspaceLongTerm, taskCreateParams } from "./agent-tools.js";
+import { createTaskCreateTool, createTaskLogToolWithContext, createTaskDocumentWriteTool, createTaskDocumentReadTool, createListAgentsTool, createDelegateTaskTool, createGetAgentConfigTool, createUpdateAgentConfigTool, createSendMessageTool, createReadMessagesTool, createMemoryTools, createReadEvaluationsTool, createUpdateIdentityTool, createReflectOnPerformanceTool, createWebFetchTool, readAgentMemoryWorkspaceLongTerm, taskCreateParams } from "./agent-tools.js";
 import { AgentLogger } from "./agent-logger.js";
 import {
   resolveAgentInstructionsWithRatings,
@@ -1753,6 +1753,8 @@ export class HeartbeatMonitor {
           heartbeatTools = this.createHeartbeatTools(agentId, taskStore, taskId!, runContext, audit, this.messageStore);
         }
 
+        heartbeatTools.push(createWebFetchTool());
+
         let memorySettings: Settings | undefined;
         try {
           memorySettings = await getHeartbeatMemorySettings(taskStore);

packages/engine/src/agent-tools.ts

@@ -20,6 +20,7 @@ import type { ToolDefinition } from "@mariozechner/pi-coding-agent";
 import { Type, type Static } from "@mariozechner/pi-ai";
 import type { AgentReflectionService } from "./agent-reflection.js";
 import { createLogger } from "./logger.js";
+import { fetchWebContent, WebFetchError } from "./web-fetch.js";
 
 // ── Tool parameter schemas (canonical definitions) ────────────────────────
 
@@ -130,6 +131,13 @@ export const memoryGetParams = Type.Object({
   lineCount: Type.Optional(Type.Number({ description: "Number of lines to read (default: 120, max: 400)" })),
 });
 
+export const webFetchParams = Type.Object({
+  url: Type.String({ description: "URL to fetch (http/https only)" }),
+  prompt: Type.Optional(Type.String({ description: "Optional extraction hint for downstream summarization" })),
+  timeoutMs: Type.Optional(Type.Number({ description: "Request timeout in milliseconds" })),
+  maxBytes: Type.Optional(Type.Number({ description: "Maximum content bytes to return" })),
+});
+
 export const researchRunParams = Type.Object({
   query: Type.String({ description: "Research question or topic to investigate" }),
   wait_for_completion: Type.Optional(Type.Boolean({ description: "Wait for completion in this call (default: false)" })),
@@ -921,6 +929,60 @@ export function createMemoryAppendTool(rootDir: string, settings?: MemoryToolSet
   };
 }
 
+export function createWebFetchTool(options?: { allowPrivateHosts?: boolean }): ToolDefinition {
+  return {
+    name: "fn_web_fetch",
+    label: "WebFetch",
+    description: "Fetch and extract readable text from a URL (lightweight HTTP fetch, no JS rendering).",
+    parameters: webFetchParams,
+    execute: async (_id: string, params: Static<typeof webFetchParams>) => {
+      try {
+        const result = await fetchWebContent(params.url, {
+          timeoutMs: params.timeoutMs,
+          maxBytes: params.maxBytes,
+          allowPrivateHosts: options?.allowPrivateHosts ?? false,
+        });
+        const sections = [
+          `URL: ${result.finalUrl}`,
+          `Status: ${result.status}`,
+          `Content-Type: ${result.contentType}`,
+          params.prompt ? `Prompt: ${params.prompt}` : undefined,
+          result.title ? `Title: ${result.title}` : undefined,
+          "",
+          result.content,
+          result.truncated ? "\n[truncated to maxBytes]" : "",
+        ].filter(Boolean);
+        return {
+          content: [{ type: "text" as const, text: sections.join("\n") }],
+          details: {
+            finalUrl: result.finalUrl,
+            status: result.status,
+            contentType: result.contentType,
+            title: result.title,
+            truncated: result.truncated,
+            bytesRead: result.bytesRead,
+            prompt: params.prompt,
+          },
+        };
+      } catch (error) {
+        if (error instanceof WebFetchError) {
+          return {
+            content: [{ type: "text" as const, text: `ERROR [${error.code}]: ${error.message}` }],
+            details: { code: error.code, message: error.message },
+            isError: true,
+          };
+        }
+        const message = error instanceof Error ? error.message : String(error);
+        return {
+          content: [{ type: "text" as const, text: `ERROR [network-error]: ${message}` }],
+          details: { code: "network-error", message },
+          isError: true,
+        };
+      }
+    },
+  };
+}
+
 export function createMemoryTools(rootDir: string, settings?: MemoryToolSettings, options?: MemoryToolOptions): ToolDefinition[] {
   if (settings?.memoryEnabled === false) {
     return [];

packages/engine/src/executor.ts

@@ -61,6 +61,7 @@ import {
   createGetAgentConfigTool,
   createListAgentsTool,
   createMemoryTools,
+  createWebFetchTool,
   createReadMessagesTool,
   createReflectOnPerformanceTool,
   createUpdateAgentConfigTool,
@@ -2936,6 +2937,7 @@ export class TaskExecutor {
             getSettings: async () => this.store.getSettings(),
           })
           : []),
+        createWebFetchTool(),
         ...createMemoryTools(this.rootDir, settings, assignedAgent ? {
           agentMemory: {
             agentId: assignedAgent.id,

packages/engine/src/index.ts

@@ -72,6 +72,7 @@ export { HEARTBEAT_PROCEDURE, HEARTBEAT_SYSTEM_PROMPT, HEARTBEAT_NO_TASK_SYSTEM_
 export { WorktreePool, scanIdleWorktrees, cleanupOrphanedWorktrees, reapOrphanWorktrees } from "./worktree-pool.js";
 export { generateReservedWorktreeName, generateWorktreeName, planTaskWorktreePath, slugify } from "./worktree-names.js";
 export { createLogger, type Logger } from "./logger.js";
+export { fetchWebContent, assertSafeUrl, WebFetchError, type WebFetchOptions, type WebFetchResult, type WebFetchErrorCode } from "./web-fetch.js";
 export { isUsageLimitError, UsageLimitPauser } from "./usage-limit-detector.js";
 export { withRateLimitRetry } from "./rate-limit-retry.js";
 export { ResearchOrchestrator, type ResearchOrchestratorOptions, type ResearchOrchestratorStatus, type ResearchOrchestratorStartOptions } from "./research-orchestrator.js";

packages/engine/src/research/providers/page-fetch-provider.ts

@@ -1,6 +1,7 @@
 import type { ResearchProviderConfig, ResearchSource } from "@fusion/core";
 import type { ResearchProvider } from "../../research-step-runner.js";
 import { createLogger } from "../../logger.js";
+import { fetchWebContent, WebFetchError } from "../../web-fetch.js";
 import { ResearchProviderError, type ResearchFetchResult } from "../types.js";
 
 const log = createLogger("research:page-fetch");
@@ -28,61 +29,33 @@ export class PageFetchProvider implements ResearchProvider {
 
   async fetchContent(url: string, config: ResearchProviderConfig = {}, signal?: AbortSignal): Promise<ResearchFetchResult> {
     const timeoutMs = Number(config.timeoutMs ?? this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
-    const timeoutSignal = AbortSignal.timeout(timeoutMs);
-    const requestSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
 
     try {
-      const response = await fetch(url, {
-        method: "GET",
-        redirect: "follow",
-        headers: {
-          "User-Agent": (config.metadata?.userAgent as string) ?? this.options.userAgent ?? DEFAULT_USER_AGENT,
-        },
-        signal: requestSignal,
+      const result = await fetchWebContent(url, {
+        timeoutMs,
+        maxBytes: MAX_CONTENT_CHARS,
+        userAgent: (config.metadata?.userAgent as string) ?? this.options.userAgent ?? DEFAULT_USER_AGENT,
+        signal,
       });
 
-      if (!response.ok) {
-        throw new ResearchProviderError({
-          providerType: "page-fetch",
-          code: response.status >= 500 ? "provider-unavailable" : "network-error",
-          message: `fetch failed with status ${response.status}`,
-          retryable: response.status >= 500,
-        });
-      }
-
-      const contentType = response.headers.get("content-type") ?? "application/octet-stream";
-      const mimeType = contentType.split(";")[0].trim().toLowerCase();
-      const raw = await response.text();
       const metadata: Record<string, unknown> = {
         url,
-        contentType,
-        contentLength: raw.length,
+        contentType: result.contentType,
+        contentLength: result.bytesRead,
+        title: result.title,
+        description: result.description,
       };
 
-      if (mimeType.includes("text/html")) {
-        const extracted = extractHtml(raw);
-        metadata.title = extracted.title;
-        metadata.description = extracted.description;
-        metadata.contentLength = extracted.content.length;
-        return { content: truncate(extracted.content), metadata, mimeType };
-      }
-
-      if (mimeType.includes("application/json") || looksLikeJson(raw)) {
-        const pretty = JSON.stringify(JSON.parse(raw), null, 2);
-        return { content: truncate(pretty), metadata, mimeType };
-      }
-
-      if (mimeType.includes("text/") || mimeType.includes("markdown")) {
-        return { content: truncate(raw), metadata, mimeType };
-      }
-
-      throw new ResearchProviderError({
-        providerType: "page-fetch",
-        code: "provider-unavailable",
-        message: `unsupported mime type: ${mimeType}`,
-      });
+      return {
+        content: result.content,
+        metadata,
+        mimeType: result.mimeType,
+      };
     } catch (error) {
       if (error instanceof ResearchProviderError) throw error;
+      if (error instanceof WebFetchError) {
+        throw mapWebFetchError(error);
+      }
       if (error instanceof DOMException && error.name === "AbortError") {
         throw new ResearchProviderError({ providerType: "page-fetch", code: "abort", message: "Fetch aborted", cause: error });
       }
@@ -101,23 +74,33 @@ export class PageFetchProvider implements ResearchProvider {
   }
 }
 
-function extractHtml(html: string): { title?: string; description?: string; content: string } {
-  const title = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.trim();
-  const description = html.match(/<meta[^>]*name=["']description["'][^>]*content=["']([^"']+)["'][^>]*>/i)?.[1]?.trim();
-  const stripped = html
-    .replace(/<script[\s\S]*?<\/script>/gi, " ")
-    .replace(/<style[\s\S]*?<\/style>/gi, " ")
-    .replace(/<(nav|footer|header)[\s\S]*?<\/\1>/gi, " ");
-  const main = stripped.match(/<(main|article)[^>]*>([\s\S]*?)<\/\1>/i)?.[2] ?? stripped.match(/<body[^>]*>([\s\S]*?)<\/body>/i)?.[1] ?? stripped;
-  const text = main.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
-  return { title, description, content: text };
-}
-
-function truncate(value: string): string {
-  return value.length > MAX_CONTENT_CHARS ? value.slice(0, MAX_CONTENT_CHARS) : value;
-}
-
-function looksLikeJson(value: string): boolean {
-  const trimmed = value.trim();
-  return trimmed.startsWith("{") || trimmed.startsWith("[");
+function mapWebFetchError(error: WebFetchError): ResearchProviderError {
+  switch (error.code) {
+    case "timeout":
+      return new ResearchProviderError({ providerType: "page-fetch", code: "timeout", message: error.message, retryable: true, cause: error });
+    case "unsupported-mime":
+      return new ResearchProviderError({ providerType: "page-fetch", code: "provider-unavailable", message: error.message, cause: error });
+    case "http-error": {
+      const isServerError = /status\s+5\d\d/.test(error.message);
+      return new ResearchProviderError({
+        providerType: "page-fetch",
+        code: isServerError ? "provider-unavailable" : "network-error",
+        message: error.message,
+        retryable: isServerError,
+        cause: error,
+      });
+    }
+    case "network-error":
+      if (error.cause instanceof DOMException && error.cause.name === "AbortError") {
+        return new ResearchProviderError({ providerType: "page-fetch", code: "abort", message: "Fetch aborted", cause: error.cause });
+      }
+      return new ResearchProviderError({ providerType: "page-fetch", code: "network-error", message: error.message, retryable: true, cause: error });
+    case "blocked-host":
+    case "blocked-scheme":
+    case "invalid-url":
+    case "too-large":
+      return new ResearchProviderError({ providerType: "page-fetch", code: "network-error", message: error.message, cause: error });
+    default:
+      return new ResearchProviderError({ providerType: "page-fetch", code: "network-error", message: error.message, retryable: true, cause: error });
+  }
 }

packages/engine/src/step-session-executor.ts

@@ -39,6 +39,7 @@ import {
   createDelegateTaskTool,
   createListAgentsTool,
   createMemoryTools,
+  createWebFetchTool,
   createReadMessagesTool,
   createSendMessageTool,
   createTaskCreateTool,
@@ -901,6 +902,7 @@ export class StepSessionExecutor {
                 createTaskDocumentReadTool(this.options.store, taskDetail.id),
               ]
             : [];
+          const webFetchTool = createWebFetchTool();
           const memoryTools = createMemoryTools(this.options.rootDir, settings);
 
           // Task log and create tools — task context for step sessions.
@@ -965,6 +967,7 @@ Follow instructions precisely and avoid unrelated changes.`,
             customTools: [
               ...pluginTools,
               ...documentTools,
+              webFetchTool,
               ...memoryTools,
               ...taskLogTool,
               ...taskCreateTool,

packages/engine/src/web-fetch.ts

@@ -0,0 +1,248 @@
+import { lookup } from "node:dns/promises";
+import { isIP } from "node:net";
+
+export const dnsResolver = {
+  lookup,
+};
+
+const DEFAULT_TIMEOUT_MS = 30_000;
+const DEFAULT_MAX_BYTES = 500 * 1024;
+const DEFAULT_USER_AGENT = "FusionWebFetch/1.0";
+
+export interface WebFetchOptions {
+  timeoutMs?: number;
+  maxBytes?: number;
+  userAgent?: string;
+  allowPrivateHosts?: boolean;
+  signal?: AbortSignal;
+}
+
+export interface WebFetchResult {
+  url: string;
+  finalUrl: string;
+  status: number;
+  contentType: string;
+  mimeType: string;
+  title?: string;
+  description?: string;
+  content: string;
+  truncated: boolean;
+  bytesRead: number;
+}
+
+export type WebFetchErrorCode =
+  | "invalid-url"
+  | "blocked-host"
+  | "blocked-scheme"
+  | "timeout"
+  | "too-large"
+  | "unsupported-mime"
+  | "http-error"
+  | "network-error";
+
+export class WebFetchError extends Error {
+  readonly code: WebFetchErrorCode;
+
+  constructor(code: WebFetchErrorCode, message: string, options?: { cause?: unknown }) {
+    super(message, options);
+    this.name = "WebFetchError";
+    this.code = code;
+  }
+}
+
+export async function assertSafeUrl(url: string, allowPrivateHosts = false): Promise<void> {
+  let parsed: URL;
+  try {
+    parsed = new URL(url);
+  } catch (error) {
+    throw new WebFetchError("invalid-url", "Invalid URL", { cause: error });
+  }
+
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+    throw new WebFetchError("blocked-scheme", `Blocked URL scheme: ${parsed.protocol}`);
+  }
+
+  if (allowPrivateHosts) {
+    return;
+  }
+
+  const host = normalizeHost(parsed.hostname);
+  if (isIpBlocked(host)) {
+    throw new WebFetchError("blocked-host", `Blocked private host: ${host}`);
+  }
+
+  if (isIP(host) === 0) {
+    try {
+      const resolved = await dnsResolver.lookup(host, { all: true });
+      if (resolved.some((entry) => isIpBlocked(entry.address))) {
+        throw new WebFetchError("blocked-host", `Blocked private host: ${host}`);
+      }
+    } catch (error) {
+      if (error instanceof WebFetchError) {
+        throw error;
+      }
+      throw new WebFetchError("network-error", `DNS lookup failed for ${host}`, { cause: error });
+    }
+  }
+}
+
+export async function fetchWebContent(url: string, options: WebFetchOptions = {}): Promise<WebFetchResult> {
+  const timeoutMs = Number(options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+  const maxBytes = Number(options.maxBytes ?? DEFAULT_MAX_BYTES);
+
+  await assertSafeUrl(url, options.allowPrivateHosts ?? false);
+
+  const timeoutSignal = AbortSignal.timeout(timeoutMs);
+  const requestSignal = options.signal ? AbortSignal.any([options.signal, timeoutSignal]) : timeoutSignal;
+
+  try {
+    if (requestSignal.aborted) {
+      throw new DOMException("Aborted", "AbortError");
+    }
+
+    const response = await fetch(url, {
+      method: "GET",
+      redirect: "follow",
+      headers: {
+        "User-Agent": options.userAgent ?? DEFAULT_USER_AGENT,
+      },
+      signal: requestSignal,
+    });
+
+    if (!response.ok) {
+      throw new WebFetchError("http-error", `fetch failed with status ${response.status}`);
+    }
+
+    const contentType = response.headers.get("content-type") ?? "application/octet-stream";
+    const mimeType = contentType.split(";")[0].trim().toLowerCase();
+    const raw = await response.text();
+
+    let title: string | undefined;
+    let description: string | undefined;
+    let content: string;
+
+    if (mimeType.includes("text/html")) {
+      const extracted = extractHtml(raw);
+      title = extracted.title;
+      description = extracted.description;
+      content = extracted.content;
+    } else if (mimeType.includes("application/json") || looksLikeJson(raw)) {
+      content = JSON.stringify(JSON.parse(raw), null, 2);
+    } else if (mimeType.includes("text/") || mimeType.includes("markdown")) {
+      content = raw;
+    } else {
+      throw new WebFetchError("unsupported-mime", `unsupported mime type: ${mimeType}`);
+    }
+
+    const bytesRead = content.length;
+    if (bytesRead > maxBytes) {
+      return {
+        url,
+        finalUrl: response.url || url,
+        status: response.status,
+        contentType,
+        mimeType,
+        title,
+        description,
+        content: content.slice(0, maxBytes),
+        truncated: true,
+        bytesRead,
+      };
+    }
+
+    return {
+      url,
+      finalUrl: response.url || url,
+      status: response.status,
+      contentType,
+      mimeType,
+      title,
+      description,
+      content,
+      truncated: false,
+      bytesRead,
+    };
+  } catch (error) {
+    if (error instanceof WebFetchError) {
+      throw error;
+    }
+    if (error instanceof DOMException && error.name === "AbortError") {
+      const timedOut = timeoutSignal.aborted && !(options.signal?.aborted ?? false);
+      throw new WebFetchError(timedOut ? "timeout" : "network-error", timedOut ? "Fetch timed out" : "Fetch aborted", { cause: error });
+    }
+    if (error instanceof Error && error.name === "TimeoutError") {
+      throw new WebFetchError("timeout", error.message, { cause: error });
+    }
+    throw new WebFetchError("network-error", error instanceof Error ? error.message : "fetch failed", { cause: error });
+  }
+}
+
+function normalizeHost(host: string): string {
+  if (host.startsWith("[") && host.endsWith("]")) {
+    return host.slice(1, -1);
+  }
+  return host;
+}
+
+function extractHtml(html: string): { title?: string; description?: string; content: string } {
+  const title = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.trim();
+  const description = html.match(/<meta[^>]*name=["']description["'][^>]*content=["']([^"']+)["'][^>]*>/i)?.[1]?.trim();
+  const stripped = html
+    .replace(/<script[\s\S]*?<\/script>/gi, " ")
+    .replace(/<style[\s\S]*?<\/style>/gi, " ")
+    .replace(/<(nav|footer|header)[\s\S]*?<\/\1>/gi, " ");
+  const main = stripped.match(/<(main|article)[^>]*>([\s\S]*?)<\/\1>/i)?.[2] ?? stripped.match(/<body[^>]*>([\s\S]*?)<\/body>/i)?.[1] ?? stripped;
+  const content = main.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+  return { title, description, content };
+}
+
+function looksLikeJson(value: string): boolean {
+  const trimmed = value.trim();
+  return trimmed.startsWith("{") || trimmed.startsWith("[");
+}
+
+function isIpBlocked(address: string): boolean {
+  const version = isIP(address);
+  if (version === 4) {
+    const normalized = normalizeMappedIpv4(address);
+    if (normalized) {
+      return isIpv4Blocked(normalized);
+    }
+    return isIpv4Blocked(address);
+  }
+
+  if (version === 6) {
+    const normalized = address.toLowerCase();
+    if (normalized === "::1") return true;
+    if (normalized.startsWith("fc") || normalized.startsWith("fd")) return true;
+    if (normalized.startsWith("fe8") || normalized.startsWith("fe9") || normalized.startsWith("fea") || normalized.startsWith("feb")) return true;
+    const mapped = normalizeMappedIpv4(normalized);
+    if (mapped) {
+      return isIpv4Blocked(mapped);
+    }
+  }
+
+  return false;
+}
+
+function normalizeMappedIpv4(address: string): string | null {
+  const lower = address.toLowerCase();
+  if (!lower.startsWith("::ffff:")) {
+    return null;
+  }
+  const candidate = lower.slice(7);
+  return isIP(candidate) === 4 ? candidate : null;
+}
+
+function isIpv4Blocked(address: string): boolean {
+  const parts = address.split(".").map((part) => Number(part));
+  const [a, b] = parts;
+  if (a === 0) return true;
+  if (a === 10) return true;
+  if (a === 127) return true;
+  if (a === 169 && b === 254) return true;
+  if (a === 172 && b >= 16 && b <= 31) return true;
+  if (a === 192 && b === 168) return true;
+  if (a === 100 && b >= 64 && b <= 127) return true;
+  return false;
+}