feat(FN-2993): merge fusion/fn-2993
Commits merged: - fix(FN-2993): resolve lint and build verification issues - fix(FN-2993): address provider registry review feedback - feat(FN-2993): complete Step 7 — add provider registry - feat(FN-2993): complete Step 6 — add LLM synthesis provider - feat(FN-2993): complete Step 5 — add local docs provider - feat(FN-2993): complete Step 4 — add GitHub provider - fix(FN-2993): align provider config types with step runner contract - fix(FN-2993): resolve page fetch test module ordering - test(FN-2993): expand page fetch coverage for timeout and truncation - feat(FN-2993): complete Step 3 — add page fetch provider - fix(FN-2993): align web search provider interface and abort assertions - feat(FN-2993): complete Step 2 — add web search provider - feat(FN-2993): complete Step 1 — define provider types and settings fields Files changed: packages/core/src/index.ts | 2 +- packages/core/src/settings-schema.ts | 11 + packages/core/src/types.ts | 25 ++ .../app/components/CustomProvidersSection.tsx | 1 - packages/engine/src/index.ts | 19 ++ .../research/__tests__/provider-registry.test.ts | 49 +++ packages/engine/src/research/provider-registry.ts | 110 +++++++ .../providers/__tests__/github-provider.test.ts | 143 +++++++++ .../__tests__/llm-synthesis-provider.test.ts | 107 +++++++ .../__tests__/local-docs-provider.test.ts | 79 +++++ .../__tests__/page-fetch-provider.test.ts | 92 ++++++ .../__tests__/web-search-provider.test.ts | 101 +++++++ .../src/research/providers/github-provider.ts | 327 +++++++++++++++++++++ packages/engine/src/research/providers/index.ts | 5 + .../research/providers/llm-synthesis-provider.ts | 216 ++++++++++++++ .../src/research/providers/local-docs-provider.ts | 238 +++++++++++++++ .../src/research/providers/page-fetch-provider.ts | 123 ++++++++ .../src/research/providers/web-search-provider.ts | 269 +++++++++++++++++ packages/engine/src/research/types.ts | 50 ++++ 19 files changed, 1965 insertions(+), 2 deletions(-) Fusion-Task-Id: FN-2993
This commit is contained in:
Fusion
@@ -0,0 +1,143 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { runGhAsyncMock, runGhJsonAsyncMock, isGhAvailableMock, isGhAuthenticatedMock } = vi.hoisted(() => ({
|
||||
runGhAsyncMock: vi.fn(),
|
||||
runGhJsonAsyncMock: vi.fn(),
|
||||
isGhAvailableMock: vi.fn(),
|
||||
isGhAuthenticatedMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@fusion/core", async () => {
|
||||
const actual = await vi.importActual<typeof import("@fusion/core")>("@fusion/core");
|
||||
return {
|
||||
...actual,
|
||||
runGhAsync: runGhAsyncMock,
|
||||
runGhJsonAsync: runGhJsonAsyncMock,
|
||||
isGhAvailable: isGhAvailableMock,
|
||||
isGhAuthenticated: isGhAuthenticatedMock,
|
||||
};
|
||||
});
|
||||
|
||||
import { GitHubProvider } from "../github-provider.js";
|
||||
|
||||
describe("GitHubProvider", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
isGhAvailableMock.mockReturnValue(true);
|
||||
isGhAuthenticatedMock.mockReturnValue(true);
|
||||
});
|
||||
|
||||
it("searches repositories", async () => {
|
||||
runGhJsonAsyncMock.mockResolvedValueOnce([
|
||||
{ fullName: "org/repo", description: "desc", htmlUrl: "https://github.com/org/repo", stargazersCount: 12, language: "ts", updatedAt: "2026" },
|
||||
]);
|
||||
const provider = new GitHubProvider();
|
||||
|
||||
const results = await provider.search("query", { metadata: { searchType: "repos" } });
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0]).toMatchObject({ type: "github", title: "org/repo", reference: "https://github.com/org/repo" });
|
||||
});
|
||||
|
||||
it("searches issues", async () => {
|
||||
runGhJsonAsyncMock.mockResolvedValueOnce([
|
||||
{ title: "Issue", body: "body", htmlUrl: "https://github.com/org/repo/issues/1", state: "open", labels: [{ name: "bug" }] },
|
||||
]);
|
||||
|
||||
const provider = new GitHubProvider();
|
||||
const results = await provider.search("query", { metadata: { searchType: "issues" } });
|
||||
|
||||
expect(results[0]?.metadata).toMatchObject({ resultType: "issue", state: "open", labels: ["bug"] });
|
||||
});
|
||||
|
||||
it("supports combined search", async () => {
|
||||
runGhJsonAsyncMock
|
||||
.mockResolvedValueOnce([{ fullName: "org/repo", htmlUrl: "https://github.com/org/repo" }])
|
||||
.mockResolvedValueOnce([{ title: "Issue", htmlUrl: "https://github.com/org/repo/issues/1" }]);
|
||||
|
||||
const provider = new GitHubProvider();
|
||||
const results = await provider.search("query", {});
|
||||
expect(results).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("fetches repo README", async () => {
|
||||
runGhJsonAsyncMock.mockResolvedValueOnce({
|
||||
content: Buffer.from("# Hello").toString("base64"),
|
||||
encoding: "base64",
|
||||
name: "README.md",
|
||||
});
|
||||
|
||||
const provider = new GitHubProvider();
|
||||
const result = await provider.fetchContent("https://github.com/org/repo", {});
|
||||
|
||||
expect(result.content).toContain("# Hello");
|
||||
expect(result.metadata).toMatchObject({ kind: "repo-readme" });
|
||||
});
|
||||
|
||||
it("fetches issue content", async () => {
|
||||
runGhAsyncMock.mockResolvedValueOnce("Issue body\nComments");
|
||||
|
||||
const provider = new GitHubProvider();
|
||||
const result = await provider.fetchContent("https://github.com/org/repo/issues/123", {});
|
||||
|
||||
expect(result.content).toContain("Issue body");
|
||||
expect(result.metadata).toMatchObject({ kind: "issue", number: "123" });
|
||||
});
|
||||
|
||||
it("fetches pr content", async () => {
|
||||
runGhAsyncMock.mockResolvedValueOnce("PR body\nComments");
|
||||
|
||||
const provider = new GitHubProvider();
|
||||
const result = await provider.fetchContent("https://github.com/org/repo/pull/9", {});
|
||||
expect(result.metadata).toMatchObject({ kind: "pr", number: "9" });
|
||||
});
|
||||
|
||||
it("fetches file content from blob url", async () => {
|
||||
runGhJsonAsyncMock.mockResolvedValueOnce({
|
||||
content: Buffer.from("file content").toString("base64"),
|
||||
encoding: "base64",
|
||||
name: "index.ts",
|
||||
});
|
||||
|
||||
const provider = new GitHubProvider();
|
||||
const result = await provider.fetchContent("https://github.com/org/repo/blob/main/src/index.ts", {});
|
||||
expect(result.content).toContain("file content");
|
||||
expect(result.metadata).toMatchObject({ kind: "file", path: "src/index.ts" });
|
||||
});
|
||||
|
||||
it("reports configuration state", () => {
|
||||
const provider = new GitHubProvider();
|
||||
expect(provider.isConfigured()).toBe(true);
|
||||
|
||||
isGhAvailableMock.mockReturnValue(false);
|
||||
expect(provider.isConfigured()).toBe(false);
|
||||
|
||||
isGhAvailableMock.mockReturnValue(true);
|
||||
isGhAuthenticatedMock.mockReturnValue(false);
|
||||
expect(provider.isConfigured()).toBe(false);
|
||||
});
|
||||
|
||||
it("maps abort and timeout errors", async () => {
|
||||
runGhJsonAsyncMock.mockRejectedValueOnce(Object.assign(new Error("gh command aborted"), { code: "ABORT_ERR", stderr: "", stdout: "" }));
|
||||
const provider = new GitHubProvider();
|
||||
await expect(provider.search("q", {})).rejects.toMatchObject({ code: "abort" });
|
||||
|
||||
runGhJsonAsyncMock.mockRejectedValueOnce(Object.assign(new Error("gh command timed out after 30000ms"), { code: null, stderr: "", stdout: "" }));
|
||||
await expect(provider.search("q", {})).rejects.toMatchObject({ code: "timeout" });
|
||||
});
|
||||
|
||||
it("maps rate limit and auth failures", async () => {
|
||||
const provider = new GitHubProvider();
|
||||
|
||||
runGhJsonAsyncMock.mockRejectedValueOnce(Object.assign(new Error("API rate limit exceeded"), { code: 403, stderr: "", stdout: "" }));
|
||||
await expect(provider.search("q", {})).rejects.toMatchObject({ code: "rate-limited", retryable: true });
|
||||
|
||||
runGhJsonAsyncMock.mockRejectedValueOnce(Object.assign(new Error("authentication required"), { code: 401, stderr: "", stdout: "" }));
|
||||
await expect(provider.search("q", {})).rejects.toMatchObject({ code: "auth-failed" });
|
||||
});
|
||||
|
||||
it("errors on unsupported urls", async () => {
|
||||
const provider = new GitHubProvider();
|
||||
await expect(provider.fetchContent("https://example.com/a", {})).rejects.toMatchObject({ code: "provider-unavailable" });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,107 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { createFnAgentMock, promptWithFallbackMock, disposeMock } = vi.hoisted(() => ({
|
||||
disposeMock: vi.fn(),
|
||||
createFnAgentMock: vi.fn(),
|
||||
promptWithFallbackMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../../../pi.js", () => ({
|
||||
createFnAgent: createFnAgentMock,
|
||||
promptWithFallback: promptWithFallbackMock,
|
||||
}));
|
||||
|
||||
import { LLMSynthesisProvider, buildSynthesisPrompt, extractCitations } from "../llm-synthesis-provider.js";
|
||||
|
||||
describe("LLMSynthesisProvider", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
const session = { state: { messages: [] as Array<{ role: string; content: string }> }, dispose: disposeMock };
|
||||
createFnAgentMock.mockResolvedValue({ session });
|
||||
promptWithFallbackMock.mockImplementation(async (s: typeof session, _prompt: string) => {
|
||||
s.state.messages.push({ role: "assistant", content: '```json\n{"summary":"ok","confidence":0.8}\n```\n[1]' });
|
||||
});
|
||||
});
|
||||
|
||||
it("builds synthesis output with citations", async () => {
|
||||
const provider = new LLMSynthesisProvider({ projectRoot: "/tmp" });
|
||||
const result = await provider.synthesize(
|
||||
{
|
||||
query: "q",
|
||||
round: 1,
|
||||
sources: [{ id: "1", type: "web", reference: "https://a", status: "completed", title: "A", content: "hello" }],
|
||||
},
|
||||
{ provider: "openai", modelId: "gpt-5" },
|
||||
);
|
||||
|
||||
expect(promptWithFallbackMock).toHaveBeenCalled();
|
||||
expect(result.citations).toEqual(["https://a"]);
|
||||
expect(result.confidence).toBe(0.8);
|
||||
expect(disposeMock).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("includes query and sources in prompt", () => {
|
||||
const prompt = buildSynthesisPrompt(
|
||||
{ query: "query", round: 2, sources: [{ id: "s", type: "web", reference: "https://x", status: "completed", title: "Title", content: "Body" }] },
|
||||
[{ id: "s", type: "web", reference: "https://x", status: "completed", title: "Title", content: "Body" }],
|
||||
);
|
||||
expect(prompt).toContain("Query: query");
|
||||
expect(prompt).toContain("Reference: https://x");
|
||||
expect(prompt).toContain("Return valid JSON");
|
||||
});
|
||||
|
||||
it("extracts citations by index", () => {
|
||||
const citations = extractCitations("Findings [1] and [2]", [
|
||||
{ id: "1", type: "web", reference: "a", status: "completed" },
|
||||
{ id: "2", type: "web", reference: "b", status: "completed" },
|
||||
]);
|
||||
expect(citations).toEqual(["a", "b"]);
|
||||
});
|
||||
|
||||
it("maps abort and timeout", async () => {
|
||||
const provider = new LLMSynthesisProvider({ projectRoot: "/tmp", timeoutMs: 20 });
|
||||
promptWithFallbackMock.mockImplementation(async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
});
|
||||
await expect(
|
||||
provider.synthesize({ query: "q", round: 1, sources: [] }, { provider: "openai", modelId: "gpt-4o" }),
|
||||
).rejects.toMatchObject({ code: "timeout" });
|
||||
|
||||
promptWithFallbackMock.mockImplementation(async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 80));
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const pending = provider.synthesize({ query: "q", round: 1, sources: [] }, { provider: "openai", modelId: "gpt-4o" }, controller.signal);
|
||||
controller.abort();
|
||||
await expect(pending).rejects.toMatchObject({ code: "abort" });
|
||||
});
|
||||
|
||||
it("reports provider availability", () => {
|
||||
const provider = new LLMSynthesisProvider({ projectRoot: "/tmp" });
|
||||
expect(provider.isConfigured()).toBe(true);
|
||||
});
|
||||
|
||||
it("truncates sources to character budget", async () => {
|
||||
const provider = new LLMSynthesisProvider({ projectRoot: "/tmp" });
|
||||
const manySources = Array.from({ length: 30 }, (_, i) => ({
|
||||
id: String(i),
|
||||
type: "web" as const,
|
||||
reference: `https://src/${i}`,
|
||||
status: "completed" as const,
|
||||
content: "x".repeat(8000),
|
||||
metadata: { confidence: i },
|
||||
}));
|
||||
|
||||
await provider.synthesize({ query: "q", round: 1, sources: manySources }, { provider: "openai", modelId: "small-model" });
|
||||
const promptArg = promptWithFallbackMock.mock.calls[0]?.[1] as string;
|
||||
expect(promptArg.length).toBeLessThan(35_000);
|
||||
});
|
||||
|
||||
it("maps model failures", async () => {
|
||||
const provider = new LLMSynthesisProvider({ projectRoot: "/tmp" });
|
||||
promptWithFallbackMock.mockRejectedValueOnce(new Error("model down"));
|
||||
await expect(provider.synthesize({ query: "q", round: 1, sources: [] }, { provider: "openai", modelId: "gpt-4o" })).rejects.toMatchObject({
|
||||
code: "provider-unavailable",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,79 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import os from "node:os";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { LocalDocsProvider } from "../local-docs-provider.js";
|
||||
|
||||
describe("LocalDocsProvider", () => {
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of tempDirs) rmSync(dir, { recursive: true, force: true });
|
||||
tempDirs.length = 0;
|
||||
});
|
||||
|
||||
function makeProject() {
|
||||
const root = mkdtempSync(join(os.tmpdir(), "fn-test-research-"));
|
||||
tempDirs.push(root);
|
||||
mkdirSync(join(root, "docs"), { recursive: true });
|
||||
writeFileSync(join(root, "README.md"), "Fusion research provider docs");
|
||||
writeFileSync(join(root, "docs", "guide.md"), "This guide covers provider architecture and confidence scoring");
|
||||
return root;
|
||||
}
|
||||
|
||||
it("searches keywords across docs", async () => {
|
||||
const root = makeProject();
|
||||
const provider = new LocalDocsProvider({ projectRoot: root });
|
||||
|
||||
const results = await provider.search("provider confidence", {});
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
expect(results[0]?.reference).toContain("docs/guide.md");
|
||||
});
|
||||
|
||||
it("fetches local file content", async () => {
|
||||
const root = makeProject();
|
||||
const provider = new LocalDocsProvider({ projectRoot: root });
|
||||
|
||||
const result = await provider.fetchContent("README.md", {});
|
||||
expect(result.content).toContain("Fusion research");
|
||||
expect(result.metadata).toMatchObject({ extension: ".md" });
|
||||
});
|
||||
|
||||
it("prevents path traversal", async () => {
|
||||
const root = makeProject();
|
||||
const provider = new LocalDocsProvider({ projectRoot: root });
|
||||
await expect(provider.fetchContent("../../etc/passwd", {})).rejects.toMatchObject({ code: "provider-unavailable" });
|
||||
});
|
||||
|
||||
it("skips binary files during search", async () => {
|
||||
const root = makeProject();
|
||||
writeFileSync(join(root, "docs", "binary.dat"), Buffer.from([0, 1, 2, 3, 4]));
|
||||
const provider = new LocalDocsProvider({ projectRoot: root });
|
||||
|
||||
const results = await provider.search("binary", {});
|
||||
expect(results.find((entry) => String(entry.reference).includes("binary.dat"))).toBeUndefined();
|
||||
});
|
||||
|
||||
it("skips oversized files during search", async () => {
|
||||
const root = makeProject();
|
||||
writeFileSync(join(root, "docs", "large.md"), "x".repeat(1024 * 1024 + 128));
|
||||
const provider = new LocalDocsProvider({ projectRoot: root });
|
||||
|
||||
const results = await provider.search("xxxx", {});
|
||||
expect(results.find((entry) => String(entry.reference).includes("large.md"))).toBeUndefined();
|
||||
});
|
||||
|
||||
it("supports abort signal", async () => {
|
||||
const root = makeProject();
|
||||
for (let i = 0; i < 20; i += 1) {
|
||||
writeFileSync(join(root, "docs", `doc-${i}.md`), `line ${i}`);
|
||||
}
|
||||
|
||||
const provider = new LocalDocsProvider({ projectRoot: root });
|
||||
const controller = new AbortController();
|
||||
const promise = provider.search("line", {}, controller.signal);
|
||||
controller.abort();
|
||||
|
||||
await expect(promise).rejects.toMatchObject({ code: "abort" });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,92 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { MAX_CONTENT_CHARS, PageFetchProvider } from "../page-fetch-provider.js";
|
||||
|
||||
describe("PageFetchProvider", () => {
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
global.fetch = originalFetch;
|
||||
});
|
||||
|
||||
it("extracts readable html content", async () => {
|
||||
global.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
headers: new Headers({ "content-type": "text/html" }),
|
||||
text: async () =>
|
||||
"<html><head><title>Hello</title><meta name='description' content='Desc'></head><body><main><h1>Title</h1><p>Body</p></main></body></html>",
|
||||
} as Response);
|
||||
|
||||
const provider = new PageFetchProvider();
|
||||
const result = await provider.fetchContent("https://example.com", {});
|
||||
|
||||
expect(result.content).toContain("Title Body");
|
||||
expect(result.metadata).toMatchObject({ title: "Hello", description: "Desc", contentType: "text/html" });
|
||||
expect(result.mimeType).toBe("text/html");
|
||||
});
|
||||
|
||||
it("pretty prints json", async () => {
|
||||
global.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
headers: new Headers({ "content-type": "application/json" }),
|
||||
text: async () => '{"a":1}',
|
||||
} as Response);
|
||||
|
||||
const provider = new PageFetchProvider();
|
||||
const result = await provider.fetchContent("https://example.com", {});
|
||||
expect(result.content).toContain('"a": 1');
|
||||
});
|
||||
|
||||
it("passes through text", async () => {
|
||||
global.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
headers: new Headers({ "content-type": "text/plain" }),
|
||||
text: async () => "hello world",
|
||||
} as Response);
|
||||
|
||||
const provider = new PageFetchProvider();
|
||||
const result = await provider.fetchContent("https://example.com", {});
|
||||
expect(result.content).toBe("hello world");
|
||||
});
|
||||
|
||||
it("maps status errors", async () => {
|
||||
global.fetch = vi.fn().mockResolvedValue({ ok: false, status: 404 } as Response);
|
||||
const provider = new PageFetchProvider();
|
||||
await expect(provider.fetchContent("https://example.com", {})).rejects.toMatchObject({ code: "network-error" });
|
||||
});
|
||||
|
||||
it("maps 500 as provider unavailable", async () => {
|
||||
global.fetch = vi.fn().mockResolvedValue({ ok: false, status: 500 } as Response);
|
||||
const provider = new PageFetchProvider();
|
||||
await expect(provider.fetchContent("https://example.com", {})).rejects.toMatchObject({ code: "provider-unavailable" });
|
||||
});
|
||||
|
||||
it("truncates large content", async () => {
|
||||
global.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
headers: new Headers({ "content-type": "text/plain" }),
|
||||
text: async () => "x".repeat(600 * 1024),
|
||||
} as Response);
|
||||
const provider = new PageFetchProvider();
|
||||
const result = await provider.fetchContent("https://example.com", {});
|
||||
expect(result.content.length).toBe(MAX_CONTENT_CHARS);
|
||||
});
|
||||
|
||||
it("maps timeout", async () => {
|
||||
global.fetch = vi.fn().mockRejectedValue(Object.assign(new Error("Timed out"), { name: "TimeoutError" }));
|
||||
const provider = new PageFetchProvider({ timeoutMs: 50 });
|
||||
await expect(provider.fetchContent("https://example.com", {})).rejects.toMatchObject({ code: "timeout" });
|
||||
});
|
||||
|
||||
it("maps abort", async () => {
|
||||
global.fetch = vi.fn().mockImplementation(async (_url, init: RequestInit) => {
|
||||
await new Promise((_, reject) => init.signal?.addEventListener("abort", () => reject(new DOMException("Aborted", "AbortError"))));
|
||||
return { ok: true, headers: new Headers(), text: async () => "" } as Response;
|
||||
});
|
||||
const provider = new PageFetchProvider();
|
||||
const controller = new AbortController();
|
||||
const pending = provider.fetchContent("https://example.com", {}, controller.signal);
|
||||
controller.abort();
|
||||
await expect(pending).rejects.toMatchObject({ code: "abort" });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,101 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { ResearchProviderError } from "../../types.js";
|
||||
import { WebSearchProvider } from "../web-search-provider.js";
|
||||
|
||||
describe("WebSearchProvider", () => {
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
global.fetch = originalFetch;
|
||||
});
|
||||
|
||||
it("validates configuration for each backend", () => {
|
||||
expect(new WebSearchProvider({ backend: "none" }).isConfigured()).toBe(false);
|
||||
expect(new WebSearchProvider({ backend: "searxng", searxngUrl: "https://sx" }).isConfigured()).toBe(true);
|
||||
expect(new WebSearchProvider({ backend: "brave", braveApiKey: "k" }).isConfigured()).toBe(true);
|
||||
expect(new WebSearchProvider({ backend: "google", googleApiKey: "k", googleCx: "cx" }).isConfigured()).toBe(true);
|
||||
expect(new WebSearchProvider({ backend: "tavily", tavilyApiKey: "k" }).isConfigured()).toBe(true);
|
||||
});
|
||||
|
||||
it("normalizes searxng results", async () => {
|
||||
global.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ results: [{ url: "https://a", title: "A", content: "Snippet" }] }),
|
||||
} as Response);
|
||||
|
||||
const provider = new WebSearchProvider({ backend: "searxng", searxngUrl: "https://sx" });
|
||||
const results = await provider.search("fusion", {});
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0]).toMatchObject({
|
||||
title: "A",
|
||||
reference: "https://a",
|
||||
excerpt: "Snippet",
|
||||
metadata: { backend: "searxng", rank: 1 },
|
||||
});
|
||||
});
|
||||
|
||||
it("limits max results", async () => {
|
||||
global.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
web: {
|
||||
results: [
|
||||
{ url: "https://a", title: "A", description: "1" },
|
||||
{ url: "https://b", title: "B", description: "2" },
|
||||
],
|
||||
},
|
||||
}),
|
||||
} as Response);
|
||||
|
||||
const provider = new WebSearchProvider({ backend: "brave", braveApiKey: "k" });
|
||||
const results = await provider.search("fusion", { maxResults: 1 });
|
||||
expect(results).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("retries 429 and succeeds", async () => {
|
||||
global.fetch = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ ok: false, status: 429 } as Response)
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({ items: [{ link: "https://x", title: "X", snippet: "S" }] }),
|
||||
} as Response);
|
||||
|
||||
const provider = new WebSearchProvider({ backend: "google", googleApiKey: "k", googleCx: "cx" });
|
||||
const promise = provider.search("fusion", {});
|
||||
await vi.runAllTimersAsync();
|
||||
const results = await promise;
|
||||
|
||||
expect(global.fetch).toHaveBeenCalledTimes(2);
|
||||
expect(results[0]?.reference).toBe("https://x");
|
||||
});
|
||||
|
||||
it("classifies auth failures", async () => {
|
||||
global.fetch = vi.fn().mockResolvedValue({ ok: false, status: 401 } as Response);
|
||||
const provider = new WebSearchProvider({ backend: "tavily", tavilyApiKey: "k" });
|
||||
await expect(provider.search("fusion", {})).rejects.toMatchObject({ code: "auth-failed" });
|
||||
});
|
||||
|
||||
it("propagates abort signal", async () => {
|
||||
global.fetch = vi.fn().mockImplementation(async (_url, init: RequestInit) => {
|
||||
await new Promise((_, reject) => {
|
||||
init.signal?.addEventListener("abort", () => reject(new DOMException("Aborted", "AbortError")));
|
||||
});
|
||||
return { ok: true, json: async () => ({}) } as Response;
|
||||
});
|
||||
|
||||
const controller = new AbortController();
|
||||
const provider = new WebSearchProvider({ backend: "brave", braveApiKey: "k" });
|
||||
const pending = provider.search("fusion", {}, controller.signal);
|
||||
controller.abort();
|
||||
await expect(pending).rejects.toMatchObject({ code: "abort", providerType: "web-search" });
|
||||
});
|
||||
|
||||
});
|
||||
327
packages/engine/src/research/providers/github-provider.ts
Normal file
327
packages/engine/src/research/providers/github-provider.ts
Normal file
@@ -0,0 +1,327 @@
|
||||
import {
|
||||
isGhAuthenticated,
|
||||
isGhAvailable,
|
||||
runGhAsync,
|
||||
runGhJsonAsync,
|
||||
type GhError,
|
||||
type ResearchProviderConfig,
|
||||
type ResearchSource,
|
||||
} from "@fusion/core";
|
||||
import type { ResearchProvider } from "../../research-step-runner.js";
|
||||
import { createLogger } from "../../logger.js";
|
||||
import { ResearchProviderError, type ResearchFetchResult } from "../types.js";
|
||||
|
||||
const log = createLogger("research:github");
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
|
||||
type SearchType = "repos" | "issues" | "both";
|
||||
|
||||
interface GitHubRepoResult {
|
||||
fullName: string;
|
||||
description?: string;
|
||||
htmlUrl: string;
|
||||
stargazersCount?: number;
|
||||
language?: string;
|
||||
updatedAt?: string;
|
||||
}
|
||||
|
||||
interface GitHubIssueResult {
|
||||
title: string;
|
||||
body?: string;
|
||||
htmlUrl: string;
|
||||
state?: string;
|
||||
labels?: Array<{ name?: string }>;
|
||||
}
|
||||
|
||||
interface ParsedGitHubUrl {
|
||||
owner: string;
|
||||
repo: string;
|
||||
kind: "repo" | "issue" | "pr" | "file";
|
||||
number?: string;
|
||||
filePath?: string;
|
||||
ref?: string;
|
||||
}
|
||||
|
||||
export class GitHubProvider implements ResearchProvider {
|
||||
readonly type = "github";
|
||||
|
||||
async search(query: string, config: ResearchProviderConfig = {}, signal?: AbortSignal): Promise<ResearchSource[]> {
|
||||
const timeoutMs = Number(config.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||
const maxResults = Number(config.maxResults ?? 10);
|
||||
const searchType = ((config.metadata?.searchType as SearchType | undefined) ?? "both");
|
||||
|
||||
const sources: ResearchSource[] = [];
|
||||
|
||||
try {
|
||||
if (searchType === "repos" || searchType === "both") {
|
||||
const repos = await runGhJsonAsync<GitHubRepoResult[]>(
|
||||
[
|
||||
"search",
|
||||
"repos",
|
||||
query,
|
||||
"--json",
|
||||
"fullName,description,htmlUrl,stargazersCount,language,updatedAt",
|
||||
"--limit",
|
||||
String(maxResults),
|
||||
],
|
||||
{ signal, timeoutMs },
|
||||
);
|
||||
|
||||
sources.push(
|
||||
...repos.slice(0, maxResults).map((repo, idx) => ({
|
||||
id: `github-repo-${idx}-${repo.htmlUrl}`,
|
||||
type: "github" as const,
|
||||
reference: repo.htmlUrl,
|
||||
title: repo.fullName,
|
||||
excerpt: repo.description ?? "",
|
||||
status: "completed" as const,
|
||||
metadata: {
|
||||
resultType: "repo",
|
||||
stars: repo.stargazersCount ?? 0,
|
||||
language: repo.language,
|
||||
updatedAt: repo.updatedAt,
|
||||
rank: idx + 1,
|
||||
},
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
if (searchType === "issues" || searchType === "both") {
|
||||
const issues = await runGhJsonAsync<GitHubIssueResult[]>(
|
||||
[
|
||||
"search",
|
||||
"issues",
|
||||
query,
|
||||
"--json",
|
||||
"title,body,htmlUrl,state,labels",
|
||||
"--limit",
|
||||
String(maxResults),
|
||||
],
|
||||
{ signal, timeoutMs },
|
||||
);
|
||||
|
||||
sources.push(
|
||||
...issues.slice(0, maxResults).map((issue, idx) => ({
|
||||
id: `github-issue-${idx}-${issue.htmlUrl}`,
|
||||
type: "github" as const,
|
||||
reference: issue.htmlUrl,
|
||||
title: issue.title,
|
||||
excerpt: issue.body?.slice(0, 280) ?? "",
|
||||
status: "completed" as const,
|
||||
metadata: {
|
||||
resultType: "issue",
|
||||
state: issue.state,
|
||||
labels: (issue.labels ?? []).map((label) => label.name).filter(Boolean),
|
||||
rank: idx + 1,
|
||||
},
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
return sources;
|
||||
} catch (error) {
|
||||
throw this.mapGhError(error);
|
||||
}
|
||||
}
|
||||
|
||||
async fetchContent(url: string, config: ResearchProviderConfig = {}, signal?: AbortSignal): Promise<ResearchFetchResult> {
|
||||
const parsed = parseGitHubUrl(url);
|
||||
if (!parsed) {
|
||||
throw new ResearchProviderError({
|
||||
providerType: "github",
|
||||
code: "provider-unavailable",
|
||||
message: "Unsupported GitHub URL",
|
||||
});
|
||||
}
|
||||
|
||||
const timeoutMs = Number(config.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
if (parsed.kind === "repo") {
|
||||
const readme = await runGhJsonAsync<{ content?: string; encoding?: string; name?: string }>(
|
||||
["api", `repos/${parsed.owner}/${parsed.repo}/readme`],
|
||||
{ signal, timeoutMs },
|
||||
);
|
||||
|
||||
if (readme.encoding !== "base64" || !readme.content) {
|
||||
throw new ResearchProviderError({
|
||||
providerType: "github",
|
||||
code: "provider-unavailable",
|
||||
message: "Unsupported README encoding",
|
||||
});
|
||||
}
|
||||
|
||||
const content = Buffer.from(readme.content.replace(/\n/g, ""), "base64").toString("utf-8");
|
||||
return {
|
||||
content,
|
||||
metadata: {
|
||||
url,
|
||||
owner: parsed.owner,
|
||||
repo: parsed.repo,
|
||||
kind: "repo-readme",
|
||||
name: readme.name,
|
||||
},
|
||||
mimeType: "text/markdown",
|
||||
};
|
||||
}
|
||||
|
||||
if (parsed.kind === "issue") {
|
||||
const issue = await runGhAsync(["issue", "view", parsed.number ?? "", "--repo", `${parsed.owner}/${parsed.repo}`, "--comments"], {
|
||||
signal,
|
||||
timeoutMs,
|
||||
});
|
||||
return {
|
||||
content: issue,
|
||||
metadata: { url, owner: parsed.owner, repo: parsed.repo, kind: "issue", number: parsed.number },
|
||||
mimeType: "text/plain",
|
||||
};
|
||||
}
|
||||
|
||||
if (parsed.kind === "pr") {
|
||||
const pr = await runGhAsync(["pr", "view", parsed.number ?? "", "--repo", `${parsed.owner}/${parsed.repo}`, "--comments"], {
|
||||
signal,
|
||||
timeoutMs,
|
||||
});
|
||||
return {
|
||||
content: pr,
|
||||
metadata: { url, owner: parsed.owner, repo: parsed.repo, kind: "pr", number: parsed.number },
|
||||
mimeType: "text/plain",
|
||||
};
|
||||
}
|
||||
|
||||
const apiPath = `repos/${parsed.owner}/${parsed.repo}/contents/${parsed.filePath ?? ""}${parsed.ref ? `?ref=${encodeURIComponent(parsed.ref)}` : ""}`;
|
||||
const file = await runGhJsonAsync<{ content?: string; encoding?: string; name?: string }>(["api", apiPath], {
|
||||
signal,
|
||||
timeoutMs,
|
||||
});
|
||||
const content = file.encoding === "base64" && file.content
|
||||
? Buffer.from(file.content.replace(/\n/g, ""), "base64").toString("utf-8")
|
||||
: (file.content ?? "");
|
||||
|
||||
return {
|
||||
content,
|
||||
metadata: {
|
||||
url,
|
||||
owner: parsed.owner,
|
||||
repo: parsed.repo,
|
||||
kind: "file",
|
||||
path: parsed.filePath,
|
||||
ref: parsed.ref,
|
||||
name: file.name,
|
||||
},
|
||||
mimeType: "text/plain",
|
||||
};
|
||||
} catch (error) {
|
||||
throw this.mapGhError(error);
|
||||
}
|
||||
}
|
||||
|
||||
isConfigured(): boolean {
|
||||
return isGhAvailable() && isGhAuthenticated();
|
||||
}
|
||||
|
||||
private mapGhError(error: unknown): ResearchProviderError {
|
||||
if (error instanceof ResearchProviderError) return error;
|
||||
|
||||
if (isGhError(error)) {
|
||||
const message = `${error.message}${error.stderr ? `: ${error.stderr}` : ""}`;
|
||||
const lowered = message.toLowerCase();
|
||||
|
||||
if (error.code === "ABORT_ERR" || lowered.includes("aborted")) {
|
||||
return new ResearchProviderError({ providerType: "github", code: "abort", message, cause: error });
|
||||
}
|
||||
if (lowered.includes("timed out")) {
|
||||
return new ResearchProviderError({
|
||||
providerType: "github",
|
||||
code: "timeout",
|
||||
message,
|
||||
retryable: true,
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
if (error.code === 403 || lowered.includes("rate limit")) {
|
||||
return new ResearchProviderError({
|
||||
providerType: "github",
|
||||
code: "rate-limited",
|
||||
message,
|
||||
retryable: true,
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
if (error.code === 401 || lowered.includes("not logged") || lowered.includes("authentication")) {
|
||||
return new ResearchProviderError({
|
||||
providerType: "github",
|
||||
code: "auth-failed",
|
||||
message,
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
if (error.code === 404 || lowered.includes("not found")) {
|
||||
return new ResearchProviderError({
|
||||
providerType: "github",
|
||||
code: "provider-unavailable",
|
||||
message,
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
|
||||
return new ResearchProviderError({
|
||||
providerType: "github",
|
||||
code: "network-error",
|
||||
message,
|
||||
retryable: true,
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
|
||||
log.warn("github provider error", { error });
|
||||
return new ResearchProviderError({
|
||||
providerType: "github",
|
||||
code: "network-error",
|
||||
message: error instanceof Error ? error.message : "GitHub provider failed",
|
||||
retryable: true,
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function isGhError(value: unknown): value is GhError {
|
||||
return value instanceof Error && "stderr" in value && "stdout" in value && "code" in value;
|
||||
}
|
||||
|
||||
function parseGitHubUrl(url: string): ParsedGitHubUrl | null {
|
||||
let parsedUrl: URL;
|
||||
try {
|
||||
parsedUrl = new URL(url);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!/(^|\.)github\.com$/i.test(parsedUrl.hostname)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const segments = parsedUrl.pathname.split("/").filter(Boolean);
|
||||
if (segments.length < 2) return null;
|
||||
|
||||
const [owner, repo, section, ...rest] = segments;
|
||||
if (!section) {
|
||||
return { owner, repo, kind: "repo" };
|
||||
}
|
||||
|
||||
if (section === "issues" && rest[0]) {
|
||||
return { owner, repo, kind: "issue", number: rest[0] };
|
||||
}
|
||||
|
||||
if (section === "pull" && rest[0]) {
|
||||
return { owner, repo, kind: "pr", number: rest[0] };
|
||||
}
|
||||
|
||||
if (section === "blob" && rest.length >= 2) {
|
||||
return { owner, repo, kind: "file", ref: rest[0], filePath: rest.slice(1).join("/") };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export { parseGitHubUrl };
|
||||
5
packages/engine/src/research/providers/index.ts
Normal file
5
packages/engine/src/research/providers/index.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
export { WebSearchProvider, type WebSearchProviderOptions } from "./web-search-provider.js";
|
||||
export { PageFetchProvider, type PageFetchProviderOptions } from "./page-fetch-provider.js";
|
||||
export { GitHubProvider } from "./github-provider.js";
|
||||
export { LocalDocsProvider, type LocalDocsProviderOptions } from "./local-docs-provider.js";
|
||||
export { LLMSynthesisProvider, type LLMSynthesisProviderOptions } from "./llm-synthesis-provider.js";
|
||||
216
packages/engine/src/research/providers/llm-synthesis-provider.ts
Normal file
216
packages/engine/src/research/providers/llm-synthesis-provider.ts
Normal file
@@ -0,0 +1,216 @@
|
||||
import type { ResearchProviderConfig, ResearchSource, ResearchSynthesisRequest, ResearchSynthesisResult, ResolvedModelSelection } from "@fusion/core";
|
||||
import type { ResearchProvider } from "../../research-step-runner.js";
|
||||
import { createLogger } from "../../logger.js";
|
||||
import { createFnAgent, promptWithFallback } from "../../pi.js";
|
||||
import { ResearchProviderError } from "../types.js";
|
||||
|
||||
const log = createLogger("research:llm-synthesis");
|
||||
const DEFAULT_TIMEOUT_MS = 120_000;
|
||||
const LARGE_MODEL_CONTEXT_CHARS = 100_000;
|
||||
const SMALL_MODEL_CONTEXT_CHARS = 30_000;
|
||||
|
||||
export interface LLMSynthesisProviderOptions {
|
||||
projectRoot: string;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
export class LLMSynthesisProvider implements ResearchProvider {
|
||||
readonly type = "llm-synthesis";
|
||||
|
||||
constructor(private readonly options: LLMSynthesisProviderOptions) {}
|
||||
|
||||
isConfigured(): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
async search(_query: string, _options: ResearchProviderConfig = {}, _signal?: AbortSignal): Promise<ResearchSource[]> {
|
||||
return [];
|
||||
}
|
||||
|
||||
async fetchContent(
|
||||
_url: string,
|
||||
_options: ResearchProviderConfig = {},
|
||||
_signal?: AbortSignal,
|
||||
): Promise<{ content: string; metadata: Record<string, unknown> }> {
|
||||
return { content: "", metadata: {} };
|
||||
}
|
||||
|
||||
async synthesize(
|
||||
request: ResearchSynthesisRequest,
|
||||
modelSelection: ResolvedModelSelection,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ResearchSynthesisResult> {
|
||||
if (!modelSelection?.provider || !modelSelection?.modelId) {
|
||||
throw new ResearchProviderError({ providerType: "llm-synthesis", code: "provider-unavailable", message: "Synthesis model is not configured" });
|
||||
}
|
||||
|
||||
const timeoutSignal = AbortSignal.timeout(this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||
const requestSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
|
||||
|
||||
try {
|
||||
const cappedSources = this.applySourceBudget(request.sources, modelSelection);
|
||||
const prompt = buildSynthesisPrompt(request, cappedSources);
|
||||
|
||||
const { session } = await createFnAgent({
|
||||
cwd: this.options.projectRoot,
|
||||
tools: "readonly",
|
||||
systemPrompt: "You synthesize research findings into concise, cited outputs.",
|
||||
defaultProvider: modelSelection.provider,
|
||||
defaultModelId: modelSelection.modelId,
|
||||
});
|
||||
|
||||
try {
|
||||
await Promise.race([
|
||||
promptWithFallback(session, prompt),
|
||||
new Promise<never>((_, reject) => {
|
||||
requestSignal.addEventListener(
|
||||
"abort",
|
||||
() => reject(new ResearchProviderError({
|
||||
providerType: "llm-synthesis",
|
||||
code: signal?.aborted ? "abort" : "timeout",
|
||||
message: signal?.aborted ? "Synthesis aborted" : "Synthesis timed out",
|
||||
retryable: !signal?.aborted,
|
||||
})),
|
||||
{ once: true },
|
||||
);
|
||||
}),
|
||||
]);
|
||||
|
||||
if (requestSignal.aborted) {
|
||||
throw new ResearchProviderError({ providerType: "llm-synthesis", code: signal?.aborted ? "abort" : "timeout", message: signal?.aborted ? "Synthesis aborted" : "Synthesis timed out", retryable: !signal?.aborted });
|
||||
}
|
||||
|
||||
const responseText = extractAssistantText(session);
|
||||
if (!responseText) {
|
||||
throw new ResearchProviderError({ providerType: "llm-synthesis", code: "provider-unavailable", message: "No synthesis response received" });
|
||||
}
|
||||
|
||||
return {
|
||||
output: responseText,
|
||||
citations: extractCitations(responseText, cappedSources),
|
||||
confidence: extractConfidence(responseText),
|
||||
metadata: {
|
||||
sourceCount: cappedSources.length,
|
||||
truncated: cappedSources.length < request.sources.length,
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
session.dispose();
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof ResearchProviderError) throw error;
|
||||
if (error instanceof DOMException && error.name === "AbortError") {
|
||||
throw new ResearchProviderError({ providerType: "llm-synthesis", code: "abort", message: "Synthesis aborted", cause: error });
|
||||
}
|
||||
log.warn("llm synthesis failed", { error });
|
||||
throw new ResearchProviderError({
|
||||
providerType: "llm-synthesis",
|
||||
code: "provider-unavailable",
|
||||
message: error instanceof Error ? error.message : "Synthesis failed",
|
||||
retryable: true,
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private applySourceBudget(sources: ResearchSource[], modelSelection: ResolvedModelSelection): ResearchSource[] {
|
||||
// Approximation: 1 token ~= 4 English chars. We cap at 80% of an estimated
|
||||
// context window to preserve room for prompt/instructions and response.
|
||||
const contextChars = isLargeModel(modelSelection.modelId) ? LARGE_MODEL_CONTEXT_CHARS : SMALL_MODEL_CONTEXT_CHARS;
|
||||
const budget = Math.floor(contextChars * 0.8);
|
||||
const ordered = [...sources].sort((a, b) => scoreSource(b) - scoreSource(a));
|
||||
|
||||
let total = 0;
|
||||
const kept: ResearchSource[] = [];
|
||||
for (const source of ordered) {
|
||||
const chunk = `${source.title ?? ""}\n${source.excerpt ?? ""}\n${source.content ?? ""}`;
|
||||
if (total + chunk.length > budget) continue;
|
||||
total += chunk.length;
|
||||
kept.push(source);
|
||||
}
|
||||
|
||||
return kept.length > 0 ? kept : ordered.slice(0, 1);
|
||||
}
|
||||
}
|
||||
|
||||
function isLargeModel(modelId?: string): boolean {
|
||||
const id = modelId?.toLowerCase() ?? "";
|
||||
return id.includes("gpt-5") || id.includes("claude-opus") || id.includes("gemini-2.5-pro") || id.includes("sonnet");
|
||||
}
|
||||
|
||||
function scoreSource(source: ResearchSource): number {
|
||||
const confidence = typeof source.metadata?.confidence === "number" ? source.metadata.confidence : 0;
|
||||
const hasContent = source.content ? 1 : 0;
|
||||
return confidence * 10 + hasContent;
|
||||
}
|
||||
|
||||
function buildSynthesisPrompt(request: ResearchSynthesisRequest, sources: ResearchSource[]): string {
|
||||
const format = request.desiredFormat ?? "markdown";
|
||||
const renderedSources = sources
|
||||
.map(
|
||||
(source, index) => `Source [${index + 1}]\nTitle: ${source.title ?? source.reference}\nReference: ${source.reference}\nExcerpt: ${source.excerpt ?? ""}\nContent: ${source.content ?? ""}`,
|
||||
)
|
||||
.join("\n\n");
|
||||
|
||||
return [
|
||||
"You are a research synthesis assistant.",
|
||||
`Query: ${request.query}`,
|
||||
`Round: ${request.round}`,
|
||||
`Desired format: ${format}`,
|
||||
"Analyze the provided sources and produce: summary, key findings, contradictions, confidence, and follow-up queries.",
|
||||
"Cite source references inline using [n] notation where n is source number.",
|
||||
request.instructions ? `Additional instructions: ${request.instructions}` : "",
|
||||
"Sources:",
|
||||
renderedSources,
|
||||
'Return valid JSON: {"summary": string, "findings": [{"statement": string, "citations": string[]}], "confidence": number, "followUps": string[]}',
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
function extractAssistantText(session: { state?: { messages?: Array<{ role?: string; content?: unknown }> } }): string | undefined {
|
||||
const messages = session.state?.messages;
|
||||
if (!Array.isArray(messages)) return undefined;
|
||||
for (let i = messages.length - 1; i >= 0; i -= 1) {
|
||||
const msg = messages[i];
|
||||
if (msg.role !== "assistant") continue;
|
||||
if (typeof msg.content === "string" && msg.content.trim()) return msg.content;
|
||||
if (Array.isArray(msg.content)) {
|
||||
for (const part of msg.content) {
|
||||
if (typeof part === "object" && part && "text" in part && typeof (part as { text?: unknown }).text === "string") {
|
||||
return (part as { text: string }).text;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function extractCitations(text: string, sources: ResearchSource[]): string[] {
|
||||
const matches = text.match(/\[(\d+)\]/g) ?? [];
|
||||
const refs = new Set<string>();
|
||||
for (const match of matches) {
|
||||
const idx = Number.parseInt(match.replace(/\D/g, ""), 10) - 1;
|
||||
if (Number.isFinite(idx) && idx >= 0 && idx < sources.length) {
|
||||
refs.add(sources[idx].reference);
|
||||
}
|
||||
}
|
||||
return [...refs];
|
||||
}
|
||||
|
||||
function extractConfidence(text: string): number | undefined {
|
||||
const jsonBlock = text.match(/```(?:json)?\s*([\s\S]*?)```/i)?.[1] ?? text;
|
||||
try {
|
||||
const parsed = JSON.parse(jsonBlock);
|
||||
if (typeof parsed.confidence === "number") return parsed.confidence;
|
||||
} catch {
|
||||
const match = text.match(/"confidence"\s*:\s*([0-9]*\.?[0-9]+)/i);
|
||||
if (match) {
|
||||
const parsed = Number.parseFloat(match[1]);
|
||||
if (Number.isFinite(parsed)) return parsed;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export { buildSynthesisPrompt, extractCitations };
|
||||
238
packages/engine/src/research/providers/local-docs-provider.ts
Normal file
238
packages/engine/src/research/providers/local-docs-provider.ts
Normal file
@@ -0,0 +1,238 @@
|
||||
import { promises as fs } from "node:fs";
|
||||
import { extname, join, relative, resolve } from "node:path";
|
||||
import type { ResearchProviderConfig, ResearchSource } from "@fusion/core";
|
||||
import type { ResearchProvider } from "../../research-step-runner.js";
|
||||
import { createLogger } from "../../logger.js";
|
||||
import { ResearchProviderError, type ResearchFetchResult } from "../types.js";
|
||||
|
||||
const log = createLogger("research:local-docs");
|
||||
const DEFAULT_MAX_RESULTS = 10;
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
const MAX_FILE_SIZE_BYTES = 1024 * 1024;
|
||||
const BINARY_SNIFF_BYTES = 8 * 1024;
|
||||
|
||||
export interface LocalDocsProviderOptions {
|
||||
projectRoot: string;
|
||||
timeoutMs?: number;
|
||||
maxResults?: number;
|
||||
scanPaths?: string[];
|
||||
}
|
||||
|
||||
export class LocalDocsProvider implements ResearchProvider {
|
||||
readonly type = "local-docs";
|
||||
private readonly projectRoot: string;
|
||||
private readonly scanPaths: string[];
|
||||
|
||||
constructor(private readonly options: LocalDocsProviderOptions) {
|
||||
this.projectRoot = resolve(options.projectRoot);
|
||||
this.scanPaths = options.scanPaths ?? ["docs", "README.md", "AGENTS.md", ".fusion/memory"];
|
||||
}
|
||||
|
||||
isConfigured(): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
async search(query: string, config: ResearchProviderConfig = {}, signal?: AbortSignal): Promise<ResearchSource[]> {
|
||||
const timeoutMs = Number(config.timeoutMs ?? this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||
const maxResults = Number(config.maxResults ?? this.options.maxResults ?? DEFAULT_MAX_RESULTS);
|
||||
const terms = query.toLowerCase().split(/\s+/).filter(Boolean);
|
||||
|
||||
const files = await this.withTimeout(this.collectCandidateFiles(signal), timeoutMs, signal);
|
||||
const results: Array<{ source: ResearchSource; score: number }> = [];
|
||||
|
||||
for (const file of files) {
|
||||
this.throwIfAborted(signal);
|
||||
const content = await this.safeReadText(file, signal);
|
||||
if (!content) continue;
|
||||
const lower = content.toLowerCase();
|
||||
let score = 0;
|
||||
for (const term of terms) {
|
||||
const matches = lower.match(new RegExp(escapeRegex(term), "g"));
|
||||
score += matches?.length ?? 0;
|
||||
}
|
||||
if (score <= 0) continue;
|
||||
|
||||
const relPath = relative(this.projectRoot, file);
|
||||
results.push({
|
||||
score,
|
||||
source: {
|
||||
id: `local-docs-${relPath}`,
|
||||
type: "local",
|
||||
reference: relPath,
|
||||
title: relPath,
|
||||
excerpt: buildExcerpt(content, terms),
|
||||
status: "completed",
|
||||
metadata: { score, path: relPath },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return results
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.slice(0, maxResults)
|
||||
.map((item) => item.source);
|
||||
}
|
||||
|
||||
async fetchContent(filePath: string, config: ResearchProviderConfig = {}, signal?: AbortSignal): Promise<ResearchFetchResult> {
|
||||
const timeoutMs = Number(config.timeoutMs ?? this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||
const resolvedPath = resolve(this.projectRoot, filePath);
|
||||
if (!resolvedPath.startsWith(this.projectRoot)) {
|
||||
throw new ResearchProviderError({
|
||||
providerType: "local-docs",
|
||||
code: "provider-unavailable",
|
||||
message: "Path traversal is not allowed",
|
||||
});
|
||||
}
|
||||
|
||||
const stat = await this.withTimeout(fs.stat(resolvedPath), timeoutMs, signal);
|
||||
if (!stat.isFile()) {
|
||||
throw new ResearchProviderError({ providerType: "local-docs", code: "provider-unavailable", message: "Path is not a file" });
|
||||
}
|
||||
|
||||
const content = await this.withTimeout(fs.readFile(resolvedPath), timeoutMs, signal);
|
||||
const sniff = content.subarray(0, BINARY_SNIFF_BYTES);
|
||||
if (sniff.includes(0)) {
|
||||
throw new ResearchProviderError({ providerType: "local-docs", code: "provider-unavailable", message: "Binary file is not supported" });
|
||||
}
|
||||
|
||||
const text = content.toString("utf-8");
|
||||
return {
|
||||
content: text.length > MAX_FILE_SIZE_BYTES ? text.slice(0, MAX_FILE_SIZE_BYTES) : text,
|
||||
metadata: {
|
||||
path: relative(this.projectRoot, resolvedPath),
|
||||
size: stat.size,
|
||||
modifiedAt: stat.mtime.toISOString(),
|
||||
extension: extname(resolvedPath),
|
||||
},
|
||||
mimeType: "text/plain",
|
||||
};
|
||||
}
|
||||
|
||||
private async collectCandidateFiles(signal?: AbortSignal): Promise<string[]> {
|
||||
const ignorePatterns = await this.readGitignore();
|
||||
const files: string[] = [];
|
||||
|
||||
for (const pathEntry of this.scanPaths) {
|
||||
const target = resolve(this.projectRoot, pathEntry);
|
||||
if (!target.startsWith(this.projectRoot)) continue;
|
||||
try {
|
||||
const stat = await fs.stat(target);
|
||||
if (stat.isDirectory()) {
|
||||
await this.walk(target, files, ignorePatterns, signal);
|
||||
} else if (stat.isFile()) {
|
||||
files.push(target);
|
||||
}
|
||||
} catch {
|
||||
// ignore missing entries
|
||||
}
|
||||
}
|
||||
|
||||
const rootEntries = await fs.readdir(this.projectRoot, { withFileTypes: true });
|
||||
for (const entry of rootEntries) {
|
||||
if (entry.isFile() && entry.name.toLowerCase().endsWith(".md")) {
|
||||
files.push(join(this.projectRoot, entry.name));
|
||||
}
|
||||
}
|
||||
|
||||
return [...new Set(files)];
|
||||
}
|
||||
|
||||
private async walk(dir: string, out: string[], ignorePatterns: string[], signal?: AbortSignal): Promise<void> {
|
||||
this.throwIfAborted(signal);
|
||||
const entries = await fs.readdir(dir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
this.throwIfAborted(signal);
|
||||
const fullPath = join(dir, entry.name);
|
||||
const relPath = relative(this.projectRoot, fullPath).replace(/\\/g, "/");
|
||||
if (matchesGitignore(relPath, ignorePatterns)) continue;
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await this.walk(fullPath, out, ignorePatterns, signal);
|
||||
} else if (entry.isFile()) {
|
||||
out.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async safeReadText(filePath: string, signal?: AbortSignal): Promise<string | undefined> {
|
||||
try {
|
||||
const stat = await fs.stat(filePath);
|
||||
if (stat.size > MAX_FILE_SIZE_BYTES) return undefined;
|
||||
const content = await fs.readFile(filePath);
|
||||
if (content.subarray(0, BINARY_SNIFF_BYTES).includes(0)) return undefined;
|
||||
this.throwIfAborted(signal);
|
||||
return content.toString("utf-8");
|
||||
} catch (error) {
|
||||
log.warn("failed to read local docs file", { filePath, error });
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
private async readGitignore(): Promise<string[]> {
|
||||
try {
|
||||
const content = await fs.readFile(join(this.projectRoot, ".gitignore"), "utf-8");
|
||||
return content
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line && !line.startsWith("#"));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private throwIfAborted(signal?: AbortSignal): void {
|
||||
if (signal?.aborted) {
|
||||
throw new ResearchProviderError({ providerType: "local-docs", code: "abort", message: "Local docs scan aborted" });
|
||||
}
|
||||
}
|
||||
|
||||
private async withTimeout<T>(promise: Promise<T>, timeoutMs: number, signal?: AbortSignal): Promise<T> {
|
||||
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
||||
const combined = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
|
||||
return await Promise.race([
|
||||
promise,
|
||||
new Promise<never>((_, reject) => {
|
||||
combined.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
reject(
|
||||
new ResearchProviderError({
|
||||
providerType: "local-docs",
|
||||
code: signal?.aborted ? "abort" : "timeout",
|
||||
message: signal?.aborted ? "Local docs operation aborted" : `Local docs operation timed out after ${timeoutMs}ms`,
|
||||
retryable: !signal?.aborted,
|
||||
}),
|
||||
);
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
}),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
function buildExcerpt(content: string, terms: string[]): string {
|
||||
const lower = content.toLowerCase();
|
||||
const first = terms.find((term) => lower.includes(term));
|
||||
if (!first) return content.slice(0, 220).replace(/\s+/g, " ").trim();
|
||||
|
||||
const idx = lower.indexOf(first);
|
||||
const start = Math.max(0, idx - 80);
|
||||
const end = Math.min(content.length, idx + 140);
|
||||
return content.slice(start, end).replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function matchesGitignore(relPath: string, patterns: string[]): boolean {
|
||||
return patterns.some((pattern) => {
|
||||
if (pattern.endsWith("/")) return relPath.startsWith(pattern.slice(0, -1));
|
||||
if (pattern.includes("*")) {
|
||||
const regex = new RegExp(`^${pattern.split("*").map(escapeRegex).join(".*")}$`);
|
||||
return regex.test(relPath);
|
||||
}
|
||||
return relPath === pattern || relPath.startsWith(`${pattern}/`);
|
||||
});
|
||||
}
|
||||
|
||||
function escapeRegex(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
123
packages/engine/src/research/providers/page-fetch-provider.ts
Normal file
123
packages/engine/src/research/providers/page-fetch-provider.ts
Normal file
@@ -0,0 +1,123 @@
|
||||
import type { ResearchProviderConfig, ResearchSource } from "@fusion/core";
|
||||
import type { ResearchProvider } from "../../research-step-runner.js";
|
||||
import { createLogger } from "../../logger.js";
|
||||
import { ResearchProviderError, type ResearchFetchResult } from "../types.js";
|
||||
|
||||
const log = createLogger("research:page-fetch");
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_USER_AGENT = "FusionResearchBot/1.0";
|
||||
export const MAX_CONTENT_CHARS = 500 * 1024;
|
||||
|
||||
export interface PageFetchProviderOptions {
|
||||
timeoutMs?: number;
|
||||
userAgent?: string;
|
||||
}
|
||||
|
||||
export class PageFetchProvider implements ResearchProvider {
|
||||
readonly type = "page-fetch";
|
||||
|
||||
constructor(private readonly options: PageFetchProviderOptions = {}) {}
|
||||
|
||||
isConfigured(): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
async search(_query: string, _config: ResearchProviderConfig = {}, _signal?: AbortSignal): Promise<ResearchSource[]> {
|
||||
return [];
|
||||
}
|
||||
|
||||
async fetchContent(url: string, config: ResearchProviderConfig = {}, signal?: AbortSignal): Promise<ResearchFetchResult> {
|
||||
const timeoutMs = Number(config.timeoutMs ?? this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
||||
const requestSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
redirect: "follow",
|
||||
headers: {
|
||||
"User-Agent": (config.metadata?.userAgent as string) ?? this.options.userAgent ?? DEFAULT_USER_AGENT,
|
||||
},
|
||||
signal: requestSignal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResearchProviderError({
|
||||
providerType: "page-fetch",
|
||||
code: response.status >= 500 ? "provider-unavailable" : "network-error",
|
||||
message: `fetch failed with status ${response.status}`,
|
||||
retryable: response.status >= 500,
|
||||
});
|
||||
}
|
||||
|
||||
const contentType = response.headers.get("content-type") ?? "application/octet-stream";
|
||||
const mimeType = contentType.split(";")[0].trim().toLowerCase();
|
||||
const raw = await response.text();
|
||||
const metadata: Record<string, unknown> = {
|
||||
url,
|
||||
contentType,
|
||||
contentLength: raw.length,
|
||||
};
|
||||
|
||||
if (mimeType.includes("text/html")) {
|
||||
const extracted = extractHtml(raw);
|
||||
metadata.title = extracted.title;
|
||||
metadata.description = extracted.description;
|
||||
metadata.contentLength = extracted.content.length;
|
||||
return { content: truncate(extracted.content), metadata, mimeType };
|
||||
}
|
||||
|
||||
if (mimeType.includes("application/json") || looksLikeJson(raw)) {
|
||||
const pretty = JSON.stringify(JSON.parse(raw), null, 2);
|
||||
return { content: truncate(pretty), metadata, mimeType };
|
||||
}
|
||||
|
||||
if (mimeType.includes("text/") || mimeType.includes("markdown")) {
|
||||
return { content: truncate(raw), metadata, mimeType };
|
||||
}
|
||||
|
||||
throw new ResearchProviderError({
|
||||
providerType: "page-fetch",
|
||||
code: "provider-unavailable",
|
||||
message: `unsupported mime type: ${mimeType}`,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof ResearchProviderError) throw error;
|
||||
if (error instanceof DOMException && error.name === "AbortError") {
|
||||
throw new ResearchProviderError({ providerType: "page-fetch", code: "abort", message: "Fetch aborted", cause: error });
|
||||
}
|
||||
if (error instanceof Error && error.name === "TimeoutError") {
|
||||
throw new ResearchProviderError({ providerType: "page-fetch", code: "timeout", message: error.message, retryable: true, cause: error });
|
||||
}
|
||||
log.warn("page fetch failed", { error });
|
||||
throw new ResearchProviderError({
|
||||
providerType: "page-fetch",
|
||||
code: "network-error",
|
||||
message: error instanceof Error ? error.message : "fetch failed",
|
||||
retryable: true,
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function extractHtml(html: string): { title?: string; description?: string; content: string } {
|
||||
const title = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.trim();
|
||||
const description = html.match(/<meta[^>]*name=["']description["'][^>]*content=["']([^"']+)["'][^>]*>/i)?.[1]?.trim();
|
||||
const stripped = html
|
||||
.replace(/<script[\s\S]*?<\/script>/gi, " ")
|
||||
.replace(/<style[\s\S]*?<\/style>/gi, " ")
|
||||
.replace(/<(nav|footer|header)[\s\S]*?<\/\1>/gi, " ");
|
||||
const main = stripped.match(/<(main|article)[^>]*>([\s\S]*?)<\/\1>/i)?.[2] ?? stripped.match(/<body[^>]*>([\s\S]*?)<\/body>/i)?.[1] ?? stripped;
|
||||
const text = main.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
|
||||
return { title, description, content: text };
|
||||
}
|
||||
|
||||
function truncate(value: string): string {
|
||||
return value.length > MAX_CONTENT_CHARS ? value.slice(0, MAX_CONTENT_CHARS) : value;
|
||||
}
|
||||
|
||||
function looksLikeJson(value: string): boolean {
|
||||
const trimmed = value.trim();
|
||||
return trimmed.startsWith("{") || trimmed.startsWith("[");
|
||||
}
|
||||
269
packages/engine/src/research/providers/web-search-provider.ts
Normal file
269
packages/engine/src/research/providers/web-search-provider.ts
Normal file
@@ -0,0 +1,269 @@
|
||||
import type { ResearchProviderConfig, ResearchSource, WebSearchBackend } from "@fusion/core";
|
||||
import type { ResearchProvider } from "../../research-step-runner.js";
|
||||
import { createLogger } from "../../logger.js";
|
||||
import { ResearchProviderError } from "../types.js";
|
||||
|
||||
const log = createLogger("research:web-search");
|
||||
|
||||
export interface WebSearchProviderOptions {
|
||||
backend?: WebSearchBackend;
|
||||
searxngUrl?: string;
|
||||
braveApiKey?: string;
|
||||
googleApiKey?: string;
|
||||
googleCx?: string;
|
||||
tavilyApiKey?: string;
|
||||
maxResults?: number;
|
||||
timeoutMs?: number;
|
||||
userAgent?: string;
|
||||
}
|
||||
|
||||
const DEFAULT_MAX_RESULTS = 10;
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
const RETRY_BASE_DELAY_MS = 1_000;
|
||||
const RETRY_MAX_DELAY_MS = 10_000;
|
||||
const RETRY_MAX_ATTEMPTS = 3;
|
||||
|
||||
export class WebSearchProvider implements ResearchProvider {
|
||||
readonly type = "web-search";
|
||||
|
||||
constructor(private readonly options: WebSearchProviderOptions = {}) {}
|
||||
|
||||
isConfigured(): boolean {
|
||||
const backend = this.options.backend ?? "none";
|
||||
if (backend === "none") return false;
|
||||
if (backend === "searxng") return Boolean(this.options.searxngUrl);
|
||||
if (backend === "brave") return Boolean(this.options.braveApiKey);
|
||||
if (backend === "google") return Boolean(this.options.googleApiKey && this.options.googleCx);
|
||||
if (backend === "tavily") return Boolean(this.options.tavilyApiKey);
|
||||
return false;
|
||||
}
|
||||
|
||||
async search(query: string, config: ResearchProviderConfig = {}, signal?: AbortSignal): Promise<ResearchSource[]> {
|
||||
const backend = this.options.backend ?? "none";
|
||||
if (!this.isConfigured()) return [];
|
||||
|
||||
const maxResults = Number(config.maxResults ?? this.options.maxResults ?? DEFAULT_MAX_RESULTS);
|
||||
const timeoutMs = Number(config.timeoutMs ?? this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
||||
const requestSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
|
||||
|
||||
const results = await this.withHttpRetry(async () => {
|
||||
if (backend === "searxng") return this.searchSearxng(query, maxResults, requestSignal);
|
||||
if (backend === "brave") return this.searchBrave(query, maxResults, requestSignal);
|
||||
if (backend === "google") return this.searchGoogle(query, maxResults, requestSignal);
|
||||
if (backend === "tavily") return this.searchTavily(query, maxResults, requestSignal);
|
||||
return [];
|
||||
}, requestSignal, backend);
|
||||
|
||||
return results.slice(0, maxResults).map((result, index) => ({
|
||||
id: `${backend}-${index}-${result.url}`,
|
||||
type: "web",
|
||||
reference: result.url,
|
||||
title: result.title,
|
||||
excerpt: result.snippet,
|
||||
status: "completed",
|
||||
metadata: {
|
||||
backend,
|
||||
rank: index + 1,
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
async fetchContent(
|
||||
_url: string,
|
||||
_options: ResearchProviderConfig = {},
|
||||
_signal?: AbortSignal,
|
||||
): Promise<{ content: string; metadata: Record<string, unknown> }> {
|
||||
return { content: "", metadata: {} };
|
||||
}
|
||||
|
||||
private async searchSearxng(query: string, maxResults: number, signal: AbortSignal) {
|
||||
const base = this.options.searxngUrl?.replace(/\/$/, "") ?? "";
|
||||
const url = new URL(`${base}/search`);
|
||||
url.searchParams.set("q", query);
|
||||
url.searchParams.set("format", "json");
|
||||
|
||||
const response = await this.requestJson<{ results?: Array<{ url: string; title?: string; content?: string }> }>(
|
||||
url.toString(),
|
||||
{
|
||||
method: "GET",
|
||||
signal,
|
||||
},
|
||||
"searxng",
|
||||
);
|
||||
|
||||
return (response.results ?? []).slice(0, maxResults).map((item) => ({
|
||||
url: item.url,
|
||||
title: item.title ?? item.url,
|
||||
snippet: item.content ?? "",
|
||||
}));
|
||||
}
|
||||
|
||||
private async searchBrave(query: string, maxResults: number, signal: AbortSignal) {
|
||||
const url = new URL("https://api.search.brave.com/res/v1/web/search");
|
||||
url.searchParams.set("q", query);
|
||||
url.searchParams.set("count", String(maxResults));
|
||||
|
||||
const response = await this.requestJson<{ web?: { results?: Array<{ url: string; title?: string; description?: string }> } }>(
|
||||
url.toString(),
|
||||
{
|
||||
method: "GET",
|
||||
headers: {
|
||||
"X-Subscription-Token": this.options.braveApiKey ?? "",
|
||||
"User-Agent": this.options.userAgent ?? "FusionResearchBot/1.0",
|
||||
},
|
||||
signal,
|
||||
},
|
||||
"brave",
|
||||
);
|
||||
|
||||
return (response.web?.results ?? []).slice(0, maxResults).map((item) => ({
|
||||
url: item.url,
|
||||
title: item.title ?? item.url,
|
||||
snippet: item.description ?? "",
|
||||
}));
|
||||
}
|
||||
|
||||
private async searchGoogle(query: string, maxResults: number, signal: AbortSignal) {
|
||||
const url = new URL("https://www.googleapis.com/customsearch/v1");
|
||||
url.searchParams.set("q", query);
|
||||
url.searchParams.set("num", String(maxResults));
|
||||
url.searchParams.set("key", this.options.googleApiKey ?? "");
|
||||
url.searchParams.set("cx", this.options.googleCx ?? "");
|
||||
|
||||
const response = await this.requestJson<{ items?: Array<{ link: string; title?: string; snippet?: string }> }>(
|
||||
url.toString(),
|
||||
{
|
||||
method: "GET",
|
||||
signal,
|
||||
},
|
||||
"google",
|
||||
);
|
||||
|
||||
return (response.items ?? []).slice(0, maxResults).map((item) => ({
|
||||
url: item.link,
|
||||
title: item.title ?? item.link,
|
||||
snippet: item.snippet ?? "",
|
||||
}));
|
||||
}
|
||||
|
||||
private async searchTavily(query: string, maxResults: number, signal: AbortSignal) {
|
||||
const response = await this.requestJson<{ results?: Array<{ url: string; title?: string; content?: string }> }>(
|
||||
"https://api.tavily.com/search",
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": this.options.userAgent ?? "FusionResearchBot/1.0",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
api_key: this.options.tavilyApiKey,
|
||||
query,
|
||||
max_results: maxResults,
|
||||
}),
|
||||
signal,
|
||||
},
|
||||
"tavily",
|
||||
);
|
||||
|
||||
return (response.results ?? []).slice(0, maxResults).map((item) => ({
|
||||
url: item.url,
|
||||
title: item.title ?? item.url,
|
||||
snippet: item.content ?? "",
|
||||
}));
|
||||
}
|
||||
|
||||
private async withHttpRetry<T>(
|
||||
operation: () => Promise<T>,
|
||||
signal: AbortSignal,
|
||||
backend: WebSearchBackend,
|
||||
): Promise<T> {
|
||||
let attempt = 0;
|
||||
let lastError: unknown;
|
||||
while (attempt < RETRY_MAX_ATTEMPTS) {
|
||||
if (signal.aborted) {
|
||||
throw new ResearchProviderError({ providerType: "web-search", code: "abort", message: "Search aborted" });
|
||||
}
|
||||
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
const isRetryableHttp =
|
||||
error instanceof ResearchProviderError &&
|
||||
(error.code === "rate-limited" || error.code === "network-error" || error.code === "provider-unavailable") &&
|
||||
error.retryable;
|
||||
|
||||
attempt += 1;
|
||||
if (!isRetryableHttp || attempt >= RETRY_MAX_ATTEMPTS) {
|
||||
break;
|
||||
}
|
||||
|
||||
const delay = Math.min(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), RETRY_MAX_DELAY_MS);
|
||||
const jitter = delay * (Math.random() * 0.2 - 0.1);
|
||||
const totalDelay = Math.max(0, delay + jitter);
|
||||
log.warn(`retrying ${backend} search`, { attempt, totalDelay });
|
||||
await sleep(totalDelay, signal);
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
private async requestJson<T>(url: string, init: RequestInit, backend: WebSearchBackend): Promise<T> {
|
||||
try {
|
||||
const response = await fetch(url, init);
|
||||
if (!response.ok) {
|
||||
const message = `${backend} request failed with status ${response.status}`;
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
throw new ResearchProviderError({ providerType: "web-search", code: "auth-failed", message });
|
||||
}
|
||||
if (response.status === 429) {
|
||||
throw new ResearchProviderError({
|
||||
providerType: "web-search",
|
||||
code: "rate-limited",
|
||||
message,
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
if (response.status >= 500) {
|
||||
throw new ResearchProviderError({
|
||||
providerType: "web-search",
|
||||
code: "provider-unavailable",
|
||||
message,
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
throw new ResearchProviderError({ providerType: "web-search", code: "network-error", message });
|
||||
}
|
||||
|
||||
return (await response.json()) as T;
|
||||
} catch (error) {
|
||||
if (error instanceof ResearchProviderError) throw error;
|
||||
if (error instanceof DOMException && error.name === "AbortError") {
|
||||
throw new ResearchProviderError({ providerType: "web-search", code: "abort", message: "Search aborted", cause: error });
|
||||
}
|
||||
if (error instanceof Error && error.name === "TimeoutError") {
|
||||
throw new ResearchProviderError({ providerType: "web-search", code: "timeout", message: error.message, retryable: true, cause: error });
|
||||
}
|
||||
throw new ResearchProviderError({
|
||||
providerType: "web-search",
|
||||
code: "network-error",
|
||||
message: error instanceof Error ? error.message : "Unexpected network error",
|
||||
retryable: true,
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function sleep(ms: number, signal: AbortSignal): Promise<void> {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(resolve, ms);
|
||||
const onAbort = () => {
|
||||
clearTimeout(timer);
|
||||
reject(new ResearchProviderError({ providerType: "web-search", code: "abort", message: "Search aborted" }));
|
||||
};
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user