feat(FN-2993): merge fusion/fn-2993
Commits merged: - fix(FN-2993): resolve lint and build verification issues - fix(FN-2993): address provider registry review feedback - feat(FN-2993): complete Step 7 — add provider registry - feat(FN-2993): complete Step 6 — add LLM synthesis provider - feat(FN-2993): complete Step 5 — add local docs provider - feat(FN-2993): complete Step 4 — add GitHub provider - fix(FN-2993): align provider config types with step runner contract - fix(FN-2993): resolve page fetch test module ordering - test(FN-2993): expand page fetch coverage for timeout and truncation - feat(FN-2993): complete Step 3 — add page fetch provider - fix(FN-2993): align web search provider interface and abort assertions - feat(FN-2993): complete Step 2 — add web search provider - feat(FN-2993): complete Step 1 — define provider types and settings fields Files changed: packages/core/src/index.ts | 2 +- packages/core/src/settings-schema.ts | 11 + packages/core/src/types.ts | 25 ++ .../app/components/CustomProvidersSection.tsx | 1 - packages/engine/src/index.ts | 19 ++ .../research/__tests__/provider-registry.test.ts | 49 +++ packages/engine/src/research/provider-registry.ts | 110 +++++++ .../providers/__tests__/github-provider.test.ts | 143 +++++++++ .../__tests__/llm-synthesis-provider.test.ts | 107 +++++++ .../__tests__/local-docs-provider.test.ts | 79 +++++ .../__tests__/page-fetch-provider.test.ts | 92 ++++++ .../__tests__/web-search-provider.test.ts | 101 +++++++ .../src/research/providers/github-provider.ts | 327 +++++++++++++++++++++ packages/engine/src/research/providers/index.ts | 5 + .../research/providers/llm-synthesis-provider.ts | 216 ++++++++++++++ .../src/research/providers/local-docs-provider.ts | 238 +++++++++++++++ .../src/research/providers/page-fetch-provider.ts | 123 ++++++++ .../src/research/providers/web-search-provider.ts | 269 +++++++++++++++++ packages/engine/src/research/types.ts | 50 ++++ 19 files changed, 1965 insertions(+), 2 deletions(-) Fusion-Task-Id: FN-2993
This commit is contained in:
@@ -1,5 +1,5 @@
|
|||||||
export { COLUMNS, COLUMN_LABELS, COLUMN_DESCRIPTIONS, VALID_TRANSITIONS, DEFAULT_SETTINGS, DEFAULT_GLOBAL_SETTINGS, DEFAULT_PROJECT_SETTINGS, GLOBAL_SETTINGS_KEYS, PROJECT_SETTINGS_KEYS, isGlobalSettingsKey, isProjectSettingsKey, THINKING_LEVELS, THEME_MODES, COLOR_THEMES, WORKFLOW_STEP_TEMPLATES, AGENT_PERMISSIONS, agentToConfigSnapshot, diffConfigSnapshots, isEphemeralAgent, hasAgentIdentity, CheckoutConflictError, DEFAULT_HEARTBEAT_PROCEDURE_PATH, getDefaultHeartbeatProcedurePath, EXECUTION_MODES, DEFAULT_EXECUTION_MODE, TASK_PRIORITIES, DEFAULT_TASK_PRIORITY, validateMessageMetadata, normalizeMergeConflictStrategy } from "./types.js";
|
export { COLUMNS, COLUMN_LABELS, COLUMN_DESCRIPTIONS, VALID_TRANSITIONS, DEFAULT_SETTINGS, DEFAULT_GLOBAL_SETTINGS, DEFAULT_PROJECT_SETTINGS, GLOBAL_SETTINGS_KEYS, PROJECT_SETTINGS_KEYS, isGlobalSettingsKey, isProjectSettingsKey, THINKING_LEVELS, THEME_MODES, COLOR_THEMES, WORKFLOW_STEP_TEMPLATES, AGENT_PERMISSIONS, agentToConfigSnapshot, diffConfigSnapshots, isEphemeralAgent, hasAgentIdentity, CheckoutConflictError, DEFAULT_HEARTBEAT_PROCEDURE_PATH, getDefaultHeartbeatProcedurePath, EXECUTION_MODES, DEFAULT_EXECUTION_MODE, TASK_PRIORITIES, DEFAULT_TASK_PRIORITY, validateMessageMetadata, normalizeMergeConflictStrategy } from "./types.js";
|
||||||
export type { Column, IssueInfo, IssueState, TaskSourceIssue, PrInfo, PrStatus, Task, TaskTokenUsage, TaskAttachment, TaskComment, TaskCommentInput, TaskDocument, TaskDocumentRevision, TaskDocumentCreateInput, TaskDocumentWithTask, TaskCreateInput, TaskSource, SourceType, TaskDetail, InboxTask, TodoList, TodoItem, TodoListCreateInput, TodoListUpdateInput, TodoItemCreateInput, TodoItemUpdateInput, TodoListWithItems, AgentLogEntry, AgentLogType, AgentRole, BoardConfig, MergeDetails, MergeResult, MergeConflictStrategy, CanonicalMergeConflictStrategy, Settings, GlobalSettings, ProjectSettings, SettingsScope, DaemonTokenSettings, TaskStep, StepStatus, TaskLogEntry, RunMutationContext, ActivityLogEntry, ActivityEventType, ThinkingLevel, ThemeMode, ColorTheme, ExecutionMode, TaskPriority, UnavailableNodePolicy, PlanningQuestion, PlanningSummary, PlanningResponse, PlanningQuestionType, ArchivedTaskEntry, BatchStatusRequest, BatchStatusResponse, BatchStatusEntry, BatchStatusResult, ModelPreset, WorkflowStep, WorkflowStepMode, WorkflowStepPhase, WorkflowStepInput, WorkflowStepResult, WorkflowStepTemplate, Agent, OrgTreeNode, AgentState, AgentDetail, AgentCreateInput, AgentUpdateInput, AgentApiKey, AgentApiKeyCreateResult, AgentCapability, AgentPromptTemplate, AgentPromptsConfig, AgentPermission, TaskAssignSource, AgentAccessState, AgentHeartbeatConfig, AgentBudgetConfig, AgentBudgetStatus, InstructionsBundleConfig, MessageResponseMode, AgentHeartbeatEvent, AgentHeartbeatRun, BlockedStateSnapshot, HeartbeatInvocationSource, AgentTaskSession, AgentRating, AgentRatingSummary, AgentRatingInput, AgentConfigSnapshot, RevisionFieldDiff, AgentConfigRevision, AgentStats, ReflectionTrigger, ReflectionMetrics, AgentReflection, AgentPerformanceSummary, NtfyNotificationEvent, NotificationEvent, NotificationPayload, NotificationProviderConfig, CustomProvider, SteeringComment, ParticipantType, MessageType, Message, MessageCreateInput, MessageFilter, MessageMetadata, MessageReplyReference, Mailbox, CheckoutLease, RunAuditDomain, RunAuditEvent, RunAuditEventInput, RunAuditEventFilter } from "./types.js";
|
export type { Column, IssueInfo, IssueState, TaskSourceIssue, PrInfo, PrStatus, Task, TaskTokenUsage, TaskAttachment, TaskComment, TaskCommentInput, TaskDocument, TaskDocumentRevision, TaskDocumentCreateInput, TaskDocumentWithTask, TaskCreateInput, TaskSource, SourceType, TaskDetail, InboxTask, TodoList, TodoItem, TodoListCreateInput, TodoListUpdateInput, TodoItemCreateInput, TodoItemUpdateInput, TodoListWithItems, AgentLogEntry, AgentLogType, AgentRole, BoardConfig, MergeDetails, MergeResult, MergeConflictStrategy, CanonicalMergeConflictStrategy, Settings, GlobalSettings, ProjectSettings, WebSearchBackend, SettingsScope, DaemonTokenSettings, TaskStep, StepStatus, TaskLogEntry, RunMutationContext, ActivityLogEntry, ActivityEventType, ThinkingLevel, ThemeMode, ColorTheme, ExecutionMode, TaskPriority, UnavailableNodePolicy, PlanningQuestion, PlanningSummary, PlanningResponse, PlanningQuestionType, ArchivedTaskEntry, BatchStatusRequest, BatchStatusResponse, BatchStatusEntry, BatchStatusResult, ModelPreset, WorkflowStep, WorkflowStepMode, WorkflowStepPhase, WorkflowStepInput, WorkflowStepResult, WorkflowStepTemplate, Agent, OrgTreeNode, AgentState, AgentDetail, AgentCreateInput, AgentUpdateInput, AgentApiKey, AgentApiKeyCreateResult, AgentCapability, AgentPromptTemplate, AgentPromptsConfig, AgentPermission, TaskAssignSource, AgentAccessState, AgentHeartbeatConfig, AgentBudgetConfig, AgentBudgetStatus, InstructionsBundleConfig, MessageResponseMode, AgentHeartbeatEvent, AgentHeartbeatRun, BlockedStateSnapshot, HeartbeatInvocationSource, AgentTaskSession, AgentRating, AgentRatingSummary, AgentRatingInput, AgentConfigSnapshot, RevisionFieldDiff, AgentConfigRevision, AgentStats, ReflectionTrigger, ReflectionMetrics, AgentReflection, AgentPerformanceSummary, NtfyNotificationEvent, NotificationEvent, NotificationPayload, NotificationProviderConfig, CustomProvider, SteeringComment, ParticipantType, MessageType, Message, MessageCreateInput, MessageFilter, MessageMetadata, MessageReplyReference, Mailbox, CheckoutLease, RunAuditDomain, RunAuditEvent, RunAuditEventInput, RunAuditEventFilter } from "./types.js";
|
||||||
export { AGENT_VALID_TRANSITIONS } from "./types.js";
|
export { AGENT_VALID_TRANSITIONS } from "./types.js";
|
||||||
export {
|
export {
|
||||||
BUILTIN_AGENT_PROMPTS,
|
BUILTIN_AGENT_PROMPTS,
|
||||||
|
|||||||
@@ -69,6 +69,17 @@ export const DEFAULT_GLOBAL_SETTINGS = {
|
|||||||
researchGlobalDefaultTimeout: 300000,
|
researchGlobalDefaultTimeout: 300000,
|
||||||
researchGlobalMaxSourcesPerRun: 20,
|
researchGlobalMaxSourcesPerRun: 20,
|
||||||
researchGlobalMaxSynthesisRounds: 2,
|
researchGlobalMaxSynthesisRounds: 2,
|
||||||
|
researchWebSearchProvider: "none",
|
||||||
|
researchSearxngUrl: undefined,
|
||||||
|
researchBraveApiKey: undefined,
|
||||||
|
researchGoogleSearchApiKey: undefined,
|
||||||
|
researchGoogleSearchCx: undefined,
|
||||||
|
researchTavilyApiKey: undefined,
|
||||||
|
researchGitHubEnabled: false,
|
||||||
|
researchLocalDocsEnabled: true,
|
||||||
|
researchMaxSearchResults: 10,
|
||||||
|
researchFetchTimeoutMs: 30_000,
|
||||||
|
researchUserAgent: "FusionResearchBot/1.0",
|
||||||
} satisfies CompleteSettings<GlobalSettings>;
|
} satisfies CompleteSettings<GlobalSettings>;
|
||||||
|
|
||||||
/** Default values for project-level settings. */
|
/** Default values for project-level settings. */
|
||||||
|
|||||||
@@ -1187,6 +1187,9 @@ export interface DaemonTokenSettings {
|
|||||||
* These are user preferences that persist across all fn projects.
|
* These are user preferences that persist across all fn projects.
|
||||||
* The dashboard UI shows these under a "Global" section.
|
* The dashboard UI shows these under a "Global" section.
|
||||||
*/
|
*/
|
||||||
|
/** Web search backend for auto-research provider. */
|
||||||
|
export type WebSearchBackend = "searxng" | "brave" | "google" | "tavily" | "none";
|
||||||
|
|
||||||
export interface GlobalSettings {
|
export interface GlobalSettings {
|
||||||
/** Theme mode preference: dark, light, or system (follows OS). Default: "dark". */
|
/** Theme mode preference: dark, light, or system (follows OS). Default: "dark". */
|
||||||
themeMode?: ThemeMode;
|
themeMode?: ThemeMode;
|
||||||
@@ -1405,6 +1408,28 @@ export interface GlobalSettings {
|
|||||||
/** Default maximum number of synthesis rounds per run.
|
/** Default maximum number of synthesis rounds per run.
|
||||||
* Default: 2. */
|
* Default: 2. */
|
||||||
researchGlobalMaxSynthesisRounds?: number;
|
researchGlobalMaxSynthesisRounds?: number;
|
||||||
|
/** Web search backend for auto-research. Default: "none" (disabled). */
|
||||||
|
researchWebSearchProvider?: WebSearchBackend;
|
||||||
|
/** SearXNG instance URL (required when researchWebSearchProvider is "searxng"). */
|
||||||
|
researchSearxngUrl?: string;
|
||||||
|
/** Brave Search API key (required when researchWebSearchProvider is "brave"). */
|
||||||
|
researchBraveApiKey?: string;
|
||||||
|
/** Google Custom Search API key (required when researchWebSearchProvider is "google"). */
|
||||||
|
researchGoogleSearchApiKey?: string;
|
||||||
|
/** Google Custom Search engine ID (required when researchWebSearchProvider is "google"). */
|
||||||
|
researchGoogleSearchCx?: string;
|
||||||
|
/** Tavily API key (required when researchWebSearchProvider is "tavily"). */
|
||||||
|
researchTavilyApiKey?: string;
|
||||||
|
/** Enable GitHub repository/issue search provider. Default: false. */
|
||||||
|
researchGitHubEnabled?: boolean;
|
||||||
|
/** Enable local project documentation search provider. Default: true. */
|
||||||
|
researchLocalDocsEnabled?: boolean;
|
||||||
|
/** Maximum search results per provider query. Default: 10. */
|
||||||
|
researchMaxSearchResults?: number;
|
||||||
|
/** HTTP fetch timeout in milliseconds for page/content fetching. Default: 30000. */
|
||||||
|
researchFetchTimeoutMs?: number;
|
||||||
|
/** User-Agent header for HTTP requests made by research providers. Default: "FusionResearchBot/1.0". */
|
||||||
|
researchUserAgent?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type RemoteAccessProvider = "tailscale" | "cloudflare";
|
export type RemoteAccessProvider = "tailscale" | "cloudflare";
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import {
|
|||||||
fetchCustomProviders,
|
fetchCustomProviders,
|
||||||
updateCustomProvider,
|
updateCustomProvider,
|
||||||
type CustomProvider,
|
type CustomProvider,
|
||||||
type CustomProviderConfig,
|
|
||||||
} from "../api";
|
} from "../api";
|
||||||
import { AlertCircle, Loader2, Pencil, Plus, Trash2 } from "lucide-react";
|
import { AlertCircle, Loader2, Pencil, Plus, Trash2 } from "lucide-react";
|
||||||
import { OnboardingDisclosure } from "./OnboardingDisclosure";
|
import { OnboardingDisclosure } from "./OnboardingDisclosure";
|
||||||
|
|||||||
@@ -64,6 +64,25 @@ export {
|
|||||||
type ResearchStepRunnerOptions,
|
type ResearchStepRunnerOptions,
|
||||||
type ResearchStepResult,
|
type ResearchStepResult,
|
||||||
} from "./research-step-runner.js";
|
} from "./research-step-runner.js";
|
||||||
|
export { ResearchProviderRegistry } from "./research/provider-registry.js";
|
||||||
|
export {
|
||||||
|
ResearchProviderError,
|
||||||
|
type ResearchProviderType,
|
||||||
|
type ResearchProviderConfig,
|
||||||
|
type ResearchProviderErrorCode,
|
||||||
|
type ResearchFetchResult,
|
||||||
|
} from "./research/types.js";
|
||||||
|
export {
|
||||||
|
WebSearchProvider,
|
||||||
|
type WebSearchProviderOptions,
|
||||||
|
PageFetchProvider,
|
||||||
|
type PageFetchProviderOptions,
|
||||||
|
GitHubProvider,
|
||||||
|
LocalDocsProvider,
|
||||||
|
type LocalDocsProviderOptions,
|
||||||
|
LLMSynthesisProvider,
|
||||||
|
type LLMSynthesisProviderOptions,
|
||||||
|
} from "./research/providers/index.js";
|
||||||
export { PrMonitor, type PrComment, type TrackedPr, type OnNewCommentsCallback } from "./pr-monitor.js";
|
export { PrMonitor, type PrComment, type TrackedPr, type OnNewCommentsCallback } from "./pr-monitor.js";
|
||||||
export { PrCommentHandler } from "./pr-comment-handler.js";
|
export { PrCommentHandler } from "./pr-comment-handler.js";
|
||||||
export {
|
export {
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { ResearchProviderRegistry } from "../provider-registry.js";
|
||||||
|
|
||||||
|
describe("ResearchProviderRegistry", () => {
|
||||||
|
it("instantiates providers with defaults", () => {
|
||||||
|
const registry = new ResearchProviderRegistry({}, process.cwd());
|
||||||
|
expect(registry.getProvider("web-search")).toBeDefined();
|
||||||
|
expect(registry.getProvider("page-fetch")).toBeDefined();
|
||||||
|
expect(registry.isProviderAvailable("local-docs")).toBe(true);
|
||||||
|
expect(registry.isProviderAvailable("github")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("detects search backend from credentials", () => {
|
||||||
|
const tavily = new ResearchProviderRegistry({ researchTavilyApiKey: "key" }, process.cwd());
|
||||||
|
expect(tavily.isProviderAvailable("web-search")).toBe(true);
|
||||||
|
|
||||||
|
const searx = new ResearchProviderRegistry({ researchSearxngUrl: "https://sx.local" }, process.cwd());
|
||||||
|
expect(searx.isProviderAvailable("web-search")).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns only configured providers in available list", () => {
|
||||||
|
const registry = new ResearchProviderRegistry(
|
||||||
|
{
|
||||||
|
researchWebSearchProvider: "brave",
|
||||||
|
researchBraveApiKey: "token",
|
||||||
|
researchGitHubEnabled: true,
|
||||||
|
},
|
||||||
|
process.cwd(),
|
||||||
|
);
|
||||||
|
|
||||||
|
const available = registry.getAvailableProviders();
|
||||||
|
expect(available).toContain("web-search");
|
||||||
|
expect(available).toContain("local-docs");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refreshes providers after settings changes", () => {
|
||||||
|
const registry = new ResearchProviderRegistry({ researchWebSearchProvider: "none" }, process.cwd());
|
||||||
|
expect(registry.isProviderAvailable("web-search")).toBe(false);
|
||||||
|
|
||||||
|
registry.refreshSettings({ researchWebSearchProvider: "tavily", researchTavilyApiKey: "key" });
|
||||||
|
expect(registry.isProviderAvailable("web-search")).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("gracefully degrades disabled providers", () => {
|
||||||
|
const registry = new ResearchProviderRegistry({ researchGitHubEnabled: false, researchLocalDocsEnabled: false }, process.cwd());
|
||||||
|
expect(registry.isProviderAvailable("github")).toBe(false);
|
||||||
|
expect(registry.isProviderAvailable("local-docs")).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
110
packages/engine/src/research/provider-registry.ts
Normal file
110
packages/engine/src/research/provider-registry.ts
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
import type { GlobalSettings, ProjectSettings, WebSearchBackend } from "@fusion/core";
|
||||||
|
import { createLogger } from "../logger.js";
|
||||||
|
import type { ResearchProvider } from "../research-step-runner.js";
|
||||||
|
import type { ResearchProviderType } from "./types.js";
|
||||||
|
import { GitHubProvider } from "./providers/github-provider.js";
|
||||||
|
import { LLMSynthesisProvider } from "./providers/llm-synthesis-provider.js";
|
||||||
|
import { LocalDocsProvider } from "./providers/local-docs-provider.js";
|
||||||
|
import { PageFetchProvider } from "./providers/page-fetch-provider.js";
|
||||||
|
import { WebSearchProvider } from "./providers/web-search-provider.js";
|
||||||
|
|
||||||
|
const log = createLogger("research:provider-registry");
|
||||||
|
|
||||||
|
type SettingsLike = Partial<GlobalSettings & ProjectSettings>;
|
||||||
|
|
||||||
|
export class ResearchProviderRegistry {
|
||||||
|
private providers = new Map<ResearchProviderType, ResearchProvider>();
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private settings: SettingsLike,
|
||||||
|
private readonly projectRoot: string,
|
||||||
|
) {
|
||||||
|
this.instantiateProviders();
|
||||||
|
}
|
||||||
|
|
||||||
|
getProvider(type: ResearchProviderType): ResearchProvider | undefined {
|
||||||
|
return this.providers.get(type);
|
||||||
|
}
|
||||||
|
|
||||||
|
getAvailableProviders(): ResearchProviderType[] {
|
||||||
|
return [...this.providers.entries()]
|
||||||
|
.filter(([, provider]) => provider.isConfigured())
|
||||||
|
.map(([type]) => type);
|
||||||
|
}
|
||||||
|
|
||||||
|
isProviderAvailable(type: ResearchProviderType): boolean {
|
||||||
|
const provider = this.providers.get(type);
|
||||||
|
return Boolean(provider?.isConfigured());
|
||||||
|
}
|
||||||
|
|
||||||
|
refreshSettings(settings: SettingsLike): void {
|
||||||
|
this.settings = settings;
|
||||||
|
this.instantiateProviders();
|
||||||
|
}
|
||||||
|
|
||||||
|
private instantiateProviders(): void {
|
||||||
|
const backend = this.resolveSearchBackend();
|
||||||
|
const maxResults = Number(this.settings.researchMaxSearchResults ?? 10);
|
||||||
|
const fetchTimeoutMs = Number(this.settings.researchFetchTimeoutMs ?? 30_000);
|
||||||
|
const userAgent = this.settings.researchUserAgent ?? "FusionResearchBot/1.0";
|
||||||
|
|
||||||
|
this.providers = new Map<ResearchProviderType, ResearchProvider>([
|
||||||
|
[
|
||||||
|
"web-search",
|
||||||
|
new WebSearchProvider({
|
||||||
|
backend,
|
||||||
|
searxngUrl: this.settings.researchSearxngUrl,
|
||||||
|
braveApiKey: this.settings.researchBraveApiKey,
|
||||||
|
googleApiKey: this.settings.researchGoogleSearchApiKey,
|
||||||
|
googleCx: this.settings.researchGoogleSearchCx,
|
||||||
|
tavilyApiKey: this.settings.researchTavilyApiKey,
|
||||||
|
maxResults,
|
||||||
|
timeoutMs: fetchTimeoutMs,
|
||||||
|
userAgent,
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
["page-fetch", new PageFetchProvider({ timeoutMs: fetchTimeoutMs, userAgent })],
|
||||||
|
["github", this.settings.researchGitHubEnabled ? new GitHubProvider() : new DisabledProvider("github")],
|
||||||
|
[
|
||||||
|
"local-docs",
|
||||||
|
this.settings.researchLocalDocsEnabled === false
|
||||||
|
? new DisabledProvider("local-docs")
|
||||||
|
: new LocalDocsProvider({ projectRoot: this.projectRoot, timeoutMs: fetchTimeoutMs, maxResults }),
|
||||||
|
],
|
||||||
|
["llm-synthesis", new LLMSynthesisProvider({ projectRoot: this.projectRoot })],
|
||||||
|
]);
|
||||||
|
|
||||||
|
log.log("providers refreshed", { available: this.getAvailableProviders(), backend });
|
||||||
|
}
|
||||||
|
|
||||||
|
private resolveSearchBackend(): WebSearchBackend {
|
||||||
|
const explicit = this.settings.researchWebSearchProvider;
|
||||||
|
if (explicit && explicit !== "none") return explicit;
|
||||||
|
|
||||||
|
if (this.settings.researchSearxngUrl) return "searxng";
|
||||||
|
if (this.settings.researchTavilyApiKey) return "tavily";
|
||||||
|
if (this.settings.researchBraveApiKey) return "brave";
|
||||||
|
if (this.settings.researchGoogleSearchApiKey && this.settings.researchGoogleSearchCx) return "google";
|
||||||
|
return "none";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class DisabledProvider implements ResearchProvider {
|
||||||
|
readonly type: string;
|
||||||
|
|
||||||
|
constructor(type: string) {
|
||||||
|
this.type = type;
|
||||||
|
}
|
||||||
|
|
||||||
|
isConfigured(): boolean {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
async search(): Promise<[]> {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
async fetchContent(): Promise<{ content: string; metadata: Record<string, unknown> }> {
|
||||||
|
return { content: "", metadata: {} };
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,143 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
|
||||||
|
const { runGhAsyncMock, runGhJsonAsyncMock, isGhAvailableMock, isGhAuthenticatedMock } = vi.hoisted(() => ({
|
||||||
|
runGhAsyncMock: vi.fn(),
|
||||||
|
runGhJsonAsyncMock: vi.fn(),
|
||||||
|
isGhAvailableMock: vi.fn(),
|
||||||
|
isGhAuthenticatedMock: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("@fusion/core", async () => {
|
||||||
|
const actual = await vi.importActual<typeof import("@fusion/core")>("@fusion/core");
|
||||||
|
return {
|
||||||
|
...actual,
|
||||||
|
runGhAsync: runGhAsyncMock,
|
||||||
|
runGhJsonAsync: runGhJsonAsyncMock,
|
||||||
|
isGhAvailable: isGhAvailableMock,
|
||||||
|
isGhAuthenticated: isGhAuthenticatedMock,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
import { GitHubProvider } from "../github-provider.js";
|
||||||
|
|
||||||
|
describe("GitHubProvider", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
isGhAvailableMock.mockReturnValue(true);
|
||||||
|
isGhAuthenticatedMock.mockReturnValue(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("searches repositories", async () => {
|
||||||
|
runGhJsonAsyncMock.mockResolvedValueOnce([
|
||||||
|
{ fullName: "org/repo", description: "desc", htmlUrl: "https://github.com/org/repo", stargazersCount: 12, language: "ts", updatedAt: "2026" },
|
||||||
|
]);
|
||||||
|
const provider = new GitHubProvider();
|
||||||
|
|
||||||
|
const results = await provider.search("query", { metadata: { searchType: "repos" } });
|
||||||
|
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
expect(results[0]).toMatchObject({ type: "github", title: "org/repo", reference: "https://github.com/org/repo" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("searches issues", async () => {
|
||||||
|
runGhJsonAsyncMock.mockResolvedValueOnce([
|
||||||
|
{ title: "Issue", body: "body", htmlUrl: "https://github.com/org/repo/issues/1", state: "open", labels: [{ name: "bug" }] },
|
||||||
|
]);
|
||||||
|
|
||||||
|
const provider = new GitHubProvider();
|
||||||
|
const results = await provider.search("query", { metadata: { searchType: "issues" } });
|
||||||
|
|
||||||
|
expect(results[0]?.metadata).toMatchObject({ resultType: "issue", state: "open", labels: ["bug"] });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("supports combined search", async () => {
|
||||||
|
runGhJsonAsyncMock
|
||||||
|
.mockResolvedValueOnce([{ fullName: "org/repo", htmlUrl: "https://github.com/org/repo" }])
|
||||||
|
.mockResolvedValueOnce([{ title: "Issue", htmlUrl: "https://github.com/org/repo/issues/1" }]);
|
||||||
|
|
||||||
|
const provider = new GitHubProvider();
|
||||||
|
const results = await provider.search("query", {});
|
||||||
|
expect(results).toHaveLength(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fetches repo README", async () => {
|
||||||
|
runGhJsonAsyncMock.mockResolvedValueOnce({
|
||||||
|
content: Buffer.from("# Hello").toString("base64"),
|
||||||
|
encoding: "base64",
|
||||||
|
name: "README.md",
|
||||||
|
});
|
||||||
|
|
||||||
|
const provider = new GitHubProvider();
|
||||||
|
const result = await provider.fetchContent("https://github.com/org/repo", {});
|
||||||
|
|
||||||
|
expect(result.content).toContain("# Hello");
|
||||||
|
expect(result.metadata).toMatchObject({ kind: "repo-readme" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fetches issue content", async () => {
|
||||||
|
runGhAsyncMock.mockResolvedValueOnce("Issue body\nComments");
|
||||||
|
|
||||||
|
const provider = new GitHubProvider();
|
||||||
|
const result = await provider.fetchContent("https://github.com/org/repo/issues/123", {});
|
||||||
|
|
||||||
|
expect(result.content).toContain("Issue body");
|
||||||
|
expect(result.metadata).toMatchObject({ kind: "issue", number: "123" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fetches pr content", async () => {
|
||||||
|
runGhAsyncMock.mockResolvedValueOnce("PR body\nComments");
|
||||||
|
|
||||||
|
const provider = new GitHubProvider();
|
||||||
|
const result = await provider.fetchContent("https://github.com/org/repo/pull/9", {});
|
||||||
|
expect(result.metadata).toMatchObject({ kind: "pr", number: "9" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fetches file content from blob url", async () => {
|
||||||
|
runGhJsonAsyncMock.mockResolvedValueOnce({
|
||||||
|
content: Buffer.from("file content").toString("base64"),
|
||||||
|
encoding: "base64",
|
||||||
|
name: "index.ts",
|
||||||
|
});
|
||||||
|
|
||||||
|
const provider = new GitHubProvider();
|
||||||
|
const result = await provider.fetchContent("https://github.com/org/repo/blob/main/src/index.ts", {});
|
||||||
|
expect(result.content).toContain("file content");
|
||||||
|
expect(result.metadata).toMatchObject({ kind: "file", path: "src/index.ts" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports configuration state", () => {
|
||||||
|
const provider = new GitHubProvider();
|
||||||
|
expect(provider.isConfigured()).toBe(true);
|
||||||
|
|
||||||
|
isGhAvailableMock.mockReturnValue(false);
|
||||||
|
expect(provider.isConfigured()).toBe(false);
|
||||||
|
|
||||||
|
isGhAvailableMock.mockReturnValue(true);
|
||||||
|
isGhAuthenticatedMock.mockReturnValue(false);
|
||||||
|
expect(provider.isConfigured()).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps abort and timeout errors", async () => {
|
||||||
|
runGhJsonAsyncMock.mockRejectedValueOnce(Object.assign(new Error("gh command aborted"), { code: "ABORT_ERR", stderr: "", stdout: "" }));
|
||||||
|
const provider = new GitHubProvider();
|
||||||
|
await expect(provider.search("q", {})).rejects.toMatchObject({ code: "abort" });
|
||||||
|
|
||||||
|
runGhJsonAsyncMock.mockRejectedValueOnce(Object.assign(new Error("gh command timed out after 30000ms"), { code: null, stderr: "", stdout: "" }));
|
||||||
|
await expect(provider.search("q", {})).rejects.toMatchObject({ code: "timeout" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps rate limit and auth failures", async () => {
|
||||||
|
const provider = new GitHubProvider();
|
||||||
|
|
||||||
|
runGhJsonAsyncMock.mockRejectedValueOnce(Object.assign(new Error("API rate limit exceeded"), { code: 403, stderr: "", stdout: "" }));
|
||||||
|
await expect(provider.search("q", {})).rejects.toMatchObject({ code: "rate-limited", retryable: true });
|
||||||
|
|
||||||
|
runGhJsonAsyncMock.mockRejectedValueOnce(Object.assign(new Error("authentication required"), { code: 401, stderr: "", stdout: "" }));
|
||||||
|
await expect(provider.search("q", {})).rejects.toMatchObject({ code: "auth-failed" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("errors on unsupported urls", async () => {
|
||||||
|
const provider = new GitHubProvider();
|
||||||
|
await expect(provider.fetchContent("https://example.com/a", {})).rejects.toMatchObject({ code: "provider-unavailable" });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
|
||||||
|
const { createFnAgentMock, promptWithFallbackMock, disposeMock } = vi.hoisted(() => ({
|
||||||
|
disposeMock: vi.fn(),
|
||||||
|
createFnAgentMock: vi.fn(),
|
||||||
|
promptWithFallbackMock: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("../../../pi.js", () => ({
|
||||||
|
createFnAgent: createFnAgentMock,
|
||||||
|
promptWithFallback: promptWithFallbackMock,
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { LLMSynthesisProvider, buildSynthesisPrompt, extractCitations } from "../llm-synthesis-provider.js";
|
||||||
|
|
||||||
|
describe("LLMSynthesisProvider", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
const session = { state: { messages: [] as Array<{ role: string; content: string }> }, dispose: disposeMock };
|
||||||
|
createFnAgentMock.mockResolvedValue({ session });
|
||||||
|
promptWithFallbackMock.mockImplementation(async (s: typeof session, _prompt: string) => {
|
||||||
|
s.state.messages.push({ role: "assistant", content: '```json\n{"summary":"ok","confidence":0.8}\n```\n[1]' });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("builds synthesis output with citations", async () => {
|
||||||
|
const provider = new LLMSynthesisProvider({ projectRoot: "/tmp" });
|
||||||
|
const result = await provider.synthesize(
|
||||||
|
{
|
||||||
|
query: "q",
|
||||||
|
round: 1,
|
||||||
|
sources: [{ id: "1", type: "web", reference: "https://a", status: "completed", title: "A", content: "hello" }],
|
||||||
|
},
|
||||||
|
{ provider: "openai", modelId: "gpt-5" },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(promptWithFallbackMock).toHaveBeenCalled();
|
||||||
|
expect(result.citations).toEqual(["https://a"]);
|
||||||
|
expect(result.confidence).toBe(0.8);
|
||||||
|
expect(disposeMock).toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("includes query and sources in prompt", () => {
|
||||||
|
const prompt = buildSynthesisPrompt(
|
||||||
|
{ query: "query", round: 2, sources: [{ id: "s", type: "web", reference: "https://x", status: "completed", title: "Title", content: "Body" }] },
|
||||||
|
[{ id: "s", type: "web", reference: "https://x", status: "completed", title: "Title", content: "Body" }],
|
||||||
|
);
|
||||||
|
expect(prompt).toContain("Query: query");
|
||||||
|
expect(prompt).toContain("Reference: https://x");
|
||||||
|
expect(prompt).toContain("Return valid JSON");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("extracts citations by index", () => {
|
||||||
|
const citations = extractCitations("Findings [1] and [2]", [
|
||||||
|
{ id: "1", type: "web", reference: "a", status: "completed" },
|
||||||
|
{ id: "2", type: "web", reference: "b", status: "completed" },
|
||||||
|
]);
|
||||||
|
expect(citations).toEqual(["a", "b"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps abort and timeout", async () => {
|
||||||
|
const provider = new LLMSynthesisProvider({ projectRoot: "/tmp", timeoutMs: 20 });
|
||||||
|
promptWithFallbackMock.mockImplementation(async () => {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||||
|
});
|
||||||
|
await expect(
|
||||||
|
provider.synthesize({ query: "q", round: 1, sources: [] }, { provider: "openai", modelId: "gpt-4o" }),
|
||||||
|
).rejects.toMatchObject({ code: "timeout" });
|
||||||
|
|
||||||
|
promptWithFallbackMock.mockImplementation(async () => {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 80));
|
||||||
|
});
|
||||||
|
const controller = new AbortController();
|
||||||
|
const pending = provider.synthesize({ query: "q", round: 1, sources: [] }, { provider: "openai", modelId: "gpt-4o" }, controller.signal);
|
||||||
|
controller.abort();
|
||||||
|
await expect(pending).rejects.toMatchObject({ code: "abort" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports provider availability", () => {
|
||||||
|
const provider = new LLMSynthesisProvider({ projectRoot: "/tmp" });
|
||||||
|
expect(provider.isConfigured()).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("truncates sources to character budget", async () => {
|
||||||
|
const provider = new LLMSynthesisProvider({ projectRoot: "/tmp" });
|
||||||
|
const manySources = Array.from({ length: 30 }, (_, i) => ({
|
||||||
|
id: String(i),
|
||||||
|
type: "web" as const,
|
||||||
|
reference: `https://src/${i}`,
|
||||||
|
status: "completed" as const,
|
||||||
|
content: "x".repeat(8000),
|
||||||
|
metadata: { confidence: i },
|
||||||
|
}));
|
||||||
|
|
||||||
|
await provider.synthesize({ query: "q", round: 1, sources: manySources }, { provider: "openai", modelId: "small-model" });
|
||||||
|
const promptArg = promptWithFallbackMock.mock.calls[0]?.[1] as string;
|
||||||
|
expect(promptArg.length).toBeLessThan(35_000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps model failures", async () => {
|
||||||
|
const provider = new LLMSynthesisProvider({ projectRoot: "/tmp" });
|
||||||
|
promptWithFallbackMock.mockRejectedValueOnce(new Error("model down"));
|
||||||
|
await expect(provider.synthesize({ query: "q", round: 1, sources: [] }, { provider: "openai", modelId: "gpt-4o" })).rejects.toMatchObject({
|
||||||
|
code: "provider-unavailable",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import os from "node:os";
|
||||||
|
import { afterEach, describe, expect, it } from "vitest";
|
||||||
|
import { LocalDocsProvider } from "../local-docs-provider.js";
|
||||||
|
|
||||||
|
describe("LocalDocsProvider", () => {
|
||||||
|
const tempDirs: string[] = [];
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
for (const dir of tempDirs) rmSync(dir, { recursive: true, force: true });
|
||||||
|
tempDirs.length = 0;
|
||||||
|
});
|
||||||
|
|
||||||
|
function makeProject() {
|
||||||
|
const root = mkdtempSync(join(os.tmpdir(), "fn-test-research-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
mkdirSync(join(root, "docs"), { recursive: true });
|
||||||
|
writeFileSync(join(root, "README.md"), "Fusion research provider docs");
|
||||||
|
writeFileSync(join(root, "docs", "guide.md"), "This guide covers provider architecture and confidence scoring");
|
||||||
|
return root;
|
||||||
|
}
|
||||||
|
|
||||||
|
it("searches keywords across docs", async () => {
|
||||||
|
const root = makeProject();
|
||||||
|
const provider = new LocalDocsProvider({ projectRoot: root });
|
||||||
|
|
||||||
|
const results = await provider.search("provider confidence", {});
|
||||||
|
expect(results.length).toBeGreaterThan(0);
|
||||||
|
expect(results[0]?.reference).toContain("docs/guide.md");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fetches local file content", async () => {
|
||||||
|
const root = makeProject();
|
||||||
|
const provider = new LocalDocsProvider({ projectRoot: root });
|
||||||
|
|
||||||
|
const result = await provider.fetchContent("README.md", {});
|
||||||
|
expect(result.content).toContain("Fusion research");
|
||||||
|
expect(result.metadata).toMatchObject({ extension: ".md" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("prevents path traversal", async () => {
|
||||||
|
const root = makeProject();
|
||||||
|
const provider = new LocalDocsProvider({ projectRoot: root });
|
||||||
|
await expect(provider.fetchContent("../../etc/passwd", {})).rejects.toMatchObject({ code: "provider-unavailable" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips binary files during search", async () => {
|
||||||
|
const root = makeProject();
|
||||||
|
writeFileSync(join(root, "docs", "binary.dat"), Buffer.from([0, 1, 2, 3, 4]));
|
||||||
|
const provider = new LocalDocsProvider({ projectRoot: root });
|
||||||
|
|
||||||
|
const results = await provider.search("binary", {});
|
||||||
|
expect(results.find((entry) => String(entry.reference).includes("binary.dat"))).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips oversized files during search", async () => {
|
||||||
|
const root = makeProject();
|
||||||
|
writeFileSync(join(root, "docs", "large.md"), "x".repeat(1024 * 1024 + 128));
|
||||||
|
const provider = new LocalDocsProvider({ projectRoot: root });
|
||||||
|
|
||||||
|
const results = await provider.search("xxxx", {});
|
||||||
|
expect(results.find((entry) => String(entry.reference).includes("large.md"))).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("supports abort signal", async () => {
|
||||||
|
const root = makeProject();
|
||||||
|
for (let i = 0; i < 20; i += 1) {
|
||||||
|
writeFileSync(join(root, "docs", `doc-${i}.md`), `line ${i}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const provider = new LocalDocsProvider({ projectRoot: root });
|
||||||
|
const controller = new AbortController();
|
||||||
|
const promise = provider.search("line", {}, controller.signal);
|
||||||
|
controller.abort();
|
||||||
|
|
||||||
|
await expect(promise).rejects.toMatchObject({ code: "abort" });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { MAX_CONTENT_CHARS, PageFetchProvider } from "../page-fetch-provider.js";
|
||||||
|
|
||||||
|
describe("PageFetchProvider", () => {
|
||||||
|
const originalFetch = global.fetch;
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
global.fetch = originalFetch;
|
||||||
|
});
|
||||||
|
|
||||||
|
it("extracts readable html content", async () => {
|
||||||
|
global.fetch = vi.fn().mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
headers: new Headers({ "content-type": "text/html" }),
|
||||||
|
text: async () =>
|
||||||
|
"<html><head><title>Hello</title><meta name='description' content='Desc'></head><body><main><h1>Title</h1><p>Body</p></main></body></html>",
|
||||||
|
} as Response);
|
||||||
|
|
||||||
|
const provider = new PageFetchProvider();
|
||||||
|
const result = await provider.fetchContent("https://example.com", {});
|
||||||
|
|
||||||
|
expect(result.content).toContain("Title Body");
|
||||||
|
expect(result.metadata).toMatchObject({ title: "Hello", description: "Desc", contentType: "text/html" });
|
||||||
|
expect(result.mimeType).toBe("text/html");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("pretty prints json", async () => {
|
||||||
|
global.fetch = vi.fn().mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
headers: new Headers({ "content-type": "application/json" }),
|
||||||
|
text: async () => '{"a":1}',
|
||||||
|
} as Response);
|
||||||
|
|
||||||
|
const provider = new PageFetchProvider();
|
||||||
|
const result = await provider.fetchContent("https://example.com", {});
|
||||||
|
expect(result.content).toContain('"a": 1');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("passes through text", async () => {
|
||||||
|
global.fetch = vi.fn().mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
headers: new Headers({ "content-type": "text/plain" }),
|
||||||
|
text: async () => "hello world",
|
||||||
|
} as Response);
|
||||||
|
|
||||||
|
const provider = new PageFetchProvider();
|
||||||
|
const result = await provider.fetchContent("https://example.com", {});
|
||||||
|
expect(result.content).toBe("hello world");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps status errors", async () => {
|
||||||
|
global.fetch = vi.fn().mockResolvedValue({ ok: false, status: 404 } as Response);
|
||||||
|
const provider = new PageFetchProvider();
|
||||||
|
await expect(provider.fetchContent("https://example.com", {})).rejects.toMatchObject({ code: "network-error" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps 500 as provider unavailable", async () => {
|
||||||
|
global.fetch = vi.fn().mockResolvedValue({ ok: false, status: 500 } as Response);
|
||||||
|
const provider = new PageFetchProvider();
|
||||||
|
await expect(provider.fetchContent("https://example.com", {})).rejects.toMatchObject({ code: "provider-unavailable" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("truncates large content", async () => {
|
||||||
|
global.fetch = vi.fn().mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
headers: new Headers({ "content-type": "text/plain" }),
|
||||||
|
text: async () => "x".repeat(600 * 1024),
|
||||||
|
} as Response);
|
||||||
|
const provider = new PageFetchProvider();
|
||||||
|
const result = await provider.fetchContent("https://example.com", {});
|
||||||
|
expect(result.content.length).toBe(MAX_CONTENT_CHARS);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps timeout", async () => {
|
||||||
|
global.fetch = vi.fn().mockRejectedValue(Object.assign(new Error("Timed out"), { name: "TimeoutError" }));
|
||||||
|
const provider = new PageFetchProvider({ timeoutMs: 50 });
|
||||||
|
await expect(provider.fetchContent("https://example.com", {})).rejects.toMatchObject({ code: "timeout" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps abort", async () => {
|
||||||
|
global.fetch = vi.fn().mockImplementation(async (_url, init: RequestInit) => {
|
||||||
|
await new Promise((_, reject) => init.signal?.addEventListener("abort", () => reject(new DOMException("Aborted", "AbortError"))));
|
||||||
|
return { ok: true, headers: new Headers(), text: async () => "" } as Response;
|
||||||
|
});
|
||||||
|
const provider = new PageFetchProvider();
|
||||||
|
const controller = new AbortController();
|
||||||
|
const pending = provider.fetchContent("https://example.com", {}, controller.signal);
|
||||||
|
controller.abort();
|
||||||
|
await expect(pending).rejects.toMatchObject({ code: "abort" });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { ResearchProviderError } from "../../types.js";
|
||||||
|
import { WebSearchProvider } from "../web-search-provider.js";
|
||||||
|
|
||||||
|
describe("WebSearchProvider", () => {
|
||||||
|
const originalFetch = global.fetch;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
global.fetch = originalFetch;
|
||||||
|
});
|
||||||
|
|
||||||
|
it("validates configuration for each backend", () => {
|
||||||
|
expect(new WebSearchProvider({ backend: "none" }).isConfigured()).toBe(false);
|
||||||
|
expect(new WebSearchProvider({ backend: "searxng", searxngUrl: "https://sx" }).isConfigured()).toBe(true);
|
||||||
|
expect(new WebSearchProvider({ backend: "brave", braveApiKey: "k" }).isConfigured()).toBe(true);
|
||||||
|
expect(new WebSearchProvider({ backend: "google", googleApiKey: "k", googleCx: "cx" }).isConfigured()).toBe(true);
|
||||||
|
expect(new WebSearchProvider({ backend: "tavily", tavilyApiKey: "k" }).isConfigured()).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("normalizes searxng results", async () => {
|
||||||
|
global.fetch = vi.fn().mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({ results: [{ url: "https://a", title: "A", content: "Snippet" }] }),
|
||||||
|
} as Response);
|
||||||
|
|
||||||
|
const provider = new WebSearchProvider({ backend: "searxng", searxngUrl: "https://sx" });
|
||||||
|
const results = await provider.search("fusion", {});
|
||||||
|
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
expect(results[0]).toMatchObject({
|
||||||
|
title: "A",
|
||||||
|
reference: "https://a",
|
||||||
|
excerpt: "Snippet",
|
||||||
|
metadata: { backend: "searxng", rank: 1 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("limits max results", async () => {
|
||||||
|
global.fetch = vi.fn().mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({
|
||||||
|
web: {
|
||||||
|
results: [
|
||||||
|
{ url: "https://a", title: "A", description: "1" },
|
||||||
|
{ url: "https://b", title: "B", description: "2" },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
} as Response);
|
||||||
|
|
||||||
|
const provider = new WebSearchProvider({ backend: "brave", braveApiKey: "k" });
|
||||||
|
const results = await provider.search("fusion", { maxResults: 1 });
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("retries 429 and succeeds", async () => {
|
||||||
|
global.fetch = vi
|
||||||
|
.fn()
|
||||||
|
.mockResolvedValueOnce({ ok: false, status: 429 } as Response)
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({ items: [{ link: "https://x", title: "X", snippet: "S" }] }),
|
||||||
|
} as Response);
|
||||||
|
|
||||||
|
const provider = new WebSearchProvider({ backend: "google", googleApiKey: "k", googleCx: "cx" });
|
||||||
|
const promise = provider.search("fusion", {});
|
||||||
|
await vi.runAllTimersAsync();
|
||||||
|
const results = await promise;
|
||||||
|
|
||||||
|
expect(global.fetch).toHaveBeenCalledTimes(2);
|
||||||
|
expect(results[0]?.reference).toBe("https://x");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("classifies auth failures", async () => {
|
||||||
|
global.fetch = vi.fn().mockResolvedValue({ ok: false, status: 401 } as Response);
|
||||||
|
const provider = new WebSearchProvider({ backend: "tavily", tavilyApiKey: "k" });
|
||||||
|
await expect(provider.search("fusion", {})).rejects.toMatchObject({ code: "auth-failed" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("propagates abort signal", async () => {
|
||||||
|
global.fetch = vi.fn().mockImplementation(async (_url, init: RequestInit) => {
|
||||||
|
await new Promise((_, reject) => {
|
||||||
|
init.signal?.addEventListener("abort", () => reject(new DOMException("Aborted", "AbortError")));
|
||||||
|
});
|
||||||
|
return { ok: true, json: async () => ({}) } as Response;
|
||||||
|
});
|
||||||
|
|
||||||
|
const controller = new AbortController();
|
||||||
|
const provider = new WebSearchProvider({ backend: "brave", braveApiKey: "k" });
|
||||||
|
const pending = provider.search("fusion", {}, controller.signal);
|
||||||
|
controller.abort();
|
||||||
|
await expect(pending).rejects.toMatchObject({ code: "abort", providerType: "web-search" });
|
||||||
|
});
|
||||||
|
|
||||||
|
});
|
||||||
327
packages/engine/src/research/providers/github-provider.ts
Normal file
327
packages/engine/src/research/providers/github-provider.ts
Normal file
@@ -0,0 +1,327 @@
|
|||||||
|
import {
|
||||||
|
isGhAuthenticated,
|
||||||
|
isGhAvailable,
|
||||||
|
runGhAsync,
|
||||||
|
runGhJsonAsync,
|
||||||
|
type GhError,
|
||||||
|
type ResearchProviderConfig,
|
||||||
|
type ResearchSource,
|
||||||
|
} from "@fusion/core";
|
||||||
|
import type { ResearchProvider } from "../../research-step-runner.js";
|
||||||
|
import { createLogger } from "../../logger.js";
|
||||||
|
import { ResearchProviderError, type ResearchFetchResult } from "../types.js";
|
||||||
|
|
||||||
|
const log = createLogger("research:github");
|
||||||
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||||
|
|
||||||
|
type SearchType = "repos" | "issues" | "both";
|
||||||
|
|
||||||
|
interface GitHubRepoResult {
|
||||||
|
fullName: string;
|
||||||
|
description?: string;
|
||||||
|
htmlUrl: string;
|
||||||
|
stargazersCount?: number;
|
||||||
|
language?: string;
|
||||||
|
updatedAt?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface GitHubIssueResult {
|
||||||
|
title: string;
|
||||||
|
body?: string;
|
||||||
|
htmlUrl: string;
|
||||||
|
state?: string;
|
||||||
|
labels?: Array<{ name?: string }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ParsedGitHubUrl {
|
||||||
|
owner: string;
|
||||||
|
repo: string;
|
||||||
|
kind: "repo" | "issue" | "pr" | "file";
|
||||||
|
number?: string;
|
||||||
|
filePath?: string;
|
||||||
|
ref?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class GitHubProvider implements ResearchProvider {
|
||||||
|
readonly type = "github";
|
||||||
|
|
||||||
|
async search(query: string, config: ResearchProviderConfig = {}, signal?: AbortSignal): Promise<ResearchSource[]> {
|
||||||
|
const timeoutMs = Number(config.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||||
|
const maxResults = Number(config.maxResults ?? 10);
|
||||||
|
const searchType = ((config.metadata?.searchType as SearchType | undefined) ?? "both");
|
||||||
|
|
||||||
|
const sources: ResearchSource[] = [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (searchType === "repos" || searchType === "both") {
|
||||||
|
const repos = await runGhJsonAsync<GitHubRepoResult[]>(
|
||||||
|
[
|
||||||
|
"search",
|
||||||
|
"repos",
|
||||||
|
query,
|
||||||
|
"--json",
|
||||||
|
"fullName,description,htmlUrl,stargazersCount,language,updatedAt",
|
||||||
|
"--limit",
|
||||||
|
String(maxResults),
|
||||||
|
],
|
||||||
|
{ signal, timeoutMs },
|
||||||
|
);
|
||||||
|
|
||||||
|
sources.push(
|
||||||
|
...repos.slice(0, maxResults).map((repo, idx) => ({
|
||||||
|
id: `github-repo-${idx}-${repo.htmlUrl}`,
|
||||||
|
type: "github" as const,
|
||||||
|
reference: repo.htmlUrl,
|
||||||
|
title: repo.fullName,
|
||||||
|
excerpt: repo.description ?? "",
|
||||||
|
status: "completed" as const,
|
||||||
|
metadata: {
|
||||||
|
resultType: "repo",
|
||||||
|
stars: repo.stargazersCount ?? 0,
|
||||||
|
language: repo.language,
|
||||||
|
updatedAt: repo.updatedAt,
|
||||||
|
rank: idx + 1,
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (searchType === "issues" || searchType === "both") {
|
||||||
|
const issues = await runGhJsonAsync<GitHubIssueResult[]>(
|
||||||
|
[
|
||||||
|
"search",
|
||||||
|
"issues",
|
||||||
|
query,
|
||||||
|
"--json",
|
||||||
|
"title,body,htmlUrl,state,labels",
|
||||||
|
"--limit",
|
||||||
|
String(maxResults),
|
||||||
|
],
|
||||||
|
{ signal, timeoutMs },
|
||||||
|
);
|
||||||
|
|
||||||
|
sources.push(
|
||||||
|
...issues.slice(0, maxResults).map((issue, idx) => ({
|
||||||
|
id: `github-issue-${idx}-${issue.htmlUrl}`,
|
||||||
|
type: "github" as const,
|
||||||
|
reference: issue.htmlUrl,
|
||||||
|
title: issue.title,
|
||||||
|
excerpt: issue.body?.slice(0, 280) ?? "",
|
||||||
|
status: "completed" as const,
|
||||||
|
metadata: {
|
||||||
|
resultType: "issue",
|
||||||
|
state: issue.state,
|
||||||
|
labels: (issue.labels ?? []).map((label) => label.name).filter(Boolean),
|
||||||
|
rank: idx + 1,
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return sources;
|
||||||
|
} catch (error) {
|
||||||
|
throw this.mapGhError(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fetchContent(url: string, config: ResearchProviderConfig = {}, signal?: AbortSignal): Promise<ResearchFetchResult> {
|
||||||
|
const parsed = parseGitHubUrl(url);
|
||||||
|
if (!parsed) {
|
||||||
|
throw new ResearchProviderError({
|
||||||
|
providerType: "github",
|
||||||
|
code: "provider-unavailable",
|
||||||
|
message: "Unsupported GitHub URL",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const timeoutMs = Number(config.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (parsed.kind === "repo") {
|
||||||
|
const readme = await runGhJsonAsync<{ content?: string; encoding?: string; name?: string }>(
|
||||||
|
["api", `repos/${parsed.owner}/${parsed.repo}/readme`],
|
||||||
|
{ signal, timeoutMs },
|
||||||
|
);
|
||||||
|
|
||||||
|
if (readme.encoding !== "base64" || !readme.content) {
|
||||||
|
throw new ResearchProviderError({
|
||||||
|
providerType: "github",
|
||||||
|
code: "provider-unavailable",
|
||||||
|
message: "Unsupported README encoding",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const content = Buffer.from(readme.content.replace(/\n/g, ""), "base64").toString("utf-8");
|
||||||
|
return {
|
||||||
|
content,
|
||||||
|
metadata: {
|
||||||
|
url,
|
||||||
|
owner: parsed.owner,
|
||||||
|
repo: parsed.repo,
|
||||||
|
kind: "repo-readme",
|
||||||
|
name: readme.name,
|
||||||
|
},
|
||||||
|
mimeType: "text/markdown",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parsed.kind === "issue") {
|
||||||
|
const issue = await runGhAsync(["issue", "view", parsed.number ?? "", "--repo", `${parsed.owner}/${parsed.repo}`, "--comments"], {
|
||||||
|
signal,
|
||||||
|
timeoutMs,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
content: issue,
|
||||||
|
metadata: { url, owner: parsed.owner, repo: parsed.repo, kind: "issue", number: parsed.number },
|
||||||
|
mimeType: "text/plain",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parsed.kind === "pr") {
|
||||||
|
const pr = await runGhAsync(["pr", "view", parsed.number ?? "", "--repo", `${parsed.owner}/${parsed.repo}`, "--comments"], {
|
||||||
|
signal,
|
||||||
|
timeoutMs,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
content: pr,
|
||||||
|
metadata: { url, owner: parsed.owner, repo: parsed.repo, kind: "pr", number: parsed.number },
|
||||||
|
mimeType: "text/plain",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const apiPath = `repos/${parsed.owner}/${parsed.repo}/contents/${parsed.filePath ?? ""}${parsed.ref ? `?ref=${encodeURIComponent(parsed.ref)}` : ""}`;
|
||||||
|
const file = await runGhJsonAsync<{ content?: string; encoding?: string; name?: string }>(["api", apiPath], {
|
||||||
|
signal,
|
||||||
|
timeoutMs,
|
||||||
|
});
|
||||||
|
const content = file.encoding === "base64" && file.content
|
||||||
|
? Buffer.from(file.content.replace(/\n/g, ""), "base64").toString("utf-8")
|
||||||
|
: (file.content ?? "");
|
||||||
|
|
||||||
|
return {
|
||||||
|
content,
|
||||||
|
metadata: {
|
||||||
|
url,
|
||||||
|
owner: parsed.owner,
|
||||||
|
repo: parsed.repo,
|
||||||
|
kind: "file",
|
||||||
|
path: parsed.filePath,
|
||||||
|
ref: parsed.ref,
|
||||||
|
name: file.name,
|
||||||
|
},
|
||||||
|
mimeType: "text/plain",
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
throw this.mapGhError(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
isConfigured(): boolean {
|
||||||
|
return isGhAvailable() && isGhAuthenticated();
|
||||||
|
}
|
||||||
|
|
||||||
|
private mapGhError(error: unknown): ResearchProviderError {
|
||||||
|
if (error instanceof ResearchProviderError) return error;
|
||||||
|
|
||||||
|
if (isGhError(error)) {
|
||||||
|
const message = `${error.message}${error.stderr ? `: ${error.stderr}` : ""}`;
|
||||||
|
const lowered = message.toLowerCase();
|
||||||
|
|
||||||
|
if (error.code === "ABORT_ERR" || lowered.includes("aborted")) {
|
||||||
|
return new ResearchProviderError({ providerType: "github", code: "abort", message, cause: error });
|
||||||
|
}
|
||||||
|
if (lowered.includes("timed out")) {
|
||||||
|
return new ResearchProviderError({
|
||||||
|
providerType: "github",
|
||||||
|
code: "timeout",
|
||||||
|
message,
|
||||||
|
retryable: true,
|
||||||
|
cause: error,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (error.code === 403 || lowered.includes("rate limit")) {
|
||||||
|
return new ResearchProviderError({
|
||||||
|
providerType: "github",
|
||||||
|
code: "rate-limited",
|
||||||
|
message,
|
||||||
|
retryable: true,
|
||||||
|
cause: error,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (error.code === 401 || lowered.includes("not logged") || lowered.includes("authentication")) {
|
||||||
|
return new ResearchProviderError({
|
||||||
|
providerType: "github",
|
||||||
|
code: "auth-failed",
|
||||||
|
message,
|
||||||
|
cause: error,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (error.code === 404 || lowered.includes("not found")) {
|
||||||
|
return new ResearchProviderError({
|
||||||
|
providerType: "github",
|
||||||
|
code: "provider-unavailable",
|
||||||
|
message,
|
||||||
|
cause: error,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return new ResearchProviderError({
|
||||||
|
providerType: "github",
|
||||||
|
code: "network-error",
|
||||||
|
message,
|
||||||
|
retryable: true,
|
||||||
|
cause: error,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
log.warn("github provider error", { error });
|
||||||
|
return new ResearchProviderError({
|
||||||
|
providerType: "github",
|
||||||
|
code: "network-error",
|
||||||
|
message: error instanceof Error ? error.message : "GitHub provider failed",
|
||||||
|
retryable: true,
|
||||||
|
cause: error,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isGhError(value: unknown): value is GhError {
|
||||||
|
return value instanceof Error && "stderr" in value && "stdout" in value && "code" in value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseGitHubUrl(url: string): ParsedGitHubUrl | null {
|
||||||
|
let parsedUrl: URL;
|
||||||
|
try {
|
||||||
|
parsedUrl = new URL(url);
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!/(^|\.)github\.com$/i.test(parsedUrl.hostname)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const segments = parsedUrl.pathname.split("/").filter(Boolean);
|
||||||
|
if (segments.length < 2) return null;
|
||||||
|
|
||||||
|
const [owner, repo, section, ...rest] = segments;
|
||||||
|
if (!section) {
|
||||||
|
return { owner, repo, kind: "repo" };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (section === "issues" && rest[0]) {
|
||||||
|
return { owner, repo, kind: "issue", number: rest[0] };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (section === "pull" && rest[0]) {
|
||||||
|
return { owner, repo, kind: "pr", number: rest[0] };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (section === "blob" && rest.length >= 2) {
|
||||||
|
return { owner, repo, kind: "file", ref: rest[0], filePath: rest.slice(1).join("/") };
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export { parseGitHubUrl };
|
||||||
5
packages/engine/src/research/providers/index.ts
Normal file
5
packages/engine/src/research/providers/index.ts
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
export { WebSearchProvider, type WebSearchProviderOptions } from "./web-search-provider.js";
|
||||||
|
export { PageFetchProvider, type PageFetchProviderOptions } from "./page-fetch-provider.js";
|
||||||
|
export { GitHubProvider } from "./github-provider.js";
|
||||||
|
export { LocalDocsProvider, type LocalDocsProviderOptions } from "./local-docs-provider.js";
|
||||||
|
export { LLMSynthesisProvider, type LLMSynthesisProviderOptions } from "./llm-synthesis-provider.js";
|
||||||
216
packages/engine/src/research/providers/llm-synthesis-provider.ts
Normal file
216
packages/engine/src/research/providers/llm-synthesis-provider.ts
Normal file
@@ -0,0 +1,216 @@
|
|||||||
|
import type { ResearchProviderConfig, ResearchSource, ResearchSynthesisRequest, ResearchSynthesisResult, ResolvedModelSelection } from "@fusion/core";
|
||||||
|
import type { ResearchProvider } from "../../research-step-runner.js";
|
||||||
|
import { createLogger } from "../../logger.js";
|
||||||
|
import { createFnAgent, promptWithFallback } from "../../pi.js";
|
||||||
|
import { ResearchProviderError } from "../types.js";
|
||||||
|
|
||||||
|
const log = createLogger("research:llm-synthesis");
|
||||||
|
const DEFAULT_TIMEOUT_MS = 120_000;
|
||||||
|
const LARGE_MODEL_CONTEXT_CHARS = 100_000;
|
||||||
|
const SMALL_MODEL_CONTEXT_CHARS = 30_000;
|
||||||
|
|
||||||
|
export interface LLMSynthesisProviderOptions {
|
||||||
|
projectRoot: string;
|
||||||
|
timeoutMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class LLMSynthesisProvider implements ResearchProvider {
|
||||||
|
readonly type = "llm-synthesis";
|
||||||
|
|
||||||
|
constructor(private readonly options: LLMSynthesisProviderOptions) {}
|
||||||
|
|
||||||
|
isConfigured(): boolean {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async search(_query: string, _options: ResearchProviderConfig = {}, _signal?: AbortSignal): Promise<ResearchSource[]> {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
async fetchContent(
|
||||||
|
_url: string,
|
||||||
|
_options: ResearchProviderConfig = {},
|
||||||
|
_signal?: AbortSignal,
|
||||||
|
): Promise<{ content: string; metadata: Record<string, unknown> }> {
|
||||||
|
return { content: "", metadata: {} };
|
||||||
|
}
|
||||||
|
|
||||||
|
async synthesize(
|
||||||
|
request: ResearchSynthesisRequest,
|
||||||
|
modelSelection: ResolvedModelSelection,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<ResearchSynthesisResult> {
|
||||||
|
if (!modelSelection?.provider || !modelSelection?.modelId) {
|
||||||
|
throw new ResearchProviderError({ providerType: "llm-synthesis", code: "provider-unavailable", message: "Synthesis model is not configured" });
|
||||||
|
}
|
||||||
|
|
||||||
|
const timeoutSignal = AbortSignal.timeout(this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||||
|
const requestSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const cappedSources = this.applySourceBudget(request.sources, modelSelection);
|
||||||
|
const prompt = buildSynthesisPrompt(request, cappedSources);
|
||||||
|
|
||||||
|
const { session } = await createFnAgent({
|
||||||
|
cwd: this.options.projectRoot,
|
||||||
|
tools: "readonly",
|
||||||
|
systemPrompt: "You synthesize research findings into concise, cited outputs.",
|
||||||
|
defaultProvider: modelSelection.provider,
|
||||||
|
defaultModelId: modelSelection.modelId,
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await Promise.race([
|
||||||
|
promptWithFallback(session, prompt),
|
||||||
|
new Promise<never>((_, reject) => {
|
||||||
|
requestSignal.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => reject(new ResearchProviderError({
|
||||||
|
providerType: "llm-synthesis",
|
||||||
|
code: signal?.aborted ? "abort" : "timeout",
|
||||||
|
message: signal?.aborted ? "Synthesis aborted" : "Synthesis timed out",
|
||||||
|
retryable: !signal?.aborted,
|
||||||
|
})),
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (requestSignal.aborted) {
|
||||||
|
throw new ResearchProviderError({ providerType: "llm-synthesis", code: signal?.aborted ? "abort" : "timeout", message: signal?.aborted ? "Synthesis aborted" : "Synthesis timed out", retryable: !signal?.aborted });
|
||||||
|
}
|
||||||
|
|
||||||
|
const responseText = extractAssistantText(session);
|
||||||
|
if (!responseText) {
|
||||||
|
throw new ResearchProviderError({ providerType: "llm-synthesis", code: "provider-unavailable", message: "No synthesis response received" });
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
output: responseText,
|
||||||
|
citations: extractCitations(responseText, cappedSources),
|
||||||
|
confidence: extractConfidence(responseText),
|
||||||
|
metadata: {
|
||||||
|
sourceCount: cappedSources.length,
|
||||||
|
truncated: cappedSources.length < request.sources.length,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
} finally {
|
||||||
|
session.dispose();
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof ResearchProviderError) throw error;
|
||||||
|
if (error instanceof DOMException && error.name === "AbortError") {
|
||||||
|
throw new ResearchProviderError({ providerType: "llm-synthesis", code: "abort", message: "Synthesis aborted", cause: error });
|
||||||
|
}
|
||||||
|
log.warn("llm synthesis failed", { error });
|
||||||
|
throw new ResearchProviderError({
|
||||||
|
providerType: "llm-synthesis",
|
||||||
|
code: "provider-unavailable",
|
||||||
|
message: error instanceof Error ? error.message : "Synthesis failed",
|
||||||
|
retryable: true,
|
||||||
|
cause: error,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private applySourceBudget(sources: ResearchSource[], modelSelection: ResolvedModelSelection): ResearchSource[] {
|
||||||
|
// Approximation: 1 token ~= 4 English chars. We cap at 80% of an estimated
|
||||||
|
// context window to preserve room for prompt/instructions and response.
|
||||||
|
const contextChars = isLargeModel(modelSelection.modelId) ? LARGE_MODEL_CONTEXT_CHARS : SMALL_MODEL_CONTEXT_CHARS;
|
||||||
|
const budget = Math.floor(contextChars * 0.8);
|
||||||
|
const ordered = [...sources].sort((a, b) => scoreSource(b) - scoreSource(a));
|
||||||
|
|
||||||
|
let total = 0;
|
||||||
|
const kept: ResearchSource[] = [];
|
||||||
|
for (const source of ordered) {
|
||||||
|
const chunk = `${source.title ?? ""}\n${source.excerpt ?? ""}\n${source.content ?? ""}`;
|
||||||
|
if (total + chunk.length > budget) continue;
|
||||||
|
total += chunk.length;
|
||||||
|
kept.push(source);
|
||||||
|
}
|
||||||
|
|
||||||
|
return kept.length > 0 ? kept : ordered.slice(0, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isLargeModel(modelId?: string): boolean {
|
||||||
|
const id = modelId?.toLowerCase() ?? "";
|
||||||
|
return id.includes("gpt-5") || id.includes("claude-opus") || id.includes("gemini-2.5-pro") || id.includes("sonnet");
|
||||||
|
}
|
||||||
|
|
||||||
|
function scoreSource(source: ResearchSource): number {
|
||||||
|
const confidence = typeof source.metadata?.confidence === "number" ? source.metadata.confidence : 0;
|
||||||
|
const hasContent = source.content ? 1 : 0;
|
||||||
|
return confidence * 10 + hasContent;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildSynthesisPrompt(request: ResearchSynthesisRequest, sources: ResearchSource[]): string {
|
||||||
|
const format = request.desiredFormat ?? "markdown";
|
||||||
|
const renderedSources = sources
|
||||||
|
.map(
|
||||||
|
(source, index) => `Source [${index + 1}]\nTitle: ${source.title ?? source.reference}\nReference: ${source.reference}\nExcerpt: ${source.excerpt ?? ""}\nContent: ${source.content ?? ""}`,
|
||||||
|
)
|
||||||
|
.join("\n\n");
|
||||||
|
|
||||||
|
return [
|
||||||
|
"You are a research synthesis assistant.",
|
||||||
|
`Query: ${request.query}`,
|
||||||
|
`Round: ${request.round}`,
|
||||||
|
`Desired format: ${format}`,
|
||||||
|
"Analyze the provided sources and produce: summary, key findings, contradictions, confidence, and follow-up queries.",
|
||||||
|
"Cite source references inline using [n] notation where n is source number.",
|
||||||
|
request.instructions ? `Additional instructions: ${request.instructions}` : "",
|
||||||
|
"Sources:",
|
||||||
|
renderedSources,
|
||||||
|
'Return valid JSON: {"summary": string, "findings": [{"statement": string, "citations": string[]}], "confidence": number, "followUps": string[]}',
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join("\n\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractAssistantText(session: { state?: { messages?: Array<{ role?: string; content?: unknown }> } }): string | undefined {
|
||||||
|
const messages = session.state?.messages;
|
||||||
|
if (!Array.isArray(messages)) return undefined;
|
||||||
|
for (let i = messages.length - 1; i >= 0; i -= 1) {
|
||||||
|
const msg = messages[i];
|
||||||
|
if (msg.role !== "assistant") continue;
|
||||||
|
if (typeof msg.content === "string" && msg.content.trim()) return msg.content;
|
||||||
|
if (Array.isArray(msg.content)) {
|
||||||
|
for (const part of msg.content) {
|
||||||
|
if (typeof part === "object" && part && "text" in part && typeof (part as { text?: unknown }).text === "string") {
|
||||||
|
return (part as { text: string }).text;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractCitations(text: string, sources: ResearchSource[]): string[] {
|
||||||
|
const matches = text.match(/\[(\d+)\]/g) ?? [];
|
||||||
|
const refs = new Set<string>();
|
||||||
|
for (const match of matches) {
|
||||||
|
const idx = Number.parseInt(match.replace(/\D/g, ""), 10) - 1;
|
||||||
|
if (Number.isFinite(idx) && idx >= 0 && idx < sources.length) {
|
||||||
|
refs.add(sources[idx].reference);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return [...refs];
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractConfidence(text: string): number | undefined {
|
||||||
|
const jsonBlock = text.match(/```(?:json)?\s*([\s\S]*?)```/i)?.[1] ?? text;
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(jsonBlock);
|
||||||
|
if (typeof parsed.confidence === "number") return parsed.confidence;
|
||||||
|
} catch {
|
||||||
|
const match = text.match(/"confidence"\s*:\s*([0-9]*\.?[0-9]+)/i);
|
||||||
|
if (match) {
|
||||||
|
const parsed = Number.parseFloat(match[1]);
|
||||||
|
if (Number.isFinite(parsed)) return parsed;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export { buildSynthesisPrompt, extractCitations };
|
||||||
238
packages/engine/src/research/providers/local-docs-provider.ts
Normal file
238
packages/engine/src/research/providers/local-docs-provider.ts
Normal file
@@ -0,0 +1,238 @@
|
|||||||
|
import { promises as fs } from "node:fs";
|
||||||
|
import { extname, join, relative, resolve } from "node:path";
|
||||||
|
import type { ResearchProviderConfig, ResearchSource } from "@fusion/core";
|
||||||
|
import type { ResearchProvider } from "../../research-step-runner.js";
|
||||||
|
import { createLogger } from "../../logger.js";
|
||||||
|
import { ResearchProviderError, type ResearchFetchResult } from "../types.js";
|
||||||
|
|
||||||
|
const log = createLogger("research:local-docs");
|
||||||
|
const DEFAULT_MAX_RESULTS = 10;
|
||||||
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||||
|
const MAX_FILE_SIZE_BYTES = 1024 * 1024;
|
||||||
|
const BINARY_SNIFF_BYTES = 8 * 1024;
|
||||||
|
|
||||||
|
export interface LocalDocsProviderOptions {
|
||||||
|
projectRoot: string;
|
||||||
|
timeoutMs?: number;
|
||||||
|
maxResults?: number;
|
||||||
|
scanPaths?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export class LocalDocsProvider implements ResearchProvider {
|
||||||
|
readonly type = "local-docs";
|
||||||
|
private readonly projectRoot: string;
|
||||||
|
private readonly scanPaths: string[];
|
||||||
|
|
||||||
|
constructor(private readonly options: LocalDocsProviderOptions) {
|
||||||
|
this.projectRoot = resolve(options.projectRoot);
|
||||||
|
this.scanPaths = options.scanPaths ?? ["docs", "README.md", "AGENTS.md", ".fusion/memory"];
|
||||||
|
}
|
||||||
|
|
||||||
|
isConfigured(): boolean {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async search(query: string, config: ResearchProviderConfig = {}, signal?: AbortSignal): Promise<ResearchSource[]> {
|
||||||
|
const timeoutMs = Number(config.timeoutMs ?? this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||||
|
const maxResults = Number(config.maxResults ?? this.options.maxResults ?? DEFAULT_MAX_RESULTS);
|
||||||
|
const terms = query.toLowerCase().split(/\s+/).filter(Boolean);
|
||||||
|
|
||||||
|
const files = await this.withTimeout(this.collectCandidateFiles(signal), timeoutMs, signal);
|
||||||
|
const results: Array<{ source: ResearchSource; score: number }> = [];
|
||||||
|
|
||||||
|
for (const file of files) {
|
||||||
|
this.throwIfAborted(signal);
|
||||||
|
const content = await this.safeReadText(file, signal);
|
||||||
|
if (!content) continue;
|
||||||
|
const lower = content.toLowerCase();
|
||||||
|
let score = 0;
|
||||||
|
for (const term of terms) {
|
||||||
|
const matches = lower.match(new RegExp(escapeRegex(term), "g"));
|
||||||
|
score += matches?.length ?? 0;
|
||||||
|
}
|
||||||
|
if (score <= 0) continue;
|
||||||
|
|
||||||
|
const relPath = relative(this.projectRoot, file);
|
||||||
|
results.push({
|
||||||
|
score,
|
||||||
|
source: {
|
||||||
|
id: `local-docs-${relPath}`,
|
||||||
|
type: "local",
|
||||||
|
reference: relPath,
|
||||||
|
title: relPath,
|
||||||
|
excerpt: buildExcerpt(content, terms),
|
||||||
|
status: "completed",
|
||||||
|
metadata: { score, path: relPath },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return results
|
||||||
|
.sort((a, b) => b.score - a.score)
|
||||||
|
.slice(0, maxResults)
|
||||||
|
.map((item) => item.source);
|
||||||
|
}
|
||||||
|
|
||||||
|
async fetchContent(filePath: string, config: ResearchProviderConfig = {}, signal?: AbortSignal): Promise<ResearchFetchResult> {
|
||||||
|
const timeoutMs = Number(config.timeoutMs ?? this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||||
|
const resolvedPath = resolve(this.projectRoot, filePath);
|
||||||
|
if (!resolvedPath.startsWith(this.projectRoot)) {
|
||||||
|
throw new ResearchProviderError({
|
||||||
|
providerType: "local-docs",
|
||||||
|
code: "provider-unavailable",
|
||||||
|
message: "Path traversal is not allowed",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const stat = await this.withTimeout(fs.stat(resolvedPath), timeoutMs, signal);
|
||||||
|
if (!stat.isFile()) {
|
||||||
|
throw new ResearchProviderError({ providerType: "local-docs", code: "provider-unavailable", message: "Path is not a file" });
|
||||||
|
}
|
||||||
|
|
||||||
|
const content = await this.withTimeout(fs.readFile(resolvedPath), timeoutMs, signal);
|
||||||
|
const sniff = content.subarray(0, BINARY_SNIFF_BYTES);
|
||||||
|
if (sniff.includes(0)) {
|
||||||
|
throw new ResearchProviderError({ providerType: "local-docs", code: "provider-unavailable", message: "Binary file is not supported" });
|
||||||
|
}
|
||||||
|
|
||||||
|
const text = content.toString("utf-8");
|
||||||
|
return {
|
||||||
|
content: text.length > MAX_FILE_SIZE_BYTES ? text.slice(0, MAX_FILE_SIZE_BYTES) : text,
|
||||||
|
metadata: {
|
||||||
|
path: relative(this.projectRoot, resolvedPath),
|
||||||
|
size: stat.size,
|
||||||
|
modifiedAt: stat.mtime.toISOString(),
|
||||||
|
extension: extname(resolvedPath),
|
||||||
|
},
|
||||||
|
mimeType: "text/plain",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private async collectCandidateFiles(signal?: AbortSignal): Promise<string[]> {
|
||||||
|
const ignorePatterns = await this.readGitignore();
|
||||||
|
const files: string[] = [];
|
||||||
|
|
||||||
|
for (const pathEntry of this.scanPaths) {
|
||||||
|
const target = resolve(this.projectRoot, pathEntry);
|
||||||
|
if (!target.startsWith(this.projectRoot)) continue;
|
||||||
|
try {
|
||||||
|
const stat = await fs.stat(target);
|
||||||
|
if (stat.isDirectory()) {
|
||||||
|
await this.walk(target, files, ignorePatterns, signal);
|
||||||
|
} else if (stat.isFile()) {
|
||||||
|
files.push(target);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// ignore missing entries
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const rootEntries = await fs.readdir(this.projectRoot, { withFileTypes: true });
|
||||||
|
for (const entry of rootEntries) {
|
||||||
|
if (entry.isFile() && entry.name.toLowerCase().endsWith(".md")) {
|
||||||
|
files.push(join(this.projectRoot, entry.name));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return [...new Set(files)];
|
||||||
|
}
|
||||||
|
|
||||||
|
private async walk(dir: string, out: string[], ignorePatterns: string[], signal?: AbortSignal): Promise<void> {
|
||||||
|
this.throwIfAborted(signal);
|
||||||
|
const entries = await fs.readdir(dir, { withFileTypes: true });
|
||||||
|
for (const entry of entries) {
|
||||||
|
this.throwIfAborted(signal);
|
||||||
|
const fullPath = join(dir, entry.name);
|
||||||
|
const relPath = relative(this.projectRoot, fullPath).replace(/\\/g, "/");
|
||||||
|
if (matchesGitignore(relPath, ignorePatterns)) continue;
|
||||||
|
|
||||||
|
if (entry.isDirectory()) {
|
||||||
|
await this.walk(fullPath, out, ignorePatterns, signal);
|
||||||
|
} else if (entry.isFile()) {
|
||||||
|
out.push(fullPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async safeReadText(filePath: string, signal?: AbortSignal): Promise<string | undefined> {
|
||||||
|
try {
|
||||||
|
const stat = await fs.stat(filePath);
|
||||||
|
if (stat.size > MAX_FILE_SIZE_BYTES) return undefined;
|
||||||
|
const content = await fs.readFile(filePath);
|
||||||
|
if (content.subarray(0, BINARY_SNIFF_BYTES).includes(0)) return undefined;
|
||||||
|
this.throwIfAborted(signal);
|
||||||
|
return content.toString("utf-8");
|
||||||
|
} catch (error) {
|
||||||
|
log.warn("failed to read local docs file", { filePath, error });
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async readGitignore(): Promise<string[]> {
|
||||||
|
try {
|
||||||
|
const content = await fs.readFile(join(this.projectRoot, ".gitignore"), "utf-8");
|
||||||
|
return content
|
||||||
|
.split(/\r?\n/)
|
||||||
|
.map((line) => line.trim())
|
||||||
|
.filter((line) => line && !line.startsWith("#"));
|
||||||
|
} catch {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private throwIfAborted(signal?: AbortSignal): void {
|
||||||
|
if (signal?.aborted) {
|
||||||
|
throw new ResearchProviderError({ providerType: "local-docs", code: "abort", message: "Local docs scan aborted" });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async withTimeout<T>(promise: Promise<T>, timeoutMs: number, signal?: AbortSignal): Promise<T> {
|
||||||
|
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
||||||
|
const combined = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
|
||||||
|
return await Promise.race([
|
||||||
|
promise,
|
||||||
|
new Promise<never>((_, reject) => {
|
||||||
|
combined.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => {
|
||||||
|
reject(
|
||||||
|
new ResearchProviderError({
|
||||||
|
providerType: "local-docs",
|
||||||
|
code: signal?.aborted ? "abort" : "timeout",
|
||||||
|
message: signal?.aborted ? "Local docs operation aborted" : `Local docs operation timed out after ${timeoutMs}ms`,
|
||||||
|
retryable: !signal?.aborted,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildExcerpt(content: string, terms: string[]): string {
|
||||||
|
const lower = content.toLowerCase();
|
||||||
|
const first = terms.find((term) => lower.includes(term));
|
||||||
|
if (!first) return content.slice(0, 220).replace(/\s+/g, " ").trim();
|
||||||
|
|
||||||
|
const idx = lower.indexOf(first);
|
||||||
|
const start = Math.max(0, idx - 80);
|
||||||
|
const end = Math.min(content.length, idx + 140);
|
||||||
|
return content.slice(start, end).replace(/\s+/g, " ").trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function matchesGitignore(relPath: string, patterns: string[]): boolean {
|
||||||
|
return patterns.some((pattern) => {
|
||||||
|
if (pattern.endsWith("/")) return relPath.startsWith(pattern.slice(0, -1));
|
||||||
|
if (pattern.includes("*")) {
|
||||||
|
const regex = new RegExp(`^${pattern.split("*").map(escapeRegex).join(".*")}$`);
|
||||||
|
return regex.test(relPath);
|
||||||
|
}
|
||||||
|
return relPath === pattern || relPath.startsWith(`${pattern}/`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function escapeRegex(value: string): string {
|
||||||
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||||
|
}
|
||||||
123
packages/engine/src/research/providers/page-fetch-provider.ts
Normal file
123
packages/engine/src/research/providers/page-fetch-provider.ts
Normal file
@@ -0,0 +1,123 @@
|
|||||||
|
import type { ResearchProviderConfig, ResearchSource } from "@fusion/core";
|
||||||
|
import type { ResearchProvider } from "../../research-step-runner.js";
|
||||||
|
import { createLogger } from "../../logger.js";
|
||||||
|
import { ResearchProviderError, type ResearchFetchResult } from "../types.js";
|
||||||
|
|
||||||
|
const log = createLogger("research:page-fetch");
|
||||||
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||||
|
const DEFAULT_USER_AGENT = "FusionResearchBot/1.0";
|
||||||
|
export const MAX_CONTENT_CHARS = 500 * 1024;
|
||||||
|
|
||||||
|
export interface PageFetchProviderOptions {
|
||||||
|
timeoutMs?: number;
|
||||||
|
userAgent?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class PageFetchProvider implements ResearchProvider {
|
||||||
|
readonly type = "page-fetch";
|
||||||
|
|
||||||
|
constructor(private readonly options: PageFetchProviderOptions = {}) {}
|
||||||
|
|
||||||
|
isConfigured(): boolean {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async search(_query: string, _config: ResearchProviderConfig = {}, _signal?: AbortSignal): Promise<ResearchSource[]> {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
async fetchContent(url: string, config: ResearchProviderConfig = {}, signal?: AbortSignal): Promise<ResearchFetchResult> {
|
||||||
|
const timeoutMs = Number(config.timeoutMs ?? this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||||
|
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
||||||
|
const requestSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(url, {
|
||||||
|
method: "GET",
|
||||||
|
redirect: "follow",
|
||||||
|
headers: {
|
||||||
|
"User-Agent": (config.metadata?.userAgent as string) ?? this.options.userAgent ?? DEFAULT_USER_AGENT,
|
||||||
|
},
|
||||||
|
signal: requestSignal,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new ResearchProviderError({
|
||||||
|
providerType: "page-fetch",
|
||||||
|
code: response.status >= 500 ? "provider-unavailable" : "network-error",
|
||||||
|
message: `fetch failed with status ${response.status}`,
|
||||||
|
retryable: response.status >= 500,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentType = response.headers.get("content-type") ?? "application/octet-stream";
|
||||||
|
const mimeType = contentType.split(";")[0].trim().toLowerCase();
|
||||||
|
const raw = await response.text();
|
||||||
|
const metadata: Record<string, unknown> = {
|
||||||
|
url,
|
||||||
|
contentType,
|
||||||
|
contentLength: raw.length,
|
||||||
|
};
|
||||||
|
|
||||||
|
if (mimeType.includes("text/html")) {
|
||||||
|
const extracted = extractHtml(raw);
|
||||||
|
metadata.title = extracted.title;
|
||||||
|
metadata.description = extracted.description;
|
||||||
|
metadata.contentLength = extracted.content.length;
|
||||||
|
return { content: truncate(extracted.content), metadata, mimeType };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (mimeType.includes("application/json") || looksLikeJson(raw)) {
|
||||||
|
const pretty = JSON.stringify(JSON.parse(raw), null, 2);
|
||||||
|
return { content: truncate(pretty), metadata, mimeType };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (mimeType.includes("text/") || mimeType.includes("markdown")) {
|
||||||
|
return { content: truncate(raw), metadata, mimeType };
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new ResearchProviderError({
|
||||||
|
providerType: "page-fetch",
|
||||||
|
code: "provider-unavailable",
|
||||||
|
message: `unsupported mime type: ${mimeType}`,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof ResearchProviderError) throw error;
|
||||||
|
if (error instanceof DOMException && error.name === "AbortError") {
|
||||||
|
throw new ResearchProviderError({ providerType: "page-fetch", code: "abort", message: "Fetch aborted", cause: error });
|
||||||
|
}
|
||||||
|
if (error instanceof Error && error.name === "TimeoutError") {
|
||||||
|
throw new ResearchProviderError({ providerType: "page-fetch", code: "timeout", message: error.message, retryable: true, cause: error });
|
||||||
|
}
|
||||||
|
log.warn("page fetch failed", { error });
|
||||||
|
throw new ResearchProviderError({
|
||||||
|
providerType: "page-fetch",
|
||||||
|
code: "network-error",
|
||||||
|
message: error instanceof Error ? error.message : "fetch failed",
|
||||||
|
retryable: true,
|
||||||
|
cause: error,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractHtml(html: string): { title?: string; description?: string; content: string } {
|
||||||
|
const title = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.trim();
|
||||||
|
const description = html.match(/<meta[^>]*name=["']description["'][^>]*content=["']([^"']+)["'][^>]*>/i)?.[1]?.trim();
|
||||||
|
const stripped = html
|
||||||
|
.replace(/<script[\s\S]*?<\/script>/gi, " ")
|
||||||
|
.replace(/<style[\s\S]*?<\/style>/gi, " ")
|
||||||
|
.replace(/<(nav|footer|header)[\s\S]*?<\/\1>/gi, " ");
|
||||||
|
const main = stripped.match(/<(main|article)[^>]*>([\s\S]*?)<\/\1>/i)?.[2] ?? stripped.match(/<body[^>]*>([\s\S]*?)<\/body>/i)?.[1] ?? stripped;
|
||||||
|
const text = main.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
|
||||||
|
return { title, description, content: text };
|
||||||
|
}
|
||||||
|
|
||||||
|
function truncate(value: string): string {
|
||||||
|
return value.length > MAX_CONTENT_CHARS ? value.slice(0, MAX_CONTENT_CHARS) : value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function looksLikeJson(value: string): boolean {
|
||||||
|
const trimmed = value.trim();
|
||||||
|
return trimmed.startsWith("{") || trimmed.startsWith("[");
|
||||||
|
}
|
||||||
269
packages/engine/src/research/providers/web-search-provider.ts
Normal file
269
packages/engine/src/research/providers/web-search-provider.ts
Normal file
@@ -0,0 +1,269 @@
|
|||||||
|
import type { ResearchProviderConfig, ResearchSource, WebSearchBackend } from "@fusion/core";
|
||||||
|
import type { ResearchProvider } from "../../research-step-runner.js";
|
||||||
|
import { createLogger } from "../../logger.js";
|
||||||
|
import { ResearchProviderError } from "../types.js";
|
||||||
|
|
||||||
|
const log = createLogger("research:web-search");
|
||||||
|
|
||||||
|
export interface WebSearchProviderOptions {
|
||||||
|
backend?: WebSearchBackend;
|
||||||
|
searxngUrl?: string;
|
||||||
|
braveApiKey?: string;
|
||||||
|
googleApiKey?: string;
|
||||||
|
googleCx?: string;
|
||||||
|
tavilyApiKey?: string;
|
||||||
|
maxResults?: number;
|
||||||
|
timeoutMs?: number;
|
||||||
|
userAgent?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const DEFAULT_MAX_RESULTS = 10;
|
||||||
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||||
|
const RETRY_BASE_DELAY_MS = 1_000;
|
||||||
|
const RETRY_MAX_DELAY_MS = 10_000;
|
||||||
|
const RETRY_MAX_ATTEMPTS = 3;
|
||||||
|
|
||||||
|
export class WebSearchProvider implements ResearchProvider {
|
||||||
|
readonly type = "web-search";
|
||||||
|
|
||||||
|
constructor(private readonly options: WebSearchProviderOptions = {}) {}
|
||||||
|
|
||||||
|
isConfigured(): boolean {
|
||||||
|
const backend = this.options.backend ?? "none";
|
||||||
|
if (backend === "none") return false;
|
||||||
|
if (backend === "searxng") return Boolean(this.options.searxngUrl);
|
||||||
|
if (backend === "brave") return Boolean(this.options.braveApiKey);
|
||||||
|
if (backend === "google") return Boolean(this.options.googleApiKey && this.options.googleCx);
|
||||||
|
if (backend === "tavily") return Boolean(this.options.tavilyApiKey);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
async search(query: string, config: ResearchProviderConfig = {}, signal?: AbortSignal): Promise<ResearchSource[]> {
|
||||||
|
const backend = this.options.backend ?? "none";
|
||||||
|
if (!this.isConfigured()) return [];
|
||||||
|
|
||||||
|
const maxResults = Number(config.maxResults ?? this.options.maxResults ?? DEFAULT_MAX_RESULTS);
|
||||||
|
const timeoutMs = Number(config.timeoutMs ?? this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||||
|
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
||||||
|
const requestSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
|
||||||
|
|
||||||
|
const results = await this.withHttpRetry(async () => {
|
||||||
|
if (backend === "searxng") return this.searchSearxng(query, maxResults, requestSignal);
|
||||||
|
if (backend === "brave") return this.searchBrave(query, maxResults, requestSignal);
|
||||||
|
if (backend === "google") return this.searchGoogle(query, maxResults, requestSignal);
|
||||||
|
if (backend === "tavily") return this.searchTavily(query, maxResults, requestSignal);
|
||||||
|
return [];
|
||||||
|
}, requestSignal, backend);
|
||||||
|
|
||||||
|
return results.slice(0, maxResults).map((result, index) => ({
|
||||||
|
id: `${backend}-${index}-${result.url}`,
|
||||||
|
type: "web",
|
||||||
|
reference: result.url,
|
||||||
|
title: result.title,
|
||||||
|
excerpt: result.snippet,
|
||||||
|
status: "completed",
|
||||||
|
metadata: {
|
||||||
|
backend,
|
||||||
|
rank: index + 1,
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
async fetchContent(
|
||||||
|
_url: string,
|
||||||
|
_options: ResearchProviderConfig = {},
|
||||||
|
_signal?: AbortSignal,
|
||||||
|
): Promise<{ content: string; metadata: Record<string, unknown> }> {
|
||||||
|
return { content: "", metadata: {} };
|
||||||
|
}
|
||||||
|
|
||||||
|
private async searchSearxng(query: string, maxResults: number, signal: AbortSignal) {
|
||||||
|
const base = this.options.searxngUrl?.replace(/\/$/, "") ?? "";
|
||||||
|
const url = new URL(`${base}/search`);
|
||||||
|
url.searchParams.set("q", query);
|
||||||
|
url.searchParams.set("format", "json");
|
||||||
|
|
||||||
|
const response = await this.requestJson<{ results?: Array<{ url: string; title?: string; content?: string }> }>(
|
||||||
|
url.toString(),
|
||||||
|
{
|
||||||
|
method: "GET",
|
||||||
|
signal,
|
||||||
|
},
|
||||||
|
"searxng",
|
||||||
|
);
|
||||||
|
|
||||||
|
return (response.results ?? []).slice(0, maxResults).map((item) => ({
|
||||||
|
url: item.url,
|
||||||
|
title: item.title ?? item.url,
|
||||||
|
snippet: item.content ?? "",
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
private async searchBrave(query: string, maxResults: number, signal: AbortSignal) {
|
||||||
|
const url = new URL("https://api.search.brave.com/res/v1/web/search");
|
||||||
|
url.searchParams.set("q", query);
|
||||||
|
url.searchParams.set("count", String(maxResults));
|
||||||
|
|
||||||
|
const response = await this.requestJson<{ web?: { results?: Array<{ url: string; title?: string; description?: string }> } }>(
|
||||||
|
url.toString(),
|
||||||
|
{
|
||||||
|
method: "GET",
|
||||||
|
headers: {
|
||||||
|
"X-Subscription-Token": this.options.braveApiKey ?? "",
|
||||||
|
"User-Agent": this.options.userAgent ?? "FusionResearchBot/1.0",
|
||||||
|
},
|
||||||
|
signal,
|
||||||
|
},
|
||||||
|
"brave",
|
||||||
|
);
|
||||||
|
|
||||||
|
return (response.web?.results ?? []).slice(0, maxResults).map((item) => ({
|
||||||
|
url: item.url,
|
||||||
|
title: item.title ?? item.url,
|
||||||
|
snippet: item.description ?? "",
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
private async searchGoogle(query: string, maxResults: number, signal: AbortSignal) {
|
||||||
|
const url = new URL("https://www.googleapis.com/customsearch/v1");
|
||||||
|
url.searchParams.set("q", query);
|
||||||
|
url.searchParams.set("num", String(maxResults));
|
||||||
|
url.searchParams.set("key", this.options.googleApiKey ?? "");
|
||||||
|
url.searchParams.set("cx", this.options.googleCx ?? "");
|
||||||
|
|
||||||
|
const response = await this.requestJson<{ items?: Array<{ link: string; title?: string; snippet?: string }> }>(
|
||||||
|
url.toString(),
|
||||||
|
{
|
||||||
|
method: "GET",
|
||||||
|
signal,
|
||||||
|
},
|
||||||
|
"google",
|
||||||
|
);
|
||||||
|
|
||||||
|
return (response.items ?? []).slice(0, maxResults).map((item) => ({
|
||||||
|
url: item.link,
|
||||||
|
title: item.title ?? item.link,
|
||||||
|
snippet: item.snippet ?? "",
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
private async searchTavily(query: string, maxResults: number, signal: AbortSignal) {
|
||||||
|
const response = await this.requestJson<{ results?: Array<{ url: string; title?: string; content?: string }> }>(
|
||||||
|
"https://api.tavily.com/search",
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"User-Agent": this.options.userAgent ?? "FusionResearchBot/1.0",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
api_key: this.options.tavilyApiKey,
|
||||||
|
query,
|
||||||
|
max_results: maxResults,
|
||||||
|
}),
|
||||||
|
signal,
|
||||||
|
},
|
||||||
|
"tavily",
|
||||||
|
);
|
||||||
|
|
||||||
|
return (response.results ?? []).slice(0, maxResults).map((item) => ({
|
||||||
|
url: item.url,
|
||||||
|
title: item.title ?? item.url,
|
||||||
|
snippet: item.content ?? "",
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
private async withHttpRetry<T>(
|
||||||
|
operation: () => Promise<T>,
|
||||||
|
signal: AbortSignal,
|
||||||
|
backend: WebSearchBackend,
|
||||||
|
): Promise<T> {
|
||||||
|
let attempt = 0;
|
||||||
|
let lastError: unknown;
|
||||||
|
while (attempt < RETRY_MAX_ATTEMPTS) {
|
||||||
|
if (signal.aborted) {
|
||||||
|
throw new ResearchProviderError({ providerType: "web-search", code: "abort", message: "Search aborted" });
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await operation();
|
||||||
|
} catch (error) {
|
||||||
|
lastError = error;
|
||||||
|
const isRetryableHttp =
|
||||||
|
error instanceof ResearchProviderError &&
|
||||||
|
(error.code === "rate-limited" || error.code === "network-error" || error.code === "provider-unavailable") &&
|
||||||
|
error.retryable;
|
||||||
|
|
||||||
|
attempt += 1;
|
||||||
|
if (!isRetryableHttp || attempt >= RETRY_MAX_ATTEMPTS) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
const delay = Math.min(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), RETRY_MAX_DELAY_MS);
|
||||||
|
const jitter = delay * (Math.random() * 0.2 - 0.1);
|
||||||
|
const totalDelay = Math.max(0, delay + jitter);
|
||||||
|
log.warn(`retrying ${backend} search`, { attempt, totalDelay });
|
||||||
|
await sleep(totalDelay, signal);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw lastError;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async requestJson<T>(url: string, init: RequestInit, backend: WebSearchBackend): Promise<T> {
|
||||||
|
try {
|
||||||
|
const response = await fetch(url, init);
|
||||||
|
if (!response.ok) {
|
||||||
|
const message = `${backend} request failed with status ${response.status}`;
|
||||||
|
if (response.status === 401 || response.status === 403) {
|
||||||
|
throw new ResearchProviderError({ providerType: "web-search", code: "auth-failed", message });
|
||||||
|
}
|
||||||
|
if (response.status === 429) {
|
||||||
|
throw new ResearchProviderError({
|
||||||
|
providerType: "web-search",
|
||||||
|
code: "rate-limited",
|
||||||
|
message,
|
||||||
|
retryable: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (response.status >= 500) {
|
||||||
|
throw new ResearchProviderError({
|
||||||
|
providerType: "web-search",
|
||||||
|
code: "provider-unavailable",
|
||||||
|
message,
|
||||||
|
retryable: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
throw new ResearchProviderError({ providerType: "web-search", code: "network-error", message });
|
||||||
|
}
|
||||||
|
|
||||||
|
return (await response.json()) as T;
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof ResearchProviderError) throw error;
|
||||||
|
if (error instanceof DOMException && error.name === "AbortError") {
|
||||||
|
throw new ResearchProviderError({ providerType: "web-search", code: "abort", message: "Search aborted", cause: error });
|
||||||
|
}
|
||||||
|
if (error instanceof Error && error.name === "TimeoutError") {
|
||||||
|
throw new ResearchProviderError({ providerType: "web-search", code: "timeout", message: error.message, retryable: true, cause: error });
|
||||||
|
}
|
||||||
|
throw new ResearchProviderError({
|
||||||
|
providerType: "web-search",
|
||||||
|
code: "network-error",
|
||||||
|
message: error instanceof Error ? error.message : "Unexpected network error",
|
||||||
|
retryable: true,
|
||||||
|
cause: error,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sleep(ms: number, signal: AbortSignal): Promise<void> {
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
const timer = setTimeout(resolve, ms);
|
||||||
|
const onAbort = () => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
reject(new ResearchProviderError({ providerType: "web-search", code: "abort", message: "Search aborted" }));
|
||||||
|
};
|
||||||
|
signal.addEventListener("abort", onAbort, { once: true });
|
||||||
|
});
|
||||||
|
}
|
||||||
50
packages/engine/src/research/types.ts
Normal file
50
packages/engine/src/research/types.ts
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
import type { ResearchProviderConfig as CoreResearchProviderConfig } from "@fusion/core";
|
||||||
|
|
||||||
|
export type ResearchProviderType =
|
||||||
|
| "web-search"
|
||||||
|
| "page-fetch"
|
||||||
|
| "github"
|
||||||
|
| "local-docs"
|
||||||
|
| "llm-synthesis";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Provider config bag for research providers.
|
||||||
|
* Extends core orchestration config with provider-specific optional fields.
|
||||||
|
*/
|
||||||
|
export interface ResearchProviderConfig extends CoreResearchProviderConfig {
|
||||||
|
[key: string]: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ResearchProviderErrorCode =
|
||||||
|
| "timeout"
|
||||||
|
| "abort"
|
||||||
|
| "provider-unavailable"
|
||||||
|
| "rate-limited"
|
||||||
|
| "auth-failed"
|
||||||
|
| "network-error";
|
||||||
|
|
||||||
|
export class ResearchProviderError extends Error {
|
||||||
|
readonly providerType: ResearchProviderType;
|
||||||
|
readonly code: ResearchProviderErrorCode;
|
||||||
|
readonly retryable: boolean;
|
||||||
|
|
||||||
|
constructor(options: {
|
||||||
|
providerType: ResearchProviderType;
|
||||||
|
code: ResearchProviderErrorCode;
|
||||||
|
message: string;
|
||||||
|
retryable?: boolean;
|
||||||
|
cause?: unknown;
|
||||||
|
}) {
|
||||||
|
super(options.message, options.cause ? { cause: options.cause } : undefined);
|
||||||
|
this.name = "ResearchProviderError";
|
||||||
|
this.providerType = options.providerType;
|
||||||
|
this.code = options.code;
|
||||||
|
this.retryable = options.retryable ?? false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ResearchFetchResult {
|
||||||
|
content: string;
|
||||||
|
metadata: Record<string, unknown>;
|
||||||
|
mimeType?: string;
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user