root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #223 from corrm/feat/model-auto-detect

Browse Source 
feat: auto-detect models from custom providers
This commit is contained in:
gsxdsm
2026-05-14 21:35:46 -07:00
committed by GitHub
parent 049afbd117 c0f99b5e6f
commit 6ab46aa983
9 changed files with 1028 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.changeset/fix-google-provider-create.md Normal file
View File

@@ -0,0 +1,5 @@
---
"@runfusion/fusion": minor
---
Fix Google Generative AI custom provider not saving after model detection. The probe endpoint accepted `google-generative-ai` but create/update routes rejected it. Also adds SSRF protection, body validation, and fixes stale type mappings.

2
packages/core/src/types.ts
View File

@@ -396,7 +396,7 @@ export interface NotificationProviderConfig {
export interface CustomProvider {
id: string;
name: string;
apiType: "openai-compatible" | "anthropic-compatible";
apiType: "openai-compatible" | "anthropic-compatible" | "google-generative-ai";
baseUrl: string;
apiKey?: string;
models?: { id: string; name: string }[];

44
packages/dashboard/app/api/legacy.ts
View File

@@ -1775,7 +1775,7 @@ export function setLlamaCppEnabled(
export interface CustomProvider {
id: string;
name: string;
apiType: "openai-compatible" | "anthropic-compatible";
apiType: "openai-compatible" | "anthropic-compatible" | "google-generative-ai";
baseUrl: string;
apiKey?: string;
models?: { id: string; name: string }[];
@@ -1787,7 +1787,9 @@ export async function fetchCustomProviders(): Promise<CustomProviderConfig[] & {
id: provider.id,
name: provider.name,
baseUrl: provider.baseUrl,
api: provider.apiType === "anthropic-compatible" ? "anthropic-messages" : "openai-completions",
api: provider.apiType === "anthropic-compatible" ? "anthropic-messages"
: provider.apiType === "google-generative-ai" ? "google-generative-ai"
: "openai-completions",
apiKey: provider.apiKey,
models: (provider.models ?? []).map((model) => ({ id: model.id, name: model.name })),
} satisfies CustomProviderConfig));
@@ -1860,7 +1862,9 @@ export interface CustomProviderConfig {
}
export function createCustomProvider(config: CustomProviderConfig): Promise<CustomProvider> {
const apiType = config.api === "anthropic-messages" ? "anthropic-compatible" : "openai-compatible";
const apiType = config.api === "anthropic-messages" ? "anthropic-compatible"
: config.api === "google-generative-ai" ? "google-generative-ai"
: "openai-compatible";
return addCustomProvider({
name: config.name?.trim() || config.id,
apiType,
@@ -1873,6 +1877,40 @@ export function createCustomProvider(config: CustomProviderConfig): Promise<Cust
});
}
/**
* Probe a custom provider's /models endpoint to discover available models.
* Supports OpenAI-compatible, Anthropic-compatible, and Google Generative AI providers.
*/
export interface ProbeModelResult {
id: string;
name: string;
reasoning?: boolean;
contextWindow?: number;
maxTokens?: number;
}
export interface ProbeModelsResponse {
models: ProbeModelResult[];
count: number;
}
export interface ProbeModelsParams {
baseUrl: string;
apiKey?: string;
apiType: "openai-compatible" | "anthropic-compatible" | "google-generative-ai";
}
export async function probeProviderModels(params: ProbeModelsParams): Promise<ProbeModelsResponse> {
return api<ProbeModelsResponse>("/custom-providers/probe-models", {
method: "POST",
body: JSON.stringify({
baseUrl: params.baseUrl,
apiKey: params.apiKey,
apiType: params.apiType,
}),
});
}
/** Fetch authentication status for all OAuth providers */
export function fetchAuthStatus(): Promise<{
providers: AuthProvider[];

94
packages/dashboard/app/components/CustomProviderForm.tsx
View File

@@ -1,5 +1,7 @@
import { useMemo, useState } from "react";
import { useCallback, useMemo, useState } from "react";
import { Loader2, Search } from "lucide-react";
import type { CustomProviderConfig, CustomProviderModelInput } from "../api";
import { probeProviderModels } from "../api";
import "./CustomProviderForm.css";
// Reserved built-in IDs (including hidden/deprecated aliases) to prevent custom-provider collisions.
@@ -42,6 +44,8 @@ export function CustomProviderForm({ initialConfig, onSave, onCancel, saving = f
const [apiKey, setApiKey] = useState(initialConfig?.apiKey ?? "");
const [models, setModels] = useState<CustomProviderModelInput[]>(initialConfig?.models?.length ? initialConfig.models : [emptyModel()]);
const [validationError, setValidationError] = useState<string | null>(null);
const [detecting, setDetecting] = useState(false);
const [detectError, setDetectError] = useState<string | null>(null);
const canRemoveModel = models.length > 1;
@@ -55,6 +59,68 @@ export function CustomProviderForm({ initialConfig, onSave, onCancel, saving = f
setModels((prev) => (prev.length <= 1 ? prev : prev.filter((_, i) => i !== index)));
}
// Detect Models is available for all API types that expose a /models endpoint:
// - openai-completions / openai-responses → openai-compatible
// - anthropic-messages → anthropic-compatible
// - google-generative-ai → google-generative-ai
const probeApiType = api === "anthropic-messages"
? "anthropic-compatible"
: api === "google-generative-ai"
? "google-generative-ai"
: "openai-compatible";
const handleDetectModels = useCallback(async () => {
const trimmedBaseUrl = baseUrl.trim();
if (!trimmedBaseUrl) {
setDetectError("Base URL is required to detect models.");
return;
}
setDetecting(true);
setDetectError(null);
try {
const result = await probeProviderModels({
baseUrl: trimmedBaseUrl,
apiKey: apiKey.trim() || undefined,
apiType: probeApiType,
});
if (result.models.length === 0) {
setDetectError("No models found. The provider may require an API key.");
return;
}
// Merge discovered models, avoiding duplicates by ID
const existingIds = new Set(models.map((m) => m.id.trim()));
const newModels = result.models
.filter((m) => !existingIds.has(m.id.trim()))
.map((m) => ({
id: m.id,
name: m.name || m.id,
reasoning: Boolean(m.reasoning),
contextWindow: m.contextWindow,
maxTokens: m.maxTokens,
}));
if (newModels.length > 0) {
// Replace empty default rows with discovered models
setModels((prev) => {
const nonEmpty = prev.filter((m) => m.id.trim().length > 0);
return [...nonEmpty, ...newModels];
});
} else {
setDetectError("All discovered models are already in the list.");
}
} catch (err) {
setDetectError(
err instanceof Error ? err.message : "Failed to detect models",
);
} finally {
setDetecting(false);
}
}, [baseUrl, apiKey, probeApiType, models]);
function validate(): string | null {
if (!id.trim()) return "Provider ID is required.";
if (!PROVIDER_ID_PATTERN.test(id.trim())) return "Provider ID must be kebab-case.";
@@ -187,9 +253,29 @@ export function CustomProviderForm({ initialConfig, onSave, onCancel, saving = f
</div>
))}
</div>
<button type="button" className="btn btn-sm" onClick={() => setModels((prev) => [...prev, emptyModel()])} disabled={saving}>
+ Add model
</button>
<div className="custom-provider-form__model-actions" style={{ display: "flex", gap: "8px", alignItems: "center" }}>
<button type="button" className="btn btn-sm" onClick={() => setModels((prev) => [...prev, emptyModel()])} disabled={saving}>
+ Add model
</button>
<button
type="button"
className="btn btn-sm"
onClick={() => void handleDetectModels()}
disabled={saving || detecting || !baseUrl.trim()}
title="Call the provider's /models endpoint to discover available models"
>
{detecting ? (
<>
<Loader2 className="spin" size={14} /> Detecting…
</>
) : (
<>
<Search size={14} /> Detect Models
</>
)}
</button>
</div>
{detectError ? <div className="form-error" style={{ marginTop: "4px" }}>{detectError}</div> : null}
</div>
{mergedError ? <div className="form-error">{mergedError}</div> : null}

97
packages/dashboard/app/components/CustomProvidersSection.tsx
View File

@@ -3,16 +3,17 @@ import {
addCustomProvider,
deleteCustomProvider,
fetchCustomProviders,
probeProviderModels,
updateCustomProvider,
type CustomProvider,
} from "../api";
import { AlertCircle, Loader2, Pencil, Plus, Trash2 } from "lucide-react";
import { AlertCircle, Loader2, Pencil, Plus, Search, Trash2 } from "lucide-react";
import { OnboardingDisclosure } from "./OnboardingDisclosure";
import "./CustomProvidersSection.css";
type ProviderApiType = CustomProvider["apiType"];
const API_TYPES: ProviderApiType[] = ["openai-compatible", "anthropic-compatible"];
const API_TYPES: ProviderApiType[] = ["openai-compatible", "anthropic-compatible", "google-generative-ai"];
type LegacyProvider = {
id: string;
@@ -74,6 +75,8 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C
const [models, setModels] = useState("");
const [saving, setSaving] = useState(false);
const [formError, setFormError] = useState<string | null>(null);
const [detecting, setDetecting] = useState(false);
const [detectError, setDetectError] = useState<string | null>(null);
const loadProviders = useCallback(async () => {
setLoading(true);
@@ -112,6 +115,8 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C
setApiKey("");
setModels("");
setFormError(null);
setDetectError(null);
setDetecting(false);
setIsFormOpen(false);
}, []);
@@ -123,6 +128,8 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C
setApiKey("");
setModels("");
setFormError(null);
setDetectError(null);
setDetecting(false);
setIsFormOpen(true);
}, []);
@@ -134,6 +141,8 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C
setApiKey(provider.apiKey ?? "");
setModels((provider.models ?? []).map((model) => model.id).join(", "));
setFormError(null);
setDetectError(null);
setDetecting(false);
setIsFormOpen(true);
}, []);
@@ -165,6 +174,48 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C
return null;
}, [apiType, baseUrl, name]);
// Detect Models is available for all API types that expose a /models endpoint
const handleDetectModels = useCallback(async () => {
const trimmedBaseUrl = baseUrl.trim();
if (!trimmedBaseUrl) {
setDetectError("Base URL is required to detect models.");
return;
}
setDetecting(true);
setDetectError(null);
try {
const result = await probeProviderModels({
baseUrl: trimmedBaseUrl,
apiKey: apiKey.trim() || undefined,
apiType,
});
if (result.models.length > 0) {
setModels((prev) => {
const existingIds = new Set(
prev.split(",").map((s) => s.trim()).filter(Boolean),
);
const newIds = result.models
.map((m) => m.id.trim())
.filter((id) => !existingIds.has(id));
if (newIds.length === 0) return prev;
const existing = prev.trim();
return newIds.join(", ") + (existing ? ", " + existing : "");
});
} else {
setDetectError("No models found. The provider may require an API key.");
}
} catch (err) {
setDetectError(
err instanceof Error ? err.message : "Failed to detect models",
);
} finally {
setDetecting(false);
}
}, [baseUrl, apiKey, apiType]);
const handleSave = useCallback(async () => {
const validationError = validateForm();
setFormError(validationError);
@@ -327,6 +378,27 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C
/>
</div>
<div style={{ display: "flex", gap: "8px", alignItems: "center", marginTop: "4px" }}>
<button
type="button"
className="btn btn-sm"
onClick={() => void handleDetectModels()}
disabled={saving || detecting || !baseUrl.trim()}
title="Auto-detect models from the provider's /models endpoint"
>
{detecting ? (
<>
<Loader2 className="spin" size={14} /> Detecting…
</>
) : (
<>
<Search size={14} /> Detect Models
</>
)}
</button>
</div>
{detectError ? <div className="custom-provider-form-error">{detectError}</div> : null}
{formError ? <div className="custom-provider-form-error">{formError}</div> : null}
<div className="custom-provider-form-actions">
@@ -421,6 +493,27 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C
/>
</div>
<div style={{ display: "flex", gap: "8px", alignItems: "center", marginTop: "4px" }}>
<button
type="button"
className="btn btn-sm"
onClick={() => void handleDetectModels()}
disabled={saving || detecting || !baseUrl.trim()}
title="Auto-detect models from the provider's /models endpoint"
>
{detecting ? (
<>
<Loader2 className="spin" size={14} /> Detecting…
</>
) : (
<>
<Search size={14} /> Detect Models
</>
)}
</button>
</div>
{detectError ? <div className="custom-provider-form-error">{detectError}</div> : null}
{formError ? <div className="custom-provider-form-error">{formError}</div> : null}
<div className="custom-provider-form-actions">

187
packages/dashboard/app/components/__tests__/CustomProviderForm.test.tsx
View File

@@ -1,7 +1,8 @@
import { describe, it, expect, vi } from "vitest";
import { describe, it, expect, vi, beforeEach } from "vitest";
import { render, screen } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { CustomProviderForm } from "../CustomProviderForm";
import * as api from "../../api";
describe("CustomProviderForm", () => {
it("renders base fields", () => {
@@ -58,3 +59,187 @@ describe("CustomProviderForm", () => {
expect(screen.getByText("Request failed")).toBeInTheDocument();
});
});
describe("Detect Models", () => {
beforeEach(() => {
vi.resetModules();
});
it("shows the Detect Models button for openai-completions API type", () => {
render(
<CustomProviderForm
onSave={vi.fn()}
initialConfig={{
id: "my-provider",
baseUrl: "https://api.example.com/v1",
api: "openai-completions",
apiKey: "sk-test",
models: [{ id: "gpt-4o", name: "GPT 4o" }],
}}
/>
);
expect(screen.getByRole("button", { name: /detect models/i })).toBeInTheDocument();
});
it("shows the Detect Models button for openai-responses API type", () => {
render(
<CustomProviderForm
onSave={vi.fn()}
initialConfig={{
id: "my-provider",
baseUrl: "https://api.example.com/v1",
api: "openai-responses",
apiKey: "sk-test",
models: [{ id: "gpt-4o", name: "GPT 4o" }],
}}
/>
);
expect(screen.getByRole("button", { name: /detect models/i })).toBeInTheDocument();
});
it("shows the Detect Models button for anthropic-messages API type", () => {
render(
<CustomProviderForm
onSave={vi.fn()}
initialConfig={{
id: "my-provider",
baseUrl: "https://api.anthropic.com",
api: "anthropic-messages",
apiKey: "sk-ant-test",
models: [{ id: "claude-3", name: "Claude 3" }],
}}
/>
);
expect(screen.getByRole("button", { name: /detect models/i })).toBeInTheDocument();
});
it("shows the Detect Models button for google-generative-ai API type", () => {
render(
<CustomProviderForm
onSave={vi.fn()}
initialConfig={{
id: "my-provider",
baseUrl: "https://generativelanguage.googleapis.com",
api: "google-generative-ai",
apiKey: "sk-google",
models: [{ id: "gemini-pro", name: "Gemini Pro" }],
}}
/>
);
expect(screen.getByRole("button", { name: /detect models/i })).toBeInTheDocument();
});
it("calls probeProviderModels and adds discovered models", async () => {
const mockProbe = vi.spyOn(api, "probeProviderModels").mockResolvedValue({
models: [
{ id: "gpt-4o", name: "GPT 4o", reasoning: false },
{ id: "gpt-4", name: "GPT 4", reasoning: false },
],
count: 2,
});
render(
<CustomProviderForm
onSave={vi.fn()}
initialConfig={{
id: "my-provider",
baseUrl: "https://api.example.com/v1",
api: "openai-completions",
apiKey: "sk-test",
models: [{ id: "", name: "", reasoning: false }],
}}
/>
);
const user = userEvent.setup();
await user.click(screen.getByRole("button", { name: /detect models/i }));
expect(mockProbe).toHaveBeenCalledWith(
expect.objectContaining({
baseUrl: "https://api.example.com/v1",
apiKey: "sk-test",
apiType: "openai-compatible",
})
);
// Models should be added to the list
expect(screen.getByDisplayValue("gpt-4o")).toBeInTheDocument();
expect(screen.getByDisplayValue("gpt-4")).toBeInTheDocument();
});
it("deduplicates models when detecting", async () => {
const mockProbe = vi.spyOn(api, "probeProviderModels").mockResolvedValue({
models: [
{ id: "gpt-4o", name: "GPT 4o", reasoning: false },
{ id: "gpt-4", name: "GPT 4", reasoning: false },
],
count: 2,
});
render(
<CustomProviderForm
onSave={vi.fn()}
initialConfig={{
id: "my-provider",
baseUrl: "https://api.example.com/v1",
api: "openai-completions",
apiKey: "sk-test",
models: [
{ id: "gpt-4o", name: "GPT 4o", reasoning: false },
{ id: "", name: "", reasoning: false },
],
}}
/>
);
const user = userEvent.setup();
await user.click(screen.getByRole("button", { name: /detect models/i }));
// gpt-4o should appear only once (existing + deduplicated)
const gpt4oInputs = screen.queryAllByDisplayValue("gpt-4o");
expect(gpt4oInputs).toHaveLength(1);
// gpt-4 should be added
expect(screen.getByDisplayValue("gpt-4")).toBeInTheDocument();
});
it("shows error when detection fails", async () => {
const mockProbe = vi.spyOn(api, "probeProviderModels").mockRejectedValue(
new Error("Provider returned 401 Unauthorized")
);
render(
<CustomProviderForm
onSave={vi.fn()}
initialConfig={{
id: "my-provider",
baseUrl: "https://api.example.com/v1",
api: "openai-completions",
apiKey: "sk-invalid",
models: [{ id: "", name: "", reasoning: false }],
}}
/>
);
const user = userEvent.setup();
await user.click(screen.getByRole("button", { name: /detect models/i }));
expect(screen.getByText("Provider returned 401 Unauthorized")).toBeInTheDocument();
});
it("disables button when baseUrl is empty", () => {
render(
<CustomProviderForm
onSave={vi.fn()}
initialConfig={{
id: "my-provider",
baseUrl: "",
api: "openai-completions",
apiKey: "sk-test",
models: [{ id: "", name: "", reasoning: false }],
}}
/>
);
const detectBtn = screen.getByRole("button", { name: /detect models/i });
expect(detectBtn).toBeDisabled();
});
});

1
packages/dashboard/app/components/__tests__/CustomProvidersSection.test.tsx
View File

@@ -20,6 +20,7 @@ vi.mock("lucide-react", () => ({
Loader2: ({ className }: { className?: string }) => <svg data-testid="icon-loader" className={className} />,
Pencil: () => <svg data-testid="icon-pencil" />,
Plus: () => <svg data-testid="icon-plus" />,
Search: () => <svg data-testid="icon-search" />,
Trash2: () => <svg data-testid="icon-trash" />,
}));

280
packages/dashboard/src/routes/__tests__/custom-providers.test.ts
View File

@@ -239,7 +239,7 @@ describe("custom providers API routes", () => {
});
expect(res.status).toBe(400);
expect(String(res.body.error)).toContain("apiType must be either");
expect(String(res.body.error)).toContain("apiType must be");
});
it("POST /api/custom-providers rejects invalid baseUrl format", async () => {
@@ -352,3 +352,281 @@ describe("custom providers API routes", () => {
expect(String(res.body.error)).toContain("not found");
});
});
describe("POST /api/custom-providers/probe-models", () => {
let mockFetch: ReturnType<typeof vi.fn>;
beforeEach(() => {
mockFetch = vi.fn();
vi.stubGlobal("fetch", mockFetch);
});
afterEach(() => {
vi.unstubAllGlobals();
vi.clearAllMocks();
});
it("returns OpenAI-compatible models", async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
json: async () => ({
data: [
{ id: "gpt-4o", object: "model", owned_by: "system" },
{ id: "gpt-4", object: "model", owned_by: "system" },
],
}),
});
const app = setupApp(createCustomProviderStore().store);
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
baseUrl: "https://api.openai.com/v1",
apiType: "openai-compatible",
apiKey: "sk-test",
});
expect(res.status).toBe(200);
expect(res.body).toEqual({
count: 2,
models: [
{ id: "gpt-4o", name: "gpt-4o", reasoning: false },
{ id: "gpt-4", name: "gpt-4", reasoning: false },
],
});
});
it("returns Anthropic-compatible models", async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
json: async () => ({
data: [
{ id: "claude-sonnet-4-20250514", object: "model", display_name: "Claude Sonnet 4" },
{ id: "claude-haiku-4-5-20251001", object: "model", display_name: "Claude Haiku 4.5" },
{ id: "claude-opus-4-20250514", object: "model", display_name: "Claude Opus 4" },
],
}),
});
const app = setupApp(createCustomProviderStore().store);
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
baseUrl: "https://api.anthropic.com",
apiType: "anthropic-compatible",
apiKey: "sk-ant-test",
});
expect(res.status).toBe(200);
expect(res.body.count).toBe(3);
expect(res.body.models[0]).toEqual({
id: "claude-sonnet-4-20250514",
name: "Claude Sonnet 4",
reasoning: false, // standard sonnet without thinking capability
});
expect(res.body.models[2]).toEqual({
id: "claude-opus-4-20250514",
name: "Claude Opus 4",
reasoning: true, // opus detected as reasoning
});
});
it("returns Google Generative AI models", async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
json: async () => ({
models: [
{
name: "models/gemini-2.0-flash",
baseModelId: "gemini-2.0-flash",
displayName: "Gemini 2.0 Flash",
inputTokenLimit: 1048576,
outputTokenLimit: 8192,
supportedGenerationMethods: ["generateContent"],
},
{
name: "models/text-embedding-004",
baseModelId: "text-embedding-004",
displayName: "Text Embedding",
supportedGenerationMethods: ["embedContent"],
},
],
}),
});
const app = setupApp(createCustomProviderStore().store);
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
baseUrl: "https://generativelanguage.googleapis.com",
apiType: "google-generative-ai",
apiKey: "AIza-test",
});
expect(res.status).toBe(200);
// Embedding model should be filtered out
expect(res.body.count).toBe(1);
expect(res.body.models[0]).toEqual({
id: "gemini-2.0-flash",
name: "Gemini 2.0 Flash",
reasoning: false,
contextWindow: 1048576,
maxTokens: 8192,
});
});
it("excludes embedding models from OpenAI-compatible response", async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
json: async () => ({
data: [
{ id: "gpt-4o", object: "model", modalities: { input: ["text"], output: ["text"] } },
{ id: "text-embedding-3", object: "model", modalities: { input: ["text"], output: ["embedding"] } },
{ id: "whisper-large", object: "model", modalities: { input: ["audio"], output: ["text"] } },
],
}),
});
const app = setupApp(createCustomProviderStore().store);
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
baseUrl: "https://api.example.com/v1",
apiType: "openai-compatible",
});
expect(res.status).toBe(200);
expect(res.body.count).toBe(1); // embedding + audio-input both excluded
expect(res.body.models[0].id).toBe("gpt-4o");
});
it("excludes models without text input from OpenAI-compatible response", async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
json: async () => ({
data: [
{ id: "gpt-4o", object: "model", modalities: { input: ["text", "image"], output: ["text"] } },
{ id: "scribe-v2", object: "model", modalities: { input: ["audio"], output: ["text"] } },
{ id: "eleven-v3", object: "model", modalities: { input: ["text"], output: ["audio"] } },
],
}),
});
const app = setupApp(createCustomProviderStore().store);
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
baseUrl: "https://api.example.com/v1",
apiType: "openai-compatible",
});
expect(res.status).toBe(200);
expect(res.body.count).toBe(1); // only gpt-4o has text input + text output
expect(res.body.models[0].id).toBe("gpt-4o");
});
it("rejects invalid apiType for probe", async () => {
const app = setupApp(createCustomProviderStore().store);
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
baseUrl: "https://api.example.com",
apiType: "invalid",
});
expect(res.status).toBe(400);
});
it("detects reasoning models from ID", async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
json: async () => ({
data: [
{ id: "o1-preview", object: "model" },
{ id: "o3-mini", object: "model" },
{ id: "gpt-4o", object: "model" },
],
}),
});
const app = setupApp(createCustomProviderStore().store);
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
baseUrl: "https://api.openai.com/v1",
apiType: "openai-compatible",
});
expect(res.body.models[0].reasoning).toBe(true); // o1-preview
expect(res.body.models[1].reasoning).toBe(true); // o3-mini
expect(res.body.models[2].reasoning).toBe(false); // gpt-4o
});
it("returns 400 for missing baseUrl", async () => {
const app = setupApp(createCustomProviderStore().store);
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
apiType: "openai-compatible",
});
expect(res.status).toBe(400);
});
it("returns 400 for invalid URL", async () => {
const app = setupApp(createCustomProviderStore().store);
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
baseUrl: "not-a-url",
apiType: "openai-compatible",
});
expect(res.status).toBe(400);
});
it("returns error when provider returns non-200", async () => {
mockFetch.mockResolvedValueOnce({
ok: false,
status: 401,
statusText: "Unauthorized",
text: async () => "Invalid API key",
});
const app = setupApp(createCustomProviderStore().store);
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
baseUrl: "https://api.openai.com/v1",
apiType: "openai-compatible",
apiKey: "sk-invalid",
});
expect(res.status).toBe(401);
});
it("handles { models: [...] } response format", async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
json: async () => ({
models: [
{ id: "llama-3.1-8b", name: "Llama 3.1 8B" },
],
}),
});
const app = setupApp(createCustomProviderStore().store);
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
baseUrl: "https://api.example.com",
apiType: "openai-compatible",
});
expect(res.status).toBe(200);
expect(res.body.models[0]).toEqual({
id: "llama-3.1-8b",
name: "Llama 3.1 8B",
reasoning: false,
});
});
it("truncates large model lists to 100", async () => {
const manyModels = Array.from({ length: 150 }, (_, i) => ({
id: `model-${i}`,
object: "model",
}));
mockFetch.mockResolvedValueOnce({
ok: true,
json: async () => ({ data: manyModels }),
});
const app = setupApp(createCustomProviderStore().store);
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
baseUrl: "https://api.example.com",
apiType: "openai-compatible",
});
expect(res.status).toBe(200);
expect(res.body.count).toBe(100);
expect(res.body.models.length).toBe(100);
});
});

332
packages/dashboard/src/routes/register-custom-provider-routes.ts
View File

@@ -1,8 +1,13 @@
import crypto from "node:crypto";
import dns from "node:dns/promises";
import net from "node:net";
import type { CustomProvider } from "@fusion/core";
import { ApiError, badRequest, notFound } from "../api-error.js";
import type { ApiRouteRegistrar } from "./types.js";
/**
* Masks an API key for safe display, showing only the first 3 and last 4 characters.
*/
function maskApiKey(key: string): string {
if (key.length <= 8) {
return "••••••••";
@@ -10,6 +15,9 @@ function maskApiKey(key: string): string {
return key.slice(0, 3) + "•••••" + key.slice(-4);
}
/**
* Removes the raw API key from a provider object, replacing it with a masked version.
*/
function sanitizeProvider(provider: CustomProvider): CustomProvider {
if (!provider.apiKey) {
return provider;
@@ -21,6 +29,10 @@ function sanitizeProvider(provider: CustomProvider): CustomProvider {
};
}
/**
* Asserts that a value is a non-empty string and returns the trimmed value.
* @throws {ApiError} with status 400 if the value is not a non-empty string.
*/
function assertNonEmptyString(value: unknown, fieldName: string): string {
if (typeof value !== "string" || value.trim().length === 0) {
throw badRequest(`${fieldName} is required and must be a non-empty string`);
@@ -28,13 +40,21 @@ function assertNonEmptyString(value: unknown, fieldName: string): string {
return value.trim();
}
/**
* Asserts that a value is a valid custom provider API type.
* @throws {ApiError} with status 400 if the type is not recognized.
*/
function assertApiType(value: unknown): CustomProvider["apiType"] {
if (value !== "openai-compatible" && value !== "anthropic-compatible") {
throw badRequest("apiType must be either 'openai-compatible' or 'anthropic-compatible'");
if (value !== "openai-compatible" && value !== "anthropic-compatible" && value !== "google-generative-ai") {
throw badRequest("apiType must be 'openai-compatible', 'anthropic-compatible', or 'google-generative-ai'");
}
return value;
}
/**
* Asserts that a value is a valid HTTP/HTTPS URL suitable for use as a base URL.
* @throws {ApiError} with status 400 if the URL is invalid or uses an unsupported protocol.
*/
function assertBaseUrl(value: unknown): string {
const baseUrl = assertNonEmptyString(value, "baseUrl");
@@ -52,6 +72,11 @@ function assertBaseUrl(value: unknown): string {
return baseUrl;
}
/**
* Validates and normalizes a models array from a request body.
* Returns undefined if models is omitted, or an array of { id, name } objects.
* @throws {ApiError} with status 400 if the structure is invalid.
*/
function validateModels(value: unknown): Array<{ id: string; name: string }> | undefined {
if (value === undefined) {
return undefined;
@@ -74,6 +99,11 @@ function validateModels(value: unknown): Array<{ id: string; name: string }> | u
});
}
/**
* Parses and validates the body of a create-custom-provider request.
* Returns all required and optional fields except the auto-generated id.
* @throws {ApiError} with status 400 if required fields are missing or invalid.
*/
function parseCreateBody(body: unknown): Omit<CustomProvider, "id"> {
if (!body || typeof body !== "object") {
throw badRequest("request body must be an object");
@@ -103,6 +133,262 @@ function parseCreateBody(body: unknown): Omit<CustomProvider, "id"> {
return provider;
}
interface ProbeModelResult {
id: string;
name: string;
reasoning?: boolean;
contextWindow?: number;
maxTokens?: number;
}
const MAX_PROBE_MODELS = 100;
type ProbeApiType = "openai-compatible" | "anthropic-compatible" | "google-generative-ai";
/**
* Check if a model should be excluded (embedding / reranking / audio-only / no-text-input models).
*/
function isNonChatModel(m: Record<string, unknown>): boolean {
// OpenAI-compatible modalities: { input: ["text"], output: ["embedding"] }
const modalities = m.modalities as Record<string, unknown> | undefined;
if (modalities) {
// Exclude models that don't accept text input (e.g. audio-only, image-only)
if (Array.isArray(modalities.input)) {
const inputs = modalities.input.map((i: unknown) => String(i).toLowerCase());
if (!inputs.includes("text")) {
return true;
}
}
if (Array.isArray(modalities.output)) {
const outputs = modalities.output.map((o: unknown) => String(o).toLowerCase());
if (outputs.includes("embedding") || outputs.includes("scores")) {
return true;
}
// Exclude models that don't produce text output (e.g. audio-only)
if (!outputs.includes("text")) {
return true;
}
}
}
// Google supportedGenerationMethods: no generateContent = not a chat model
const methods = m.supportedGenerationMethods as unknown[] | undefined;
if (Array.isArray(methods) && methods.length > 0 && !methods.includes("generateContent")) {
return true;
}
// Heuristic: model ID contains embedding / rerank
const id = String(m.id ?? m.name ?? "").toLowerCase();
if (id.includes("embedding") || id.includes("embed-") || id.includes("-embed-") || id.includes("rerank")) {
return true;
}
return false;
}
/**
* Probe a custom provider's /models endpoint to discover available models.
* Supports OpenAI-compatible, Anthropic-compatible, and Google Generative AI providers.
*/
async function probeProviderModels(
baseUrl: string,
apiKey: string | undefined,
apiType: ProbeApiType,
): Promise<ProbeModelResult[]> {
let url: URL;
try {
url = new URL(baseUrl);
} catch {
throw badRequest("baseUrl must be a valid URL");
}
if (url.protocol !== "http:" && url.protocol !== "https:") {
throw badRequest("baseUrl must use http or https");
}
// SSRF protection: reject private/loopback/link-local hosts
const hostname = url.hostname.toLowerCase();
if (
hostname === "localhost" ||
hostname === "127.0.0.1" ||
hostname === "::1" ||
hostname === "[::1]" ||
hostname.endsWith(".local") ||
hostname.endsWith(".internal")
) {
throw badRequest("baseUrl must not be a loopback or private address");
}
// Resolve hostname to IP and check against private ranges.
// If resolution fails, let the fetch attempt proceed naturally.
try {
const resolved = await dns.lookup(hostname, { all: true });
const addresses = resolved.map((a) => a.address);
for (const addr of addresses) {
if (net.isIP(addr) === 0) continue;
const parts = addr.split(".").map(Number);
if (parts.length === 4 && !Number.isNaN(parts[0])) {
// 127.0.0.0/8
if (parts[0] === 127) throw badRequest("baseUrl must not be a loopback or private address");
// 10.0.0.0/8
if (parts[0] === 10) throw badRequest("baseUrl must not be a loopback or private address");
// 172.16.0.0/12
if (parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) throw badRequest("baseUrl must not be a loopback or private address");
// 192.168.0.0/16
if (parts[0] === 192 && parts[1] === 168) throw badRequest("baseUrl must not be a loopback or private address");
// 169.254.0.0/16 (link-local, includes cloud metadata)
if (parts[0] === 169 && parts[1] === 254) throw badRequest("baseUrl must not be a loopback or private address");
} else if (net.isIPv6(addr)) {
const lower = addr.toLowerCase();
// ::1 — IPv6 loopback
if (lower === "::1" || lower === "0:0:0:0:0:0:0:1") throw badRequest("baseUrl must not be a loopback or private address");
// fc00::/7 — Unique Local Addresses (private, RFC 4193)
if (lower.startsWith("fc") || lower.startsWith("fd")) throw badRequest("baseUrl must not be a loopback or private address");
// fe80::/10 — link-local addresses
if (lower.startsWith("fe8") || lower.startsWith("fe9") || lower.startsWith("fea") || lower.startsWith("feb")) throw badRequest("baseUrl must not be a loopback or private address");
// ::ffff:0:0/96 — IPv4-mapped IPv6 — extract embedded IPv4 and re-check
const ipv4Mapped = lower.match(/::ffff:(\d+\.\d+\.\d+\.\d+)/);
if (ipv4Mapped) {
const v4Parts = ipv4Mapped[1].split(".").map(Number);
if (v4Parts.length === 4) {
if (v4Parts[0] === 127 || v4Parts[0] === 10 ||
(v4Parts[0] === 172 && v4Parts[1] >= 16 && v4Parts[1] <= 31) ||
(v4Parts[0] === 192 && v4Parts[1] === 168) ||
(v4Parts[0] === 169 && v4Parts[1] === 254)) {
throw badRequest("baseUrl must not be a loopback or private address");
}
}
}
}
}
} catch (err) {
if (err instanceof ApiError) throw err;
// DNS resolution failed — proceed without SSRF check; the fetch will fail naturally
}
let modelsUrl: string;
const headers: Record<string, string> = {
"Content-Type": "application/json",
"User-Agent": "Fusion/1.0",
};
if (apiType === "openai-compatible") {
// OpenAI-compatible: /v1/models relative to baseUrl
const pathname = url.pathname.replace(/\/+$/, "");
const modelsPath = pathname ? pathname + "/models" : "/models";
modelsUrl = new URL(modelsPath, url.origin).toString();
if (apiKey) headers["Authorization"] = `Bearer ${apiKey}`;
} else if (apiType === "anthropic-compatible") {
// Anthropic: GET /v1/models with x-api-key header
const pathname = url.pathname.replace(/\/+$/, "");
const modelsPath = pathname ? pathname + "/models" : "/v1/models";
modelsUrl = new URL(modelsPath, url.origin).toString();
if (apiKey) headers["x-api-key"] = apiKey;
headers["anthropic-version"] = "2023-06-01";
} else {
// Google Generative AI: GET /v1beta/models?key=API_KEY
const pathname = url.pathname.replace(/\/+$/, "");
const modelsPath = pathname ? pathname + "/models" : "/v1beta/models";
modelsUrl = new URL(modelsPath, url.origin).toString();
if (apiKey) {
// Append API key as query parameter (Google convention)
const separator = modelsUrl.includes("?") ? "&" : "?";
modelsUrl = `${modelsUrl}${separator}key=${encodeURIComponent(apiKey)}`;
}
}
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 10_000);
try {
const response = await fetch(modelsUrl, {
method: "GET",
headers,
signal: controller.signal,
});
if (!response.ok) {
const errorBody = await response.text().catch(() => "");
const message = errorBody.slice(0, 200);
throw new ApiError(
response.status,
`Provider returned ${response.status} ${response.statusText}${message ? `: ${message}` : ""}`,
);
}
const data = await response.json();
const rawModels = data?.data ?? data?.models ?? [];
if (!Array.isArray(rawModels) || rawModels.length === 0) {
throw new ApiError(404, "No models found in provider response");
}
// Filter out embedding/reranking/audio-only models and truncate
const chatModels = rawModels.filter((m: Record<string, unknown>) => !isNonChatModel(m));
const trimmed = chatModels.length > MAX_PROBE_MODELS ? chatModels.slice(0, MAX_PROBE_MODELS) : chatModels;
return trimmed.map((m: Record<string, unknown>) => {
// Extract ID based on provider format
let id: string;
let name: string;
let contextWindow: number | undefined;
let maxTokens: number | undefined;
let reasoning: boolean;
if (apiType === "google-generative-ai") {
// Google: name = "models/gemini-2.0-flash", baseModelId = "gemini-2.0-flash"
id = String(m.baseModelId ?? m.name ?? "");
// Strip "models/" prefix if present
if (id.startsWith("models/")) id = id.slice(7);
name = String(m.displayName ?? id);
contextWindow = typeof m.inputTokenLimit === "number" && m.inputTokenLimit > 0
? m.inputTokenLimit
: undefined;
maxTokens = typeof m.outputTokenLimit === "number" && m.outputTokenLimit > 0
? m.outputTokenLimit
: undefined;
reasoning = Boolean(m.thinking);
} else if (apiType === "anthropic-compatible") {
// Anthropic: id = "claude-sonnet-4-20250514", display_name = "Claude Sonnet 4"
id = String(m.id ?? "");
name = String(m.display_name ?? id);
// Anthropic doesn't return context/max_tokens in the models list
reasoning = Boolean(
id.toLowerCase().includes("opus") ||
(id.toLowerCase().includes("sonnet") && id.toLowerCase().includes("think")),
);
} else {
// OpenAI-compatible
id = String(m.id ?? "");
name = String(m.name ?? m.display_name ?? id);
reasoning = Boolean(
m.reasoning ||
(Array.isArray(m.capabilities) && m.capabilities.includes("reasoning")) ||
id.toLowerCase().includes("reason") ||
id.toLowerCase().includes("o1") ||
id.toLowerCase().includes("o3"),
);
// Extract context window and max tokens from limit object
const limit = m.limit as Record<string, unknown> | undefined;
contextWindow = typeof limit?.context === "number" && limit.context > 0
? limit.context
: undefined;
maxTokens = typeof limit?.output === "number" && limit.output > 0
? limit.output
: undefined;
}
return { id, name, reasoning, contextWindow, maxTokens };
}).filter((m) => m.id.length > 0);
} finally {
clearTimeout(timeoutId);
}
}
/**
* Parses and validates the body of an update-custom-provider request.
* Returns an object with only the fields that were provided for partial updates.
* @throws {ApiError} with status 400 if provided fields are invalid.
*/
function parseUpdateBody(body: unknown): Partial<Omit<CustomProvider, "id">> {
if (!body || typeof body !== "object") {
throw badRequest("request body must be an object");
@@ -133,6 +419,11 @@ function parseUpdateBody(body: unknown): Partial<Omit<CustomProvider, "id">> {
return updates;
}
/**
* Registers custom provider CRUD routes and the probe-models endpoint.
* Routes are ordered so that static paths (probe-models) are registered after
* parameterized paths (:id) to avoid Express route conflicts.
*/
export const registerCustomProviderRoutes: ApiRouteRegistrar = (ctx) => {
const { router, store, rethrowAsApiError } = ctx;
@@ -244,5 +535,42 @@ export const registerCustomProviderRoutes: ApiRouteRegistrar = (ctx) => {
}
rethrowAsApiError(err);
}
});
// NOTE: probe-models must be registered AFTER the :id param routes
// so Express does not match "probe-models" as an :id value.
router.post("/custom-providers/probe-models", async (req, res) => {
try {
if (!req.body || typeof req.body !== "object") {
throw badRequest("request body must be an object");
}
const body = req.body as Record<string, unknown>;
const baseUrl = assertBaseUrl(body.baseUrl);
const apiKey =
typeof body.apiKey === "string" && body.apiKey.trim().length > 0
? body.apiKey.trim()
: undefined;
const rawApiType = body.apiType as string | undefined;
if (
rawApiType !== "openai-compatible" &&
rawApiType !== "anthropic-compatible" &&
rawApiType !== "google-generative-ai"
) {
throw badRequest(
"apiType must be 'openai-compatible', 'anthropic-compatible', or 'google-generative-ai'",
);
}
const apiType = rawApiType as ProbeApiType;
const models = await probeProviderModels(baseUrl, apiKey, apiType);
res.json({ models, count: models.length });
} catch (err: unknown) {
if (err instanceof ApiError) {
throw err;
}
rethrowAsApiError(err);
}
});
};