feat: auto-detect models from custom providers
Add 'Detect Models' button to custom provider forms that calls the provider's /models endpoint to discover available models automatically. - Supports OpenAI-compatible, Anthropic-compatible, and Google Generative AI providers - Auto-fills context window and max tokens from provider response - Filters out embedding, reranking, and non-text models - Removes empty default model rows after detection - Added comprehensive backend and frontend test coverage
This commit is contained in:
Islam Nofl
@@ -1868,6 +1868,41 @@ export function createCustomProvider(config: CustomProviderConfig): Promise<Cust
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe a custom provider's /models endpoint to discover available models.
|
||||
* Only works for OpenAI-compatible providers (the /models endpoint is an
|
||||
* OpenAI convention). Returns the list of models found at the provider.
|
||||
*/
|
||||
export interface ProbeModelResult {
|
||||
id: string;
|
||||
name: string;
|
||||
reasoning?: boolean;
|
||||
contextWindow?: number;
|
||||
maxTokens?: number;
|
||||
}
|
||||
|
||||
export interface ProbeModelsResponse {
|
||||
models: ProbeModelResult[];
|
||||
count: number;
|
||||
}
|
||||
|
||||
export interface ProbeModelsParams {
|
||||
baseUrl: string;
|
||||
apiKey?: string;
|
||||
apiType: "openai-compatible" | "anthropic-compatible" | "google-generative-ai";
|
||||
}
|
||||
|
||||
export async function probeProviderModels(params: ProbeModelsParams): Promise<ProbeModelsResponse> {
|
||||
return api<ProbeModelsResponse>("/custom-providers/probe-models", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
baseUrl: params.baseUrl,
|
||||
apiKey: params.apiKey,
|
||||
apiType: params.apiType,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
/** Fetch authentication status for all OAuth providers */
|
||||
export function fetchAuthStatus(): Promise<{
|
||||
providers: AuthProvider[];
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { useMemo, useState } from "react";
|
||||
import { useCallback, useMemo, useState } from "react";
|
||||
import { Loader2, Search } from "lucide-react";
|
||||
import type { CustomProviderConfig, CustomProviderModelInput } from "../api";
|
||||
import { probeProviderModels } from "../api";
|
||||
import "./CustomProviderForm.css";
|
||||
|
||||
// Reserved built-in IDs (including hidden/deprecated aliases) to prevent custom-provider collisions.
|
||||
@@ -42,6 +44,8 @@ export function CustomProviderForm({ initialConfig, onSave, onCancel, saving = f
|
||||
const [apiKey, setApiKey] = useState(initialConfig?.apiKey ?? "");
|
||||
const [models, setModels] = useState<CustomProviderModelInput[]>(initialConfig?.models?.length ? initialConfig.models : [emptyModel()]);
|
||||
const [validationError, setValidationError] = useState<string | null>(null);
|
||||
const [detecting, setDetecting] = useState(false);
|
||||
const [detectError, setDetectError] = useState<string | null>(null);
|
||||
|
||||
const canRemoveModel = models.length > 1;
|
||||
|
||||
@@ -55,6 +59,68 @@ export function CustomProviderForm({ initialConfig, onSave, onCancel, saving = f
|
||||
setModels((prev) => (prev.length <= 1 ? prev : prev.filter((_, i) => i !== index)));
|
||||
}
|
||||
|
||||
// Detect Models is available for all API types that expose a /models endpoint:
|
||||
// - openai-completions / openai-responses → openai-compatible
|
||||
// - anthropic-messages → anthropic-compatible
|
||||
// - google-generative-ai → google-generative-ai
|
||||
const probeApiType = api === "anthropic-messages"
|
||||
? "anthropic-compatible"
|
||||
: api === "google-generative-ai"
|
||||
? "google-generative-ai"
|
||||
: "openai-compatible";
|
||||
|
||||
const handleDetectModels = useCallback(async () => {
|
||||
const trimmedBaseUrl = baseUrl.trim();
|
||||
if (!trimmedBaseUrl) {
|
||||
setDetectError("Base URL is required to detect models.");
|
||||
return;
|
||||
}
|
||||
|
||||
setDetecting(true);
|
||||
setDetectError(null);
|
||||
|
||||
try {
|
||||
const result = await probeProviderModels({
|
||||
baseUrl: trimmedBaseUrl,
|
||||
apiKey: apiKey.trim() || undefined,
|
||||
apiType: probeApiType,
|
||||
});
|
||||
|
||||
if (result.models.length === 0) {
|
||||
setDetectError("No models found. The provider may require an API key.");
|
||||
return;
|
||||
}
|
||||
|
||||
// Merge discovered models, avoiding duplicates by ID
|
||||
const existingIds = new Set(models.map((m) => m.id.trim()));
|
||||
const newModels = result.models
|
||||
.filter((m) => !existingIds.has(m.id.trim()))
|
||||
.map((m) => ({
|
||||
id: m.id,
|
||||
name: m.name || m.id,
|
||||
reasoning: Boolean(m.reasoning),
|
||||
contextWindow: m.contextWindow,
|
||||
maxTokens: m.maxTokens,
|
||||
}));
|
||||
|
||||
if (newModels.length > 0) {
|
||||
// Replace empty default rows with discovered models
|
||||
setModels((prev) => {
|
||||
const nonEmpty = prev.filter((m) => m.id.trim().length > 0);
|
||||
return [...nonEmpty, ...newModels];
|
||||
});
|
||||
} else {
|
||||
setDetectError("All discovered models are already in the list.");
|
||||
}
|
||||
} catch (err) {
|
||||
setDetectError(
|
||||
err instanceof Error ? err.message : "Failed to detect models",
|
||||
);
|
||||
} finally {
|
||||
setDetecting(false);
|
||||
}
|
||||
}, [baseUrl, apiKey, probeApiType, models]);
|
||||
|
||||
function validate(): string | null {
|
||||
if (!id.trim()) return "Provider ID is required.";
|
||||
if (!PROVIDER_ID_PATTERN.test(id.trim())) return "Provider ID must be kebab-case.";
|
||||
@@ -187,9 +253,29 @@ export function CustomProviderForm({ initialConfig, onSave, onCancel, saving = f
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
<button type="button" className="btn btn-sm" onClick={() => setModels((prev) => [...prev, emptyModel()])} disabled={saving}>
|
||||
+ Add model
|
||||
</button>
|
||||
<div className="custom-provider-form__model-actions" style={{ display: "flex", gap: "8px", alignItems: "center" }}>
|
||||
<button type="button" className="btn btn-sm" onClick={() => setModels((prev) => [...prev, emptyModel()])} disabled={saving}>
|
||||
+ Add model
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm"
|
||||
onClick={() => void handleDetectModels()}
|
||||
disabled={saving || detecting || !baseUrl.trim()}
|
||||
title="Call the provider's /models endpoint to discover available models"
|
||||
>
|
||||
{detecting ? (
|
||||
<>
|
||||
<Loader2 className="spin" size={14} /> Detecting…
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Search size={14} /> Detect Models
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
{detectError ? <div className="form-error" style={{ marginTop: "4px" }}>{detectError}</div> : null}
|
||||
</div>
|
||||
|
||||
{mergedError ? <div className="form-error">{mergedError}</div> : null}
|
||||
|
||||
@@ -3,10 +3,11 @@ import {
|
||||
addCustomProvider,
|
||||
deleteCustomProvider,
|
||||
fetchCustomProviders,
|
||||
probeProviderModels,
|
||||
updateCustomProvider,
|
||||
type CustomProvider,
|
||||
} from "../api";
|
||||
import { AlertCircle, Loader2, Pencil, Plus, Trash2 } from "lucide-react";
|
||||
import { AlertCircle, Loader2, Pencil, Plus, Search, Trash2 } from "lucide-react";
|
||||
import { OnboardingDisclosure } from "./OnboardingDisclosure";
|
||||
import "./CustomProvidersSection.css";
|
||||
|
||||
@@ -74,6 +75,8 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C
|
||||
const [models, setModels] = useState("");
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [formError, setFormError] = useState<string | null>(null);
|
||||
const [detecting, setDetecting] = useState(false);
|
||||
const [detectError, setDetectError] = useState<string | null>(null);
|
||||
|
||||
const loadProviders = useCallback(async () => {
|
||||
setLoading(true);
|
||||
@@ -112,6 +115,8 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C
|
||||
setApiKey("");
|
||||
setModels("");
|
||||
setFormError(null);
|
||||
setDetectError(null);
|
||||
setDetecting(false);
|
||||
setIsFormOpen(false);
|
||||
}, []);
|
||||
|
||||
@@ -123,6 +128,8 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C
|
||||
setApiKey("");
|
||||
setModels("");
|
||||
setFormError(null);
|
||||
setDetectError(null);
|
||||
setDetecting(false);
|
||||
setIsFormOpen(true);
|
||||
}, []);
|
||||
|
||||
@@ -134,6 +141,8 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C
|
||||
setApiKey(provider.apiKey ?? "");
|
||||
setModels((provider.models ?? []).map((model) => model.id).join(", "));
|
||||
setFormError(null);
|
||||
setDetectError(null);
|
||||
setDetecting(false);
|
||||
setIsFormOpen(true);
|
||||
}, []);
|
||||
|
||||
@@ -165,6 +174,45 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C
|
||||
return null;
|
||||
}, [apiType, baseUrl, name]);
|
||||
|
||||
// Detect Models is available for all API types that expose a /models endpoint
|
||||
const handleDetectModels = useCallback(async () => {
|
||||
const trimmedBaseUrl = baseUrl.trim();
|
||||
if (!trimmedBaseUrl) {
|
||||
setDetectError("Base URL is required to detect models.");
|
||||
return;
|
||||
}
|
||||
|
||||
setDetecting(true);
|
||||
setDetectError(null);
|
||||
|
||||
try {
|
||||
const result = await probeProviderModels({
|
||||
baseUrl: trimmedBaseUrl,
|
||||
apiKey: apiKey.trim() || undefined,
|
||||
apiType,
|
||||
});
|
||||
|
||||
if (result.models.length > 0) {
|
||||
const discoveredIds = result.models.map((m) => m.id).join(", ");
|
||||
setModels((prev) => {
|
||||
const existing = prev.trim();
|
||||
if (existing) {
|
||||
return discoveredIds + ", " + existing;
|
||||
}
|
||||
return discoveredIds;
|
||||
});
|
||||
} else {
|
||||
setDetectError("No models found. The provider may require an API key.");
|
||||
}
|
||||
} catch (err) {
|
||||
setDetectError(
|
||||
err instanceof Error ? err.message : "Failed to detect models",
|
||||
);
|
||||
} finally {
|
||||
setDetecting(false);
|
||||
}
|
||||
}, [baseUrl, apiKey, apiType]);
|
||||
|
||||
const handleSave = useCallback(async () => {
|
||||
const validationError = validateForm();
|
||||
setFormError(validationError);
|
||||
@@ -327,6 +375,27 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div style={{ display: "flex", gap: "8px", alignItems: "center", marginTop: "4px" }}>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm"
|
||||
onClick={() => void handleDetectModels()}
|
||||
disabled={saving || detecting || !baseUrl.trim()}
|
||||
title="Auto-detect models from the provider's /models endpoint"
|
||||
>
|
||||
{detecting ? (
|
||||
<>
|
||||
<Loader2 className="spin" size={14} /> Detecting…
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Search size={14} /> Detect Models
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
{detectError ? <div className="custom-provider-form-error">{detectError}</div> : null}
|
||||
|
||||
{formError ? <div className="custom-provider-form-error">{formError}</div> : null}
|
||||
|
||||
<div className="custom-provider-form-actions">
|
||||
@@ -421,6 +490,27 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div style={{ display: "flex", gap: "8px", alignItems: "center", marginTop: "4px" }}>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm"
|
||||
onClick={() => void handleDetectModels()}
|
||||
disabled={saving || detecting || !baseUrl.trim()}
|
||||
title="Auto-detect models from the provider's /models endpoint"
|
||||
>
|
||||
{detecting ? (
|
||||
<>
|
||||
<Loader2 className="spin" size={14} /> Detecting…
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Search size={14} /> Detect Models
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
{detectError ? <div className="custom-provider-form-error">{detectError}</div> : null}
|
||||
|
||||
{formError ? <div className="custom-provider-form-error">{formError}</div> : null}
|
||||
|
||||
<div className="custom-provider-form-actions">
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { describe, it, expect, vi } from "vitest";
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { render, screen } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { CustomProviderForm } from "../CustomProviderForm";
|
||||
import * as api from "../../api";
|
||||
|
||||
describe("CustomProviderForm", () => {
|
||||
it("renders base fields", () => {
|
||||
@@ -58,3 +59,187 @@ describe("CustomProviderForm", () => {
|
||||
expect(screen.getByText("Request failed")).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Detect Models", () => {
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("shows the Detect Models button for openai-completions API type", () => {
|
||||
render(
|
||||
<CustomProviderForm
|
||||
onSave={vi.fn()}
|
||||
initialConfig={{
|
||||
id: "my-provider",
|
||||
baseUrl: "https://api.example.com/v1",
|
||||
api: "openai-completions",
|
||||
apiKey: "sk-test",
|
||||
models: [{ id: "gpt-4o", name: "GPT 4o" }],
|
||||
}}
|
||||
/>
|
||||
);
|
||||
expect(screen.getByRole("button", { name: /detect models/i })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows the Detect Models button for openai-responses API type", () => {
|
||||
render(
|
||||
<CustomProviderForm
|
||||
onSave={vi.fn()}
|
||||
initialConfig={{
|
||||
id: "my-provider",
|
||||
baseUrl: "https://api.example.com/v1",
|
||||
api: "openai-responses",
|
||||
apiKey: "sk-test",
|
||||
models: [{ id: "gpt-4o", name: "GPT 4o" }],
|
||||
}}
|
||||
/>
|
||||
);
|
||||
expect(screen.getByRole("button", { name: /detect models/i })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows the Detect Models button for anthropic-messages API type", () => {
|
||||
render(
|
||||
<CustomProviderForm
|
||||
onSave={vi.fn()}
|
||||
initialConfig={{
|
||||
id: "my-provider",
|
||||
baseUrl: "https://api.anthropic.com",
|
||||
api: "anthropic-messages",
|
||||
apiKey: "sk-ant-test",
|
||||
models: [{ id: "claude-3", name: "Claude 3" }],
|
||||
}}
|
||||
/>
|
||||
);
|
||||
expect(screen.getByRole("button", { name: /detect models/i })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows the Detect Models button for google-generative-ai API type", () => {
|
||||
render(
|
||||
<CustomProviderForm
|
||||
onSave={vi.fn()}
|
||||
initialConfig={{
|
||||
id: "my-provider",
|
||||
baseUrl: "https://generativelanguage.googleapis.com",
|
||||
api: "google-generative-ai",
|
||||
apiKey: "sk-google",
|
||||
models: [{ id: "gemini-pro", name: "Gemini Pro" }],
|
||||
}}
|
||||
/>
|
||||
);
|
||||
expect(screen.getByRole("button", { name: /detect models/i })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("calls probeProviderModels and adds discovered models", async () => {
|
||||
const mockProbe = vi.spyOn(api, "probeProviderModels").mockResolvedValue({
|
||||
models: [
|
||||
{ id: "gpt-4o", name: "GPT 4o", reasoning: false },
|
||||
{ id: "gpt-4", name: "GPT 4", reasoning: false },
|
||||
],
|
||||
count: 2,
|
||||
});
|
||||
|
||||
render(
|
||||
<CustomProviderForm
|
||||
onSave={vi.fn()}
|
||||
initialConfig={{
|
||||
id: "my-provider",
|
||||
baseUrl: "https://api.example.com/v1",
|
||||
api: "openai-completions",
|
||||
apiKey: "sk-test",
|
||||
models: [{ id: "", name: "", reasoning: false }],
|
||||
}}
|
||||
/>
|
||||
);
|
||||
|
||||
const user = userEvent.setup();
|
||||
await user.click(screen.getByRole("button", { name: /detect models/i }));
|
||||
|
||||
expect(mockProbe).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
baseUrl: "https://api.example.com/v1",
|
||||
apiKey: "sk-test",
|
||||
apiType: "openai-compatible",
|
||||
})
|
||||
);
|
||||
|
||||
// Models should be added to the list
|
||||
expect(screen.getByDisplayValue("gpt-4o")).toBeInTheDocument();
|
||||
expect(screen.getByDisplayValue("gpt-4")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("deduplicates models when detecting", async () => {
|
||||
const mockProbe = vi.spyOn(api, "probeProviderModels").mockResolvedValue({
|
||||
models: [
|
||||
{ id: "gpt-4o", name: "GPT 4o", reasoning: false },
|
||||
{ id: "gpt-4", name: "GPT 4", reasoning: false },
|
||||
],
|
||||
count: 2,
|
||||
});
|
||||
|
||||
render(
|
||||
<CustomProviderForm
|
||||
onSave={vi.fn()}
|
||||
initialConfig={{
|
||||
id: "my-provider",
|
||||
baseUrl: "https://api.example.com/v1",
|
||||
api: "openai-completions",
|
||||
apiKey: "sk-test",
|
||||
models: [
|
||||
{ id: "gpt-4o", name: "GPT 4o", reasoning: false },
|
||||
{ id: "", name: "", reasoning: false },
|
||||
],
|
||||
}}
|
||||
/>
|
||||
);
|
||||
|
||||
const user = userEvent.setup();
|
||||
await user.click(screen.getByRole("button", { name: /detect models/i }));
|
||||
|
||||
// gpt-4o should appear only once (existing + deduplicated)
|
||||
const gpt4oInputs = screen.queryAllByDisplayValue("gpt-4o");
|
||||
expect(gpt4oInputs).toHaveLength(1);
|
||||
// gpt-4 should be added
|
||||
expect(screen.getByDisplayValue("gpt-4")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows error when detection fails", async () => {
|
||||
const mockProbe = vi.spyOn(api, "probeProviderModels").mockRejectedValue(
|
||||
new Error("Provider returned 401 Unauthorized")
|
||||
);
|
||||
|
||||
render(
|
||||
<CustomProviderForm
|
||||
onSave={vi.fn()}
|
||||
initialConfig={{
|
||||
id: "my-provider",
|
||||
baseUrl: "https://api.example.com/v1",
|
||||
api: "openai-completions",
|
||||
apiKey: "sk-invalid",
|
||||
models: [{ id: "", name: "", reasoning: false }],
|
||||
}}
|
||||
/>
|
||||
);
|
||||
|
||||
const user = userEvent.setup();
|
||||
await user.click(screen.getByRole("button", { name: /detect models/i }));
|
||||
|
||||
expect(screen.getByText("Provider returned 401 Unauthorized")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("disables button when baseUrl is empty", () => {
|
||||
render(
|
||||
<CustomProviderForm
|
||||
onSave={vi.fn()}
|
||||
initialConfig={{
|
||||
id: "my-provider",
|
||||
baseUrl: "",
|
||||
api: "openai-completions",
|
||||
apiKey: "sk-test",
|
||||
models: [{ id: "", name: "", reasoning: false }],
|
||||
}}
|
||||
/>
|
||||
);
|
||||
const detectBtn = screen.getByRole("button", { name: /detect models/i });
|
||||
expect(detectBtn).toBeDisabled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -20,6 +20,7 @@ vi.mock("lucide-react", () => ({
|
||||
Loader2: ({ className }: { className?: string }) => <svg data-testid="icon-loader" className={className} />,
|
||||
Pencil: () => <svg data-testid="icon-pencil" />,
|
||||
Plus: () => <svg data-testid="icon-plus" />,
|
||||
Search: () => <svg data-testid="icon-search" />,
|
||||
Trash2: () => <svg data-testid="icon-trash" />,
|
||||
}));
|
||||
|
||||
|
||||
@@ -352,3 +352,281 @@ describe("custom providers API routes", () => {
|
||||
expect(String(res.body.error)).toContain("not found");
|
||||
});
|
||||
});
|
||||
|
||||
describe("POST /api/custom-providers/probe-models", () => {
|
||||
let mockFetch: ReturnType<typeof vi.fn>;
|
||||
|
||||
beforeEach(() => {
|
||||
mockFetch = vi.fn();
|
||||
vi.stubGlobal("fetch", mockFetch);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("returns OpenAI-compatible models", async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [
|
||||
{ id: "gpt-4o", object: "model", owned_by: "system" },
|
||||
{ id: "gpt-4", object: "model", owned_by: "system" },
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
const app = setupApp(createCustomProviderStore().store);
|
||||
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiType: "openai-compatible",
|
||||
apiKey: "sk-test",
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual({
|
||||
count: 2,
|
||||
models: [
|
||||
{ id: "gpt-4o", name: "gpt-4o", reasoning: false },
|
||||
{ id: "gpt-4", name: "gpt-4", reasoning: false },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("returns Anthropic-compatible models", async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [
|
||||
{ id: "claude-sonnet-4-20250514", object: "model", display_name: "Claude Sonnet 4" },
|
||||
{ id: "claude-haiku-4-5-20251001", object: "model", display_name: "Claude Haiku 4.5" },
|
||||
{ id: "claude-opus-4-20250514", object: "model", display_name: "Claude Opus 4" },
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
const app = setupApp(createCustomProviderStore().store);
|
||||
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
|
||||
baseUrl: "https://api.anthropic.com",
|
||||
apiType: "anthropic-compatible",
|
||||
apiKey: "sk-ant-test",
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.count).toBe(3);
|
||||
expect(res.body.models[0]).toEqual({
|
||||
id: "claude-sonnet-4-20250514",
|
||||
name: "Claude Sonnet 4",
|
||||
reasoning: true, // sonnet detected as reasoning
|
||||
});
|
||||
expect(res.body.models[2]).toEqual({
|
||||
id: "claude-opus-4-20250514",
|
||||
name: "Claude Opus 4",
|
||||
reasoning: true, // opus detected as reasoning
|
||||
});
|
||||
});
|
||||
|
||||
it("returns Google Generative AI models", async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
models: [
|
||||
{
|
||||
name: "models/gemini-2.0-flash",
|
||||
baseModelId: "gemini-2.0-flash",
|
||||
displayName: "Gemini 2.0 Flash",
|
||||
inputTokenLimit: 1048576,
|
||||
outputTokenLimit: 8192,
|
||||
supportedGenerationMethods: ["generateContent"],
|
||||
},
|
||||
{
|
||||
name: "models/text-embedding-004",
|
||||
baseModelId: "text-embedding-004",
|
||||
displayName: "Text Embedding",
|
||||
supportedGenerationMethods: ["embedContent"],
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
const app = setupApp(createCustomProviderStore().store);
|
||||
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
|
||||
baseUrl: "https://generativelanguage.googleapis.com",
|
||||
apiType: "google-generative-ai",
|
||||
apiKey: "AIza-test",
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
// Embedding model should be filtered out
|
||||
expect(res.body.count).toBe(1);
|
||||
expect(res.body.models[0]).toEqual({
|
||||
id: "gemini-2.0-flash",
|
||||
name: "Gemini 2.0 Flash",
|
||||
reasoning: false,
|
||||
contextWindow: 1048576,
|
||||
maxTokens: 8192,
|
||||
});
|
||||
});
|
||||
|
||||
it("excludes embedding models from OpenAI-compatible response", async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [
|
||||
{ id: "gpt-4o", object: "model", modalities: { input: ["text"], output: ["text"] } },
|
||||
{ id: "text-embedding-3", object: "model", modalities: { input: ["text"], output: ["embedding"] } },
|
||||
{ id: "whisper-large", object: "model", modalities: { input: ["audio"], output: ["text"] } },
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
const app = setupApp(createCustomProviderStore().store);
|
||||
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
|
||||
baseUrl: "https://api.example.com/v1",
|
||||
apiType: "openai-compatible",
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.count).toBe(1); // embedding + audio-input both excluded
|
||||
expect(res.body.models[0].id).toBe("gpt-4o");
|
||||
});
|
||||
|
||||
it("excludes models without text input from OpenAI-compatible response", async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [
|
||||
{ id: "gpt-4o", object: "model", modalities: { input: ["text", "image"], output: ["text"] } },
|
||||
{ id: "scribe-v2", object: "model", modalities: { input: ["audio"], output: ["text"] } },
|
||||
{ id: "eleven-v3", object: "model", modalities: { input: ["text"], output: ["audio"] } },
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
const app = setupApp(createCustomProviderStore().store);
|
||||
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
|
||||
baseUrl: "https://api.example.com/v1",
|
||||
apiType: "openai-compatible",
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.count).toBe(1); // only gpt-4o has text input + text output
|
||||
expect(res.body.models[0].id).toBe("gpt-4o");
|
||||
});
|
||||
|
||||
it("rejects invalid apiType for probe", async () => {
|
||||
const app = setupApp(createCustomProviderStore().store);
|
||||
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
|
||||
baseUrl: "https://api.example.com",
|
||||
apiType: "invalid",
|
||||
});
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it("detects reasoning models from ID", async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [
|
||||
{ id: "o1-preview", object: "model" },
|
||||
{ id: "o3-mini", object: "model" },
|
||||
{ id: "gpt-4o", object: "model" },
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
const app = setupApp(createCustomProviderStore().store);
|
||||
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiType: "openai-compatible",
|
||||
});
|
||||
|
||||
expect(res.body.models[0].reasoning).toBe(true); // o1-preview
|
||||
expect(res.body.models[1].reasoning).toBe(true); // o3-mini
|
||||
expect(res.body.models[2].reasoning).toBe(false); // gpt-4o
|
||||
});
|
||||
|
||||
it("returns 400 for missing baseUrl", async () => {
|
||||
const app = setupApp(createCustomProviderStore().store);
|
||||
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
|
||||
apiType: "openai-compatible",
|
||||
});
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it("returns 400 for invalid URL", async () => {
|
||||
const app = setupApp(createCustomProviderStore().store);
|
||||
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
|
||||
baseUrl: "not-a-url",
|
||||
apiType: "openai-compatible",
|
||||
});
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it("returns error when provider returns non-200", async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
status: 401,
|
||||
statusText: "Unauthorized",
|
||||
text: async () => "Invalid API key",
|
||||
});
|
||||
|
||||
const app = setupApp(createCustomProviderStore().store);
|
||||
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiType: "openai-compatible",
|
||||
apiKey: "sk-invalid",
|
||||
});
|
||||
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
it("handles { models: [...] } response format", async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
models: [
|
||||
{ id: "llama-3.1-8b", name: "Llama 3.1 8B" },
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
const app = setupApp(createCustomProviderStore().store);
|
||||
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
|
||||
baseUrl: "https://api.example.com",
|
||||
apiType: "openai-compatible",
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.models[0]).toEqual({
|
||||
id: "llama-3.1-8b",
|
||||
name: "Llama 3.1 8B",
|
||||
reasoning: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("truncates large model lists to 100", async () => {
|
||||
const manyModels = Array.from({ length: 150 }, (_, i) => ({
|
||||
id: `model-${i}`,
|
||||
object: "model",
|
||||
}));
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({ data: manyModels }),
|
||||
});
|
||||
|
||||
const app = setupApp(createCustomProviderStore().store);
|
||||
const res = await doRequest(app, "POST", "/api/custom-providers/probe-models", {
|
||||
baseUrl: "https://api.example.com",
|
||||
apiType: "openai-compatible",
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.count).toBe(100);
|
||||
expect(res.body.models.length).toBe(100);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -103,6 +103,199 @@ function parseCreateBody(body: unknown): Omit<CustomProvider, "id"> {
|
||||
return provider;
|
||||
}
|
||||
|
||||
interface ProbeModelResult {
|
||||
id: string;
|
||||
name: string;
|
||||
reasoning?: boolean;
|
||||
contextWindow?: number;
|
||||
maxTokens?: number;
|
||||
}
|
||||
|
||||
const MAX_PROBE_MODELS = 100;
|
||||
|
||||
type ProbeApiType = "openai-compatible" | "anthropic-compatible" | "google-generative-ai";
|
||||
|
||||
/**
|
||||
* Check if a model should be excluded (embedding / reranking / audio-only / no-text-input models).
|
||||
*/
|
||||
function isNonChatModel(m: Record<string, unknown>): boolean {
|
||||
// OpenAI-compatible modalities: { input: ["text"], output: ["embedding"] }
|
||||
const modalities = m.modalities as Record<string, unknown> | undefined;
|
||||
if (modalities) {
|
||||
// Exclude models that don't accept text input (e.g. audio-only, image-only)
|
||||
if (Array.isArray(modalities.input)) {
|
||||
const inputs = modalities.input.map((i: unknown) => String(i).toLowerCase());
|
||||
if (!inputs.includes("text")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (Array.isArray(modalities.output)) {
|
||||
const outputs = modalities.output.map((o: unknown) => String(o).toLowerCase());
|
||||
if (outputs.includes("embedding") || outputs.includes("scores")) {
|
||||
return true;
|
||||
}
|
||||
// Exclude models that don't produce text output (e.g. audio-only)
|
||||
if (!outputs.includes("text")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Google supportedGenerationMethods: no generateContent = not a chat model
|
||||
const methods = m.supportedGenerationMethods as unknown[] | undefined;
|
||||
if (Array.isArray(methods) && methods.length > 0 && !methods.includes("generateContent")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Heuristic: model ID contains embedding / rerank
|
||||
const id = String(m.id ?? m.name ?? "").toLowerCase();
|
||||
if (id.includes("embedding") || id.includes("embed-") || id.includes("-embed-") || id.includes("rerank")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe a custom provider's /models endpoint to discover available models.
|
||||
* Supports OpenAI-compatible, Anthropic-compatible, and Google Generative AI providers.
|
||||
*/
|
||||
async function probeProviderModels(
|
||||
baseUrl: string,
|
||||
apiKey: string | undefined,
|
||||
apiType: ProbeApiType,
|
||||
): Promise<ProbeModelResult[]> {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(baseUrl);
|
||||
} catch {
|
||||
throw badRequest("baseUrl must be a valid URL");
|
||||
}
|
||||
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
||||
throw badRequest("baseUrl must use http or https");
|
||||
}
|
||||
|
||||
let modelsUrl: string;
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "Fusion/1.0",
|
||||
};
|
||||
|
||||
if (apiType === "openai-compatible") {
|
||||
// OpenAI-compatible: /v1/models relative to baseUrl
|
||||
const pathname = url.pathname.replace(/\/+$/, "");
|
||||
const modelsPath = pathname ? pathname + "/models" : "/models";
|
||||
modelsUrl = new URL(modelsPath, url.origin).toString();
|
||||
if (apiKey) headers["Authorization"] = `Bearer ${apiKey}`;
|
||||
} else if (apiType === "anthropic-compatible") {
|
||||
// Anthropic: GET /v1/models with x-api-key header
|
||||
const pathname = url.pathname.replace(/\/+$/, "");
|
||||
const modelsPath = pathname ? pathname + "/models" : "/v1/models";
|
||||
modelsUrl = new URL(modelsPath, url.origin).toString();
|
||||
if (apiKey) headers["x-api-key"] = apiKey;
|
||||
headers["anthropic-version"] = "2023-06-01";
|
||||
} else {
|
||||
// Google Generative AI: GET /v1beta/models?key=API_KEY
|
||||
const pathname = url.pathname.replace(/\/+$/, "");
|
||||
const modelsPath = pathname ? pathname + "/models" : "/v1beta/models";
|
||||
modelsUrl = new URL(modelsPath, url.origin).toString();
|
||||
if (apiKey) {
|
||||
// Append API key as query parameter (Google convention)
|
||||
const separator = modelsUrl.includes("?") ? "&" : "?";
|
||||
modelsUrl = `${modelsUrl}${separator}key=${encodeURIComponent(apiKey)}`;
|
||||
}
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), 10_000);
|
||||
|
||||
try {
|
||||
const response = await fetch(modelsUrl, {
|
||||
method: "GET",
|
||||
headers,
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorBody = await response.text().catch(() => "");
|
||||
const message = errorBody.slice(0, 200);
|
||||
throw new ApiError(
|
||||
response.status,
|
||||
`Provider returned ${response.status} ${response.statusText}${message ? `: ${message}` : ""}`,
|
||||
);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const rawModels = data?.data ?? data?.models ?? [];
|
||||
|
||||
if (!Array.isArray(rawModels) || rawModels.length === 0) {
|
||||
throw new ApiError(404, "No models found in provider response");
|
||||
}
|
||||
|
||||
// Filter out embedding/reranking/audio-only models and truncate
|
||||
const chatModels = rawModels.filter((m: Record<string, unknown>) => !isNonChatModel(m));
|
||||
const trimmed = chatModels.length > MAX_PROBE_MODELS ? chatModels.slice(0, MAX_PROBE_MODELS) : chatModels;
|
||||
|
||||
return trimmed.map((m: Record<string, unknown>) => {
|
||||
// Extract ID based on provider format
|
||||
let id: string;
|
||||
let name: string;
|
||||
let contextWindow: number | undefined;
|
||||
let maxTokens: number | undefined;
|
||||
let reasoning: boolean;
|
||||
|
||||
if (apiType === "google-generative-ai") {
|
||||
// Google: name = "models/gemini-2.0-flash", baseModelId = "gemini-2.0-flash"
|
||||
id = String(m.baseModelId ?? m.name ?? "");
|
||||
// Strip "models/" prefix if present
|
||||
if (id.startsWith("models/")) id = id.slice(7);
|
||||
name = String(m.displayName ?? id);
|
||||
contextWindow = typeof m.inputTokenLimit === "number" && m.inputTokenLimit > 0
|
||||
? m.inputTokenLimit
|
||||
: undefined;
|
||||
maxTokens = typeof m.outputTokenLimit === "number" && m.outputTokenLimit > 0
|
||||
? m.outputTokenLimit
|
||||
: undefined;
|
||||
reasoning = Boolean(m.thinking);
|
||||
} else if (apiType === "anthropic-compatible") {
|
||||
// Anthropic: id = "claude-sonnet-4-20250514", display_name = "Claude Sonnet 4"
|
||||
id = String(m.id ?? "");
|
||||
name = String(m.display_name ?? id);
|
||||
// Anthropic doesn't return context/max_tokens in the models list
|
||||
reasoning = Boolean(
|
||||
id.toLowerCase().includes("opus") ||
|
||||
id.toLowerCase().includes("sonnet"),
|
||||
);
|
||||
} else {
|
||||
// OpenAI-compatible
|
||||
id = String(m.id ?? "");
|
||||
name = String(m.name ?? m.display_name ?? id);
|
||||
reasoning = Boolean(
|
||||
m.reasoning ||
|
||||
(Array.isArray(m.capabilities) && m.capabilities.includes("reasoning")) ||
|
||||
id.toLowerCase().includes("reason") ||
|
||||
id.toLowerCase().includes("o1") ||
|
||||
id.toLowerCase().includes("o3"),
|
||||
);
|
||||
// Extract context window and max tokens from limit object
|
||||
const limit = m.limit as Record<string, unknown> | undefined;
|
||||
contextWindow = typeof limit?.context === "number" && limit.context > 0
|
||||
? limit.context
|
||||
: undefined;
|
||||
maxTokens = typeof limit?.output === "number" && limit.output > 0
|
||||
? limit.output
|
||||
: undefined;
|
||||
}
|
||||
|
||||
return { id, name, reasoning, contextWindow, maxTokens };
|
||||
}).filter((m) => m.id.length > 0);
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
|
||||
function parseUpdateBody(body: unknown): Partial<Omit<CustomProvider, "id">> {
|
||||
if (!body || typeof body !== "object") {
|
||||
throw badRequest("request body must be an object");
|
||||
@@ -245,4 +438,39 @@ export const registerCustomProviderRoutes: ApiRouteRegistrar = (ctx) => {
|
||||
rethrowAsApiError(err);
|
||||
}
|
||||
});
|
||||
|
||||
// NOTE: probe-models must be registered AFTER the :id param routes
|
||||
// so Express does not match "probe-models" as an :id value.
|
||||
router.post("/custom-providers/probe-models", async (req, res) => {
|
||||
try {
|
||||
const body = req.body as Record<string, unknown>;
|
||||
|
||||
const baseUrl = assertBaseUrl(body.baseUrl);
|
||||
const apiKey =
|
||||
typeof body.apiKey === "string" && body.apiKey.trim().length > 0
|
||||
? body.apiKey.trim()
|
||||
: undefined;
|
||||
|
||||
// Probe endpoint accepts all three API types
|
||||
const rawApiType = body.apiType as string | undefined;
|
||||
if (
|
||||
rawApiType !== "openai-compatible" &&
|
||||
rawApiType !== "anthropic-compatible" &&
|
||||
rawApiType !== "google-generative-ai"
|
||||
) {
|
||||
throw badRequest(
|
||||
"apiType must be 'openai-compatible', 'anthropic-compatible', or 'google-generative-ai'",
|
||||
);
|
||||
}
|
||||
const apiType = rawApiType as ProbeApiType;
|
||||
|
||||
const models = await probeProviderModels(baseUrl, apiKey, apiType);
|
||||
res.json({ models, count: models.length });
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof ApiError) {
|
||||
throw err;
|
||||
}
|
||||
rethrowAsApiError(err);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user