FN-043: add canonical thinking levels for custom providers
Custom-provider models now expose the full thinking-level selector and faithfully transmit the selected effort. - Register all canonical thinking levels and preserve provider-specific API dialects. - Remove the redundant reasoning toggle and update provider status and settings documentation. - Add dashboard, engine, and provider registration coverage for thinking-level behavior. - Add a minor changeset for the published CLI package. Files changed: .../fn-043-custom-provider-thinking-levels.md | 7 ++ docs/settings-reference.md | 2 + .../dashboard/app/api/settings/provider-status.ts | 1 - .../app/components/CustomProviderForm.css | 5 +- .../app/components/CustomProviderForm.tsx | 18 ++--- .../__tests__/CustomProviderForm.test.tsx | 35 ++++++--- ...r-model-routes-custom-provider-thinking.test.ts | 91 ++++++++++++++++++++++ .../custom-provider-thinking-levels.test.ts | 49 ++++++++++++ .../custom-providers-openai-completions.test.ts | 2 +- .../custom-providers-openai-responses.test.ts | 3 +- .../src/__tests__/pi-create-fn-agent.test.ts | 31 +++++++- .../src/__tests__/provider-registration.test.ts | 20 ++++- .../engine/src/auth/custom-provider-registry.ts | 14 +++- packages/engine/src/pi.ts | 9 ++- 14 files changed, 251 insertions(+), 36 deletions(-) Fusion-Task-Id: FN-043 Fusion-Task-Lineage: fdd3308a-d5bd-4f54-911a-226c2b46a927 Co-authored-by: Fusion <noreply@runfusion.ai>
This commit is contained in:
Fusion Agent
@@ -0,0 +1,49 @@
|
||||
import { getSupportedThinkingLevels } from "@earendil-works/pi-ai";
|
||||
import { THINKING_LEVELS, type CustomProvider } from "@fusion/core";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildCustomProviderModels, resolveApiType } from "../auth/custom-provider-registry.js";
|
||||
|
||||
const API_TYPES = [
|
||||
"openai-compatible",
|
||||
"openai-responses",
|
||||
"anthropic-compatible",
|
||||
"google-generative-ai",
|
||||
] as const;
|
||||
|
||||
describe("custom-provider thinking levels", () => {
|
||||
it.each(API_TYPES)("registers every %s model with the canonical transmissible thinking levels", (apiType) => {
|
||||
const provider: CustomProvider = {
|
||||
id: "550e8400-e29b-41d4-a716-446655440000",
|
||||
name: "Custom Provider",
|
||||
apiType,
|
||||
baseUrl: "https://custom.example/v1",
|
||||
models: [{ id: "custom-model", name: "Custom Model" }],
|
||||
};
|
||||
|
||||
const resolvedApi = resolveApiType(apiType);
|
||||
const [model] = buildCustomProviderModels(provider, resolvedApi);
|
||||
|
||||
expect(resolvedApi).toBe(
|
||||
apiType === "openai-compatible" ? "openai-completions"
|
||||
: apiType === "anthropic-compatible" ? "anthropic-messages"
|
||||
: apiType,
|
||||
);
|
||||
expect(model).toMatchObject({
|
||||
reasoning: true,
|
||||
thinkingLevelMap: { xhigh: "xhigh", max: "max" },
|
||||
});
|
||||
expect(getSupportedThinkingLevels(model)).toEqual(THINKING_LEVELS);
|
||||
});
|
||||
|
||||
it("keeps providers with absent or empty model lists empty", () => {
|
||||
const provider = {
|
||||
id: "550e8400-e29b-41d4-a716-446655440000",
|
||||
name: "Custom Provider",
|
||||
apiType: "openai-compatible",
|
||||
baseUrl: "https://custom.example/v1",
|
||||
} as CustomProvider;
|
||||
|
||||
expect(buildCustomProviderModels(provider, "openai-completions")).toEqual([]);
|
||||
expect(buildCustomProviderModels({ ...provider, models: [] }, "openai-completions")).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -47,7 +47,7 @@ describe("custom providers openai-completions regression", () => {
|
||||
baseUrl: provider.baseUrl,
|
||||
api: "openai-completions",
|
||||
apiKey: provider.apiKey,
|
||||
models: [{ id: "my-model", name: "My Model", reasoning: false, input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 128000, maxTokens: 16384 }],
|
||||
models: [{ id: "my-model", name: "My Model", reasoning: true, thinkingLevelMap: { xhigh: "xhigh", max: "max" }, input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 128000, maxTokens: 16384 }],
|
||||
});
|
||||
await modelRegistry.refresh();
|
||||
|
||||
|
||||
@@ -53,7 +53,8 @@ describe("custom providers openai-responses regression", () => {
|
||||
models: [{
|
||||
id: "gpt-5.4",
|
||||
name: "GPT 5.4",
|
||||
reasoning: false,
|
||||
reasoning: true,
|
||||
thinkingLevelMap: { xhigh: "xhigh", max: "max" },
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 128000,
|
||||
|
||||
@@ -2041,6 +2041,14 @@ describe("createFnAgent", () => {
|
||||
anthropicPromptCaching: true,
|
||||
models: [{ id: "anthropic-model", name: "Anthropic Model" }],
|
||||
},
|
||||
{
|
||||
id: "880e8400-e29b-41d4-a716-446655440003",
|
||||
name: "Custom Google Provider",
|
||||
apiType: "google-generative-ai",
|
||||
baseUrl: "https://google.example",
|
||||
apiKey: "GOOGLE_API_KEY",
|
||||
models: [{ id: "google-model", name: "Google Model" }],
|
||||
},
|
||||
] as any);
|
||||
|
||||
const { createPiAgentSessionRaw: createFnAgent } = await import("../pi.js");
|
||||
@@ -2058,6 +2066,8 @@ describe("createFnAgent", () => {
|
||||
api: "openai-completions",
|
||||
models: [expect.objectContaining({
|
||||
id: "custom-model",
|
||||
reasoning: true,
|
||||
thinkingLevelMap: { xhigh: "xhigh", max: "max" },
|
||||
compat: expect.objectContaining({ cacheControlFormat: "anthropic" }),
|
||||
})],
|
||||
}));
|
||||
@@ -2065,16 +2075,33 @@ describe("createFnAgent", () => {
|
||||
// Opted-out (default) openai-compatible provider: no forced cache_control marker.
|
||||
const noCacheCall = registerProviderMock.mock.calls.find(([key]: [string]) => key === "custom-openai-no-caching");
|
||||
expect(noCacheCall).toBeDefined();
|
||||
const [, noCacheConfig] = noCacheCall as [string, { models: Array<{ compat?: Record<string, unknown> }> }];
|
||||
const [, noCacheConfig] = noCacheCall as [string, { models: Array<{ reasoning: boolean; thinkingLevelMap: Record<string, string>; compat?: Record<string, unknown> }> }];
|
||||
expect(noCacheConfig.models[0]).toMatchObject({
|
||||
reasoning: true,
|
||||
thinkingLevelMap: { xhigh: "xhigh", max: "max" },
|
||||
});
|
||||
expect(noCacheConfig.models[0].compat).not.toHaveProperty("cacheControlFormat");
|
||||
|
||||
// anthropic-compatible provider: opt-in is a documented no-op — pi-ai's anthropic path already
|
||||
// auto-caches without this compat flag, and openai-completions-only compat must not leak in.
|
||||
const anthropicCall = registerProviderMock.mock.calls.find(([key]: [string]) => key === "custom-anthropic-caching-opt-in");
|
||||
expect(anthropicCall).toBeDefined();
|
||||
const [, anthropicConfig] = anthropicCall as [string, { api: string; models: Array<{ compat?: Record<string, unknown> }> }];
|
||||
const [, anthropicConfig] = anthropicCall as [string, { api: string; models: Array<{ reasoning: boolean; thinkingLevelMap: Record<string, string>; compat?: Record<string, unknown> }> }];
|
||||
expect(anthropicConfig.api).toBe("anthropic-messages");
|
||||
expect(anthropicConfig.models[0]).toMatchObject({
|
||||
reasoning: true,
|
||||
thinkingLevelMap: { xhigh: "xhigh", max: "max" },
|
||||
});
|
||||
expect(anthropicConfig.models[0].compat).toBeUndefined();
|
||||
|
||||
const googleCall = registerProviderMock.mock.calls.find(([key]: [string]) => key === "custom-google-provider");
|
||||
expect(googleCall).toBeDefined();
|
||||
const [, googleConfig] = googleCall as [string, { api: string; models: Array<{ reasoning: boolean; thinkingLevelMap: Record<string, string> }> }];
|
||||
expect(googleConfig.api).toBe("google-generative-ai");
|
||||
expect(googleConfig.models[0]).toMatchObject({
|
||||
reasoning: true,
|
||||
thinkingLevelMap: { xhigh: "xhigh", max: "max" },
|
||||
});
|
||||
});
|
||||
|
||||
it("avoids lock-based SettingsManager.create when loading extension providers", async () => {
|
||||
|
||||
@@ -146,7 +146,7 @@ describe("seedDashboardProviders", () => {
|
||||
baseUrl: provider.baseUrl,
|
||||
api: "openai-completions",
|
||||
apiKey: provider.apiKey,
|
||||
models: [{ id: "acme-1", name: "Acme Model 1", reasoning: false, input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 128000, maxTokens: 16384 }],
|
||||
models: [{ id: "acme-1", name: "Acme Model 1", reasoning: true, thinkingLevelMap: { xhigh: "xhigh", max: "max" }, input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 128000, maxTokens: 16384 }],
|
||||
});
|
||||
|
||||
const registryB = await createInMemoryModelRegistry();
|
||||
@@ -184,6 +184,24 @@ describe("seedDashboardProviders", () => {
|
||||
expect(providerIds).toEqual(expect.arrayContaining(["zai", "openrouter", "acme-one", "acme-two"]));
|
||||
});
|
||||
|
||||
it("registers a google-generative-ai custom provider under the Google api key", async () => {
|
||||
const modelRegistry = makeModelRegistry();
|
||||
const provider = customProvider({ apiType: "google-generative-ai", baseUrl: "https://google.test" });
|
||||
|
||||
await registerCustomProviders(modelRegistry, [provider], vi.fn());
|
||||
|
||||
expect(modelRegistry.registerProvider).toHaveBeenCalledWith(
|
||||
"acme-ai",
|
||||
expect.objectContaining({
|
||||
api: "google-generative-ai",
|
||||
models: [expect.objectContaining({
|
||||
reasoning: true,
|
||||
thinkingLevelMap: { xhigh: "xhigh", max: "max" },
|
||||
})],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("registers an anthropic-compatible custom provider under the anthropic-messages api key (FN-7690)", async () => {
|
||||
const store = makeStore([
|
||||
customProvider({
|
||||
|
||||
@@ -17,6 +17,7 @@ interface ModelRegistryLike extends RefreshableModelRegistry {
|
||||
id: string;
|
||||
name: string;
|
||||
reasoning: boolean;
|
||||
thinkingLevelMap?: { xhigh?: string | null; max?: string | null };
|
||||
input: ("text" | "image")[];
|
||||
cost: { input: number; output: number; cacheRead: number; cacheWrite: number };
|
||||
contextWindow: number;
|
||||
@@ -50,6 +51,10 @@ export function resolveApiType(apiType: string): string {
|
||||
if (apiType === "openai-responses") {
|
||||
return "openai-responses";
|
||||
}
|
||||
// FNXC:CustomProviders 2026-08-19-15:28: Google-compatible custom providers must retain pi's Google API dialect so its shared thinking translation handles Off and every selected effort.
|
||||
if (apiType === "google-generative-ai") {
|
||||
return "google-generative-ai";
|
||||
}
|
||||
return "openai-completions";
|
||||
}
|
||||
|
||||
@@ -66,6 +71,11 @@ export function resolveApiType(apiType: string): string {
|
||||
* anthropic path already auto-caches without any flag, and `openai-responses` uses OpenAI's
|
||||
* native `prompt_cache_key`/`prompt_cache_retention` mechanism (no `cache_control` marker concept
|
||||
* per pi-ai's `OpenAIResponsesCompat`), so the opt-in is inert there by construction.
|
||||
*
|
||||
* FNXC:CustomProviders 2026-08-19-15:13:
|
||||
* Custom-provider models are presumed thinking-capable without a user-declared capability. Register
|
||||
* all seven canonical levels as transmissible so pi owns Off translation and up-then-down clamping;
|
||||
* the same registration is the source for selector display and execution.
|
||||
*/
|
||||
export function buildCustomProviderModels(
|
||||
provider: CustomProvider,
|
||||
@@ -74,6 +84,7 @@ export function buildCustomProviderModels(
|
||||
id: string;
|
||||
name: string;
|
||||
reasoning: boolean;
|
||||
thinkingLevelMap: { xhigh: string; max: string };
|
||||
input: ("text" | "image")[];
|
||||
cost: { input: number; output: number; cacheRead: number; cacheWrite: number };
|
||||
contextWindow: number;
|
||||
@@ -86,7 +97,8 @@ export function buildCustomProviderModels(
|
||||
return (provider.models ?? []).map((model) => ({
|
||||
id: model.id,
|
||||
name: model.name,
|
||||
reasoning: false,
|
||||
reasoning: true,
|
||||
thinkingLevelMap: { xhigh: "xhigh", max: "max" },
|
||||
input: ["text" as const],
|
||||
cost: {
|
||||
input: 0,
|
||||
|
||||
@@ -1147,16 +1147,23 @@ export interface AgentOptions {
|
||||
* provider register but threw "No API provider registered for api: anthropic"
|
||||
* the moment a task tried to stream.
|
||||
*
|
||||
* FNXC:CustomProviders 2026-08-19-15:28:
|
||||
* Google-compatible custom providers keep the registered Google dialect. This path and dashboard
|
||||
* registration must agree so pi performs the same thinking-level translation for every custom model.
|
||||
*
|
||||
* @param apiType - the custom provider's declared compatibility type.
|
||||
* @returns the registered pi-ai api key to stream against.
|
||||
*/
|
||||
function resolveCustomProviderApiType(apiType: string): "anthropic-messages" | "openai-responses" | "openai-completions" {
|
||||
function resolveCustomProviderApiType(apiType: string): "anthropic-messages" | "openai-responses" | "google-generative-ai" | "openai-completions" {
|
||||
if (apiType === "anthropic-compatible") {
|
||||
return "anthropic-messages";
|
||||
}
|
||||
if (apiType === "openai-responses") {
|
||||
return "openai-responses";
|
||||
}
|
||||
if (apiType === "google-generative-ai") {
|
||||
return "google-generative-ai";
|
||||
}
|
||||
return "openai-completions";
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user