FN-5834: enforce explicit developer-role compatibility for custom providers
Prevent unsupported developer-role API failures by making provider role compatibility explicit and covered by regression safeguards. - Add explicit supportsDeveloperRole capability to custom provider metadata/types and wire it through provider registration. - Update CLI custom-provider registration behavior and tests to persist and validate developer-role compatibility settings. - Add/adjust engine regression coverage for openai-completions message role conversion, including reasoning-model fallback behavior. - Document the new custom provider setting and add a published changeset for @runfusion/fusion. Files changed: .changeset/fn-5834-developer-role-compat.md | 5 ++ docs/settings-reference.md | 2 +- packages/cli/src/commands/__tests__/custom-provider-registry.test.ts | 44 +++++++++++- packages/cli/src/commands/custom-provider-registry.ts | 9 ++- packages/core/src/types.ts | 5 ++ packages/engine/src/__tests__/custom-providers-openai-completions.test.ts | 84 +++++++++++----------- 6 files changed, 106 insertions(+), 43 deletions(-) Fusion-Task-Id: FN-5834 Fusion-Task-Lineage: 4edc80c9-5e60-41e4-bce1-662d18255d88
This commit is contained in:
gsxdsm
5
.changeset/fn-5834-developer-role-compat.md
Normal file
5
.changeset/fn-5834-developer-role-compat.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@runfusion/fusion": minor
|
||||
---
|
||||
|
||||
Custom OpenAI-compatible providers now register with explicit conservative role compatibility: Fusion defaults `compat.supportsDeveloperRole` to `false` so reasoning-capable models emit the legacy `system` role instead of relying on provider URL auto-detection. Advanced users can opt in per provider with `supportsDeveloperRole: true` when their endpoint explicitly supports the `developer` role.
|
||||
@@ -57,7 +57,7 @@ Fusion automatically falls back to ntfy's JSON publish format when a notificatio
|
||||
| `webhookFormat` | `"slack" \| "discord" \| "generic"` | `"generic"` | Webhook payload format. Part of legacy flat settings. |
|
||||
| `webhookEvents` | `string[]` | `[]` | Event filter for webhook notifications. Empty/omitted means all events. Part of legacy flat settings. |
|
||||
| `notificationProviders` | `NotificationProviderConfig[]` | `[]` | Array of pluggable notification provider configurations. Each entry uses `{ id, name, enabled, config }` and is dispatched by provider ID (for example `ntfy` or `webhook`). |
|
||||
| `customProviders` | `CustomProvider[]` | `[]` | User-defined OpenAI-compatible, OpenAI Responses API (`apiType: "openai-responses"`), or Anthropic-compatible providers used by the custom-provider API (`/api/custom-providers`). Each entry uses `{ id, name, apiType, baseUrl, apiKey?, models? }`; API keys are stored raw but masked in API responses. Fusion resolves these providers from the active global settings directory (`~/.fusion`, with legacy `~/.pi/fusion` and `~/.pi/kb` migration support) so custom-provider models remain available after restart. |
|
||||
| `customProviders` | `CustomProvider[]` | `[]` | User-defined OpenAI-compatible, OpenAI Responses API (`apiType: "openai-responses"`), or Anthropic-compatible providers used by the custom-provider API (`/api/custom-providers`). Each entry uses `{ id, name, apiType, baseUrl, apiKey?, supportsDeveloperRole?, models? }`; `supportsDeveloperRole` is an OpenAI-compatible opt-in that enables `developer` role emission (default/omitted is `false`, forcing safe `system` role). API keys are stored raw but masked in API responses. Fusion resolves these providers from the active global settings directory (`~/.fusion`, with legacy `~/.pi/fusion` and `~/.pi/kb` migration support) so custom-provider models remain available after restart. |
|
||||
| `defaultProjectId` | `string` | `undefined` | Default project for multi-project CLI operations when `--project` is omitted. |
|
||||
| `setupComplete` | `boolean` | `undefined` | Tracks completion of first-run setup. |
|
||||
| `favoriteProviders` | `string[]` | `undefined` | Pinned providers shown first in model selectors. |
|
||||
|
||||
@@ -44,7 +44,7 @@ describe("custom-provider-registry", () => {
|
||||
baseUrl: "https://example.test/v1",
|
||||
api: "openai-completions",
|
||||
apiKey: "CUSTOM_KEY",
|
||||
models: [expect.objectContaining({ id: "m1", name: "Model 1" })],
|
||||
models: [expect.objectContaining({ id: "m1", name: "Model 1", compat: { supportsDeveloperRole: false } })],
|
||||
}));
|
||||
expect(registerProvider).toHaveBeenNthCalledWith(2, "anthropic-custom", expect.objectContaining({
|
||||
baseUrl: "https://anthropic.test",
|
||||
@@ -165,6 +165,30 @@ describe("custom-provider-registry", () => {
|
||||
expect(refresh).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("sets supportsDeveloperRole true only when opted in", () => {
|
||||
const registerProvider = vi.fn();
|
||||
const refresh = vi.fn();
|
||||
|
||||
registerCustomProviders(
|
||||
{ registerProvider, refresh },
|
||||
[
|
||||
{ id: "optout", name: "Optout", apiType: "openai-compatible", baseUrl: "https://one.test", models: [{ id: "m", name: "M" }] },
|
||||
{ id: "optin", name: "Optin", apiType: "openai-compatible", baseUrl: "https://two.test", supportsDeveloperRole: true, models: [{ id: "m", name: "M" }] },
|
||||
{ id: "other", name: "Other", apiType: "anthropic-compatible", baseUrl: "https://three.test", models: [{ id: "m", name: "M" }] },
|
||||
],
|
||||
vi.fn(),
|
||||
);
|
||||
|
||||
expect(registerProvider).toHaveBeenNthCalledWith(1, "optout", expect.objectContaining({
|
||||
models: [expect.objectContaining({ compat: { supportsDeveloperRole: false } })],
|
||||
}));
|
||||
expect(registerProvider).toHaveBeenNthCalledWith(2, "optin", expect.objectContaining({
|
||||
models: [expect.objectContaining({ compat: { supportsDeveloperRole: true } })],
|
||||
}));
|
||||
const anthropicModels = registerProvider.mock.calls[2]?.[1]?.models as Array<Record<string, unknown>>;
|
||||
expect(anthropicModels[0]).not.toHaveProperty("compat");
|
||||
});
|
||||
|
||||
it("reregisters changed providers", () => {
|
||||
const registerProvider = vi.fn();
|
||||
const refresh = vi.fn();
|
||||
@@ -184,6 +208,24 @@ describe("custom-provider-registry", () => {
|
||||
expect(refresh).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("reregisters when only supportsDeveloperRole changes", () => {
|
||||
const registerProvider = vi.fn();
|
||||
const refresh = vi.fn();
|
||||
|
||||
reregisterCustomProviders(
|
||||
{ registerProvider, refresh },
|
||||
[{ id: "role", name: "Provider", apiType: "openai-compatible", baseUrl: "https://one.test", models: [{ id: "m", name: "M" }] }],
|
||||
[{ id: "role", name: "Provider", apiType: "openai-compatible", baseUrl: "https://one.test", supportsDeveloperRole: true, models: [{ id: "m", name: "M" }] }],
|
||||
vi.fn(),
|
||||
);
|
||||
|
||||
expect(registerProvider).toHaveBeenCalledTimes(1);
|
||||
expect(registerProvider).toHaveBeenCalledWith("provider", expect.objectContaining({
|
||||
models: [expect.objectContaining({ compat: { supportsDeveloperRole: true } })],
|
||||
}));
|
||||
expect(refresh).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("handles empty previous/current arrays", () => {
|
||||
const registerProvider = vi.fn();
|
||||
const refresh = vi.fn();
|
||||
|
||||
@@ -13,6 +13,9 @@ interface ModelRegistryLike {
|
||||
cost: { input: number; output: number; cacheRead: number; cacheWrite: number };
|
||||
contextWindow: number;
|
||||
maxTokens: number;
|
||||
compat?: {
|
||||
supportsDeveloperRole?: boolean;
|
||||
};
|
||||
}>;
|
||||
}) => void;
|
||||
refresh: () => void;
|
||||
@@ -29,9 +32,12 @@ export function resolveApiType(apiType: string): string {
|
||||
}
|
||||
|
||||
function toProviderConfig(provider: CustomProvider) {
|
||||
const api = resolveApiType(provider.apiType);
|
||||
const supportsDeveloperRole = provider.supportsDeveloperRole === true;
|
||||
|
||||
return {
|
||||
baseUrl: provider.baseUrl,
|
||||
api: resolveApiType(provider.apiType),
|
||||
api,
|
||||
apiKey: provider.apiKey,
|
||||
models: (provider.models ?? []).map((model) => ({
|
||||
id: model.id,
|
||||
@@ -46,6 +52,7 @@ function toProviderConfig(provider: CustomProvider) {
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 16384,
|
||||
...(api === "openai-completions" ? { compat: { supportsDeveloperRole } } : {}),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -562,6 +562,11 @@ export interface CustomProvider {
|
||||
apiType: "openai-compatible" | "anthropic-compatible" | "google-generative-ai" | "openai-responses";
|
||||
baseUrl: string;
|
||||
apiKey?: string;
|
||||
/**
|
||||
* OpenAI-compatible opt-in for providers that explicitly support the `developer` role.
|
||||
* Omitted/false forces legacy `system` role emission to avoid provider 400s.
|
||||
*/
|
||||
supportsDeveloperRole?: boolean;
|
||||
models?: { id: string; name: string }[];
|
||||
}
|
||||
|
||||
|
||||
@@ -1,69 +1,73 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { AuthStorage, ModelRegistry } from "@earendil-works/pi-coding-agent";
|
||||
import { completeSimple } from "@earendil-works/pi-ai";
|
||||
import { convertMessages } from "@earendil-works/pi-ai/openai-completions";
|
||||
import { customProviderRegistryKey, type CustomProvider } from "@fusion/core";
|
||||
|
||||
function createSseResponse(): Response {
|
||||
const stream = new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode("data: {\"id\":\"chatcmpl-test\",\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"Hello from mock transport\"},\"finish_reason\":null}]}\n\n"));
|
||||
controller.enqueue(new TextEncoder().encode("data: {\"id\":\"chatcmpl-test\",\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\n"));
|
||||
controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n"));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
return new Response(stream, { status: 200, headers: { "content-type": "text/event-stream" } });
|
||||
}
|
||||
|
||||
|
||||
describe("custom providers openai-completions regression", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("registers under slug key and completes a chat round-trip", async () => {
|
||||
const authStorage = AuthStorage.inMemory();
|
||||
const modelRegistry = ModelRegistry.inMemory(authStorage);
|
||||
|
||||
const providers: CustomProvider[] = [
|
||||
{
|
||||
id: "550e8400-e29b-41d4-a716-446655440000",
|
||||
name: "My AI Provider",
|
||||
apiType: "openai-compatible",
|
||||
baseUrl: "https://example.test/v1",
|
||||
apiKey: "CUSTOM_KEY",
|
||||
models: [{ id: "my-model", name: "My Model" }],
|
||||
},
|
||||
];
|
||||
const providers: CustomProvider[] = [{
|
||||
id: "550e8400-e29b-41d4-a716-446655440000",
|
||||
name: "My AI Provider",
|
||||
apiType: "openai-compatible",
|
||||
baseUrl: "https://example.test/v1",
|
||||
apiKey: "CUSTOM_KEY",
|
||||
models: [{ id: "my-model", name: "My Model" }],
|
||||
}];
|
||||
|
||||
const provider = providers[0]!;
|
||||
modelRegistry.registerProvider(customProviderRegistryKey(provider, providers), {
|
||||
baseUrl: provider.baseUrl,
|
||||
api: "openai-completions",
|
||||
apiKey: provider.apiKey,
|
||||
models: [{
|
||||
id: "my-model",
|
||||
name: "My Model",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 128000,
|
||||
maxTokens: 16384,
|
||||
}],
|
||||
models: [{ id: "my-model", name: "My Model", reasoning: false, input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 128000, maxTokens: 16384 }],
|
||||
});
|
||||
modelRegistry.refresh();
|
||||
|
||||
const registered = modelRegistry.getAll().find((model) => model.id === "my-model");
|
||||
expect(registered?.provider).toBe("my-ai-provider");
|
||||
|
||||
vi.stubGlobal("fetch", vi.fn(async () => {
|
||||
const stream = new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode("data: {\"id\":\"chatcmpl-test\",\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"Hello from mock transport\"},\"finish_reason\":null}]}\n\n"));
|
||||
controller.enqueue(new TextEncoder().encode("data: {\"id\":\"chatcmpl-test\",\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\n"));
|
||||
controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n"));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
|
||||
return new Response(stream, {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
});
|
||||
}));
|
||||
|
||||
vi.stubGlobal("fetch", vi.fn(async () => createSseResponse()));
|
||||
const model = modelRegistry.find("my-ai-provider", "my-model");
|
||||
expect(model).toBeDefined();
|
||||
const response = await completeSimple(model!, {
|
||||
messages: [{ role: "user", content: "Hi", timestamp: Date.now() }],
|
||||
});
|
||||
const response = await completeSimple(model!, { messages: [{ role: "user", content: "Hi", timestamp: Date.now() }] });
|
||||
expect(response.role).toBe("assistant");
|
||||
});
|
||||
|
||||
it("uses system role when reasoning model explicitly disables developer role compat", () => {
|
||||
const params = convertMessages(
|
||||
{ provider: "openai", reasoning: true, input: ["text"] } as never,
|
||||
{ systemPrompt: "system instruction", messages: [] } as never,
|
||||
{ supportsDeveloperRole: false } as never,
|
||||
);
|
||||
expect(params[0]?.role).toBe("system");
|
||||
});
|
||||
|
||||
it("emits developer role when compat allows it on reasoning models", () => {
|
||||
const params = convertMessages(
|
||||
{ provider: "openai", reasoning: true, input: ["text"] } as never,
|
||||
{ systemPrompt: "system instruction", messages: [] } as never,
|
||||
{ supportsDeveloperRole: true } as never,
|
||||
);
|
||||
expect(params[0]?.role).toBe("developer");
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user