From 461a4a2711a01ebd14c6937f42036e419eaa831e Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Wed, 8 Jul 2026 13:09:10 -0700 Subject: [PATCH] FN-7689: add opt-in Anthropic-style prompt caching for custom providers Custom providers previously never enabled prompt-cache control, so agent turns re-billed the full context every request even on cache-capable backends. - Add `CustomProvider.anthropicPromptCaching` opt-in flag in @fusion/core types - Set pi-ai `compat.cacheControlFormat="anthropic"` on opted-in models in both registration paths: custom-provider-registry `toProviderConfig` and pi.ts `createFnAgent` - Expose the new toggle in the dashboard CustomProvidersSection UI (with supporting CSS) and thread it through the legacy API + custom-provider routes - Update docs (dashboard-guide, settings-reference) to document the new setting - Add engine test coverage for the caching flag across provider registration and pi-create-fn-agent paths - Add changeset for the fix Files changed: .changeset/fn-7689-custom-provider-prompt-caching.md | 7 + docs/dashboard-guide.md | 1 + docs/settings-reference.md | 2 +- packages/core/src/types.ts | 15 ++ packages/dashboard/app/api/legacy.ts | 12 ++ packages/dashboard/app/components/CustomProvidersSection.css | 24 +++ packages/dashboard/app/components/CustomProvidersSection.tsx | 58 +++++- packages/dashboard/src/routes/register-custom-provider-routes.ts | 16 ++ packages/engine/src/__tests__/pi-create-fn-agent.test.ts | 71 +++++++ packages/engine/src/__tests__/provider-registration.test.ts | 204 ++++++++++++++++++++- packages/engine/src/custom-provider-registry.ts | 71 +++++-- packages/engine/src/pi.ts | 27 ++- 12 files changed, 473 insertions(+), 35 deletions(-) Fusion-Task-Id: FN-7689 Fusion-Task-Lineage: b4f88f32-50da-4651-a546-432a95a1ab1c Co-authored-by: Fusion (runfusion.ai) --- .../fn-7689-custom-provider-prompt-caching.md | 7 + docs/dashboard-guide.md | 1 + docs/settings-reference.md | 2 +- packages/core/src/types.ts | 15 ++ packages/dashboard/app/api/legacy.ts | 12 ++ .../app/components/CustomProvidersSection.css | 24 +++ .../app/components/CustomProvidersSection.tsx | 58 ++++- .../routes/register-custom-provider-routes.ts | 16 ++ .../src/__tests__/pi-create-fn-agent.test.ts | 71 ++++++ .../__tests__/provider-registration.test.ts | 204 +++++++++++++++++- .../engine/src/custom-provider-registry.ts | 71 ++++-- packages/engine/src/pi.ts | 27 ++- 12 files changed, 473 insertions(+), 35 deletions(-) create mode 100644 .changeset/fn-7689-custom-provider-prompt-caching.md diff --git a/.changeset/fn-7689-custom-provider-prompt-caching.md b/.changeset/fn-7689-custom-provider-prompt-caching.md new file mode 100644 index 0000000000..2686f7a4f7 --- /dev/null +++ b/.changeset/fn-7689-custom-provider-prompt-caching.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": minor +--- + +summary: Custom providers can now enable Anthropic-style prompt caching to stop re-billing the full context each turn. +category: fix +dev: Sets pi-ai `compat.cacheControlFormat="anthropic"` on opted-in custom-provider models across both registration paths (custom-provider-registry `toProviderConfig` and `pi.ts` createFnAgent). Opt-in via new `CustomProvider.anthropicPromptCaching` flag (FN-7689). diff --git a/docs/dashboard-guide.md b/docs/dashboard-guide.md index 00607d3288..02558017af 100644 --- a/docs/dashboard-guide.md +++ b/docs/dashboard-guide.md @@ -363,6 +363,7 @@ The custom-provider form uses these fields: - **API type** — one of the supported API types above. - **Base URL** — the provider endpoint base URL. It must be a valid `http` or `https` URL, for example `https://api.example.com/v1`. - **API key** — optional credential for providers that require authentication. +- **Enable Anthropic-style prompt caching** — shown only for **OpenAI-compatible** and **OpenAI Responses** provider entries. Turn this on when the provider gateway proxies an Anthropic-format backend (for example a self-hosted router fronting Claude models) to enable pi-ai's `cache_control` prompt caching, which stops re-billing the full context prefix every turn. Leave it off for gateways that do not support Anthropic-style caching (Together, Fireworks, etc.) to avoid provider errors. See [`anthropicPromptCaching` in the Settings Reference](./settings-reference.md#customproviders) for details. - **Available models** — comma-separated model IDs, for example `gpt-4, gpt-3.5-turbo`. Use **Detect Models** to auto-fill **Available models** while adding or editing a provider from the provider's `/models` endpoint. Detection requires a **Base URL** and may require an **API key**, depending on the provider. Saved providers also have a row-level **Refresh Models** action that uses the stored endpoint and credential to replace the persisted model list without exposing the raw key in the browser. diff --git a/docs/settings-reference.md b/docs/settings-reference.md index 7a2d2ab054..fed20c3657 100644 --- a/docs/settings-reference.md +++ b/docs/settings-reference.md @@ -83,7 +83,7 @@ Fusion automatically falls back to ntfy's JSON publish format when a notificatio | `webhookFormat` | `"slack" \| "discord" \| "generic"` | `"generic"` | Webhook payload format. Part of legacy flat settings. | | `webhookEvents` | `string[]` | `[]` | Event filter for webhook notifications. Empty/omitted means all events. Part of legacy flat settings. | | `notificationProviders` | `NotificationProviderConfig[]` | `[]` | Array of pluggable notification provider configurations. Each entry uses `{ id, name, enabled, config }` and is dispatched by provider ID (for example `ntfy` or `webhook`). | -| `customProviders` | `CustomProvider[]` | `[]` | User-defined OpenAI-compatible, OpenAI Responses API (`apiType: "openai-responses"`), Anthropic-compatible, or Google Generative AI (`apiType: "google-generative-ai"`) providers used by the custom-provider API (`/api/custom-providers`). Each entry uses `{ id, name, apiType, baseUrl, apiKey?, supportsDeveloperRole?, models? }`; `supportsDeveloperRole` is an OpenAI-compatible opt-in that enables `developer` role emission (default/omitted is `false`, forcing safe `system` role). API keys are stored raw but masked in API responses. Fusion resolves these providers from the active global settings directory (`~/.fusion`, with legacy `~/.pi/fusion` and `~/.pi/kb` migration support) so custom-provider models remain available after restart. Dashboard, serve, and daemon startup refresh each configured provider's persisted `models` list from its `/models` endpoint on a best-effort basis; failures leave the previous list intact and do not block startup. In Settings → Authentication → Advanced: Custom Providers, use **Refresh Models** on a provider row to manually refresh that provider after changing credentials, endpoints, or upstream model availability. Saved local/LAN/internal provider URLs are eligible for this stored-provider refresh path, while the add/edit **Detect Models** form keeps stricter SSRF protections for untrusted one-off input. | +| `customProviders` | `CustomProvider[]` | `[]` | User-defined OpenAI-compatible, OpenAI Responses API (`apiType: "openai-responses"`), Anthropic-compatible, or Google Generative AI (`apiType: "google-generative-ai"`) providers used by the custom-provider API (`/api/custom-providers`). Each entry uses `{ id, name, apiType, baseUrl, apiKey?, supportsDeveloperRole?, anthropicPromptCaching?, models? }`; `supportsDeveloperRole` is an OpenAI-compatible opt-in that enables `developer` role emission (default/omitted is `false`, forcing safe `system` role). `anthropicPromptCaching` (FN-7689) is an opt-in for `openai-compatible`/`openai-responses` gateways that proxy an Anthropic-format backend (for example a self-hosted router fronting Claude models); when `true`, Fusion registers that provider's `openai-completions` models with pi-ai's `compat.cacheControlFormat = "anthropic"`, so pi-ai attaches Anthropic-style `cache_control` breakpoints to the system prompt, the last conversation message, and the last tool definition, letting the gateway serve cached-prefix reads/writes instead of re-billing the full context every turn. Default/omitted is `false` — leave it off for gateways that do not understand `cache_control` (e.g. Together, Fireworks) to avoid provider 400s. It is inert (a documented no-op) for `anthropic-compatible` providers, which already auto-cache without any flag, and for `google-generative-ai` providers, which have no `cache_control` concept. Enable it from Settings → Authentication → Advanced: Custom Providers via the "Enable Anthropic-style prompt caching" checkbox shown for OpenAI-compatible/OpenAI Responses provider entries. API keys are stored raw but masked in API responses. Fusion resolves these providers from the active global settings directory (`~/.fusion`, with legacy `~/.pi/fusion` and `~/.pi/kb` migration support) so custom-provider models remain available after restart. Dashboard, serve, and daemon startup refresh each configured provider's persisted `models` list from its `/models` endpoint on a best-effort basis; failures leave the previous list intact and do not block startup. In Settings → Authentication → Advanced: Custom Providers, use **Refresh Models** on a provider row to manually refresh that provider after changing credentials, endpoints, or upstream model availability. Saved local/LAN/internal provider URLs are eligible for this stored-provider refresh path, while the add/edit **Detect Models** form keeps stricter SSRF protections for untrusted one-off input. | | `defaultProjectId` | `string` | `undefined` | Default project for multi-project CLI operations when `--project` is omitted. | | `setupComplete` | `boolean` | `undefined` | Tracks completion of first-run setup. | | `favoriteProviders` | `string[]` | `undefined` | Pinned providers shown first in model selectors. | diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 09d4f85573..15a1021312 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -804,6 +804,21 @@ export interface CustomProvider { * Omitted/false forces legacy `system` role emission to avoid provider 400s. */ supportsDeveloperRole?: boolean; + /** + * FNXC:ProviderAuth 2026-07-08-00:00: + * FN-7689: opt-in for custom `openai-compatible`/`openai-responses` gateways that proxy an + * Anthropic-format backend (e.g. `usai/claude_4_6_sonnet`). When true, registered + * `openai-completions` models get pi-ai's `compat.cacheControlFormat = "anthropic"`, which makes + * pi-ai emit Anthropic-style `cache_control` breakpoints on the system prompt, last + * conversation message, and last tool. Without this, pi-ai's `detectCompat` only auto-enables + * caching for OpenRouter `anthropic/*` models, so a generic custom gateway re-bills the entire + * context prefix uncached every turn (measured cachedTokens=0/cacheWriteTokens=0 across 243 + * runs, ~327.5:1 input:output ratio). Default off — never force cache_control on gateways that + * did not opt in, since non-Anthropic-compatible backends (Together, Fireworks, etc.) can 400 on + * unexpected `cache_control` fields. Inert for `anthropic-compatible` (already auto-caches) and + * `google-generative-ai` (no cache_control concept). + */ + anthropicPromptCaching?: boolean; models?: { id: string; name: string }[]; } diff --git a/packages/dashboard/app/api/legacy.ts b/packages/dashboard/app/api/legacy.ts index 2aa3a9d1b8..18c1ab4722 100644 --- a/packages/dashboard/app/api/legacy.ts +++ b/packages/dashboard/app/api/legacy.ts @@ -2338,6 +2338,12 @@ export interface CustomProvider { apiType: "openai-compatible" | "anthropic-compatible" | "google-generative-ai" | "openai-responses"; baseUrl: string; apiKey?: string; + /** + * FNXC:ProviderAuth 2026-07-08-00:00: + * FN-7689: dashboard-local mirror of @fusion/core's CustomProvider.anthropicPromptCaching + * opt-in. Keep in sync with packages/core/src/types.ts. + */ + anthropicPromptCaching?: boolean; models?: { id: string; name: string }[]; } @@ -2352,6 +2358,7 @@ export async function fetchCustomProviders(): Promise ({ id: model.id, name: model.name })), } satisfies CustomProviderConfig)); return Object.assign(legacyProviders, { providers: legacyProviders }); @@ -2373,6 +2380,9 @@ export function updateCustomProvider( ...(typeof legacy.name === "string" ? { name: legacy.name } : {}), ...(typeof legacy.baseUrl === "string" ? { baseUrl: legacy.baseUrl } : {}), ...(typeof legacy.apiKey === "string" ? { apiKey: legacy.apiKey } : {}), + ...("anthropicPromptCaching" in (updates as Record) + ? { anthropicPromptCaching: (updates as Partial>).anthropicPromptCaching } + : {}), ...(Array.isArray(legacy.models) ? { models: legacy.models.map((model) => ({ @@ -2433,6 +2443,8 @@ export interface CustomProviderConfig { baseUrl: string; api: "openai-completions" | "openai-responses" | "anthropic-messages" | "google-generative-ai"; apiKey?: string; + /** FNXC:ProviderAuth 2026-07-08-00:00: FN-7689 caching opt-in, carried through the legacy shape. */ + anthropicPromptCaching?: boolean; models: CustomProviderModelInput[]; } diff --git a/packages/dashboard/app/components/CustomProvidersSection.css b/packages/dashboard/app/components/CustomProvidersSection.css index b8de2310dd..2e305d3efd 100644 --- a/packages/dashboard/app/components/CustomProvidersSection.css +++ b/packages/dashboard/app/components/CustomProvidersSection.css @@ -145,6 +145,30 @@ margin-top: var(--space-xs); } +/* +FNXC:ProviderAuth 2026-07-08-00:00: +FN-7689: checkbox row + hint text for the Anthropic prompt-caching opt-in on custom +openai-compatible/openai-responses providers. Follows the existing form-row token pattern +(no hardcoded colors/px) and reuses --text-muted for the hint, matching other secondary text. +*/ +.custom-provider-form-checkbox-row { + display: flex; + flex-direction: column; + gap: var(--space-xs); +} + +.custom-provider-form-checkbox-row label { + display: flex; + align-items: center; + gap: var(--space-xs); +} + +.custom-provider-form-hint { + color: var(--text-muted); + font-size: var(--font-size-xs); + margin: 0; +} + .custom-provider-spin { animation: custom-provider-spin calc(var(--duration-slow) * 4) linear infinite; } diff --git a/packages/dashboard/app/components/CustomProvidersSection.tsx b/packages/dashboard/app/components/CustomProvidersSection.tsx index d86da7baa6..ff0d70c441 100644 --- a/packages/dashboard/app/components/CustomProvidersSection.tsx +++ b/packages/dashboard/app/components/CustomProvidersSection.tsx @@ -23,6 +23,7 @@ type LegacyProvider = { baseUrl: string; api: "openai-completions" | "openai-responses" | "anthropic-messages" | "google-generative-ai"; apiKey?: string; + anthropicPromptCaching?: boolean; models?: Array<{ id: string; name?: string }>; }; @@ -45,6 +46,7 @@ function normalizeProviders(result: Awaited ({ id: model.id, name: model.name ?? model.id, @@ -79,6 +81,11 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C const [baseUrl, setBaseUrl] = useState(""); const [apiKey, setApiKey] = useState(""); const [models, setModels] = useState(""); + // FNXC:ProviderAuth 2026-07-08-00:00: + // FN-7689: opt-in for Anthropic-style prompt caching on openai-compatible/openai-responses + // custom gateways that proxy an Anthropic backend. Shown only for those two apiTypes — + // anthropic-compatible already auto-caches and google-generative-ai has no cache_control concept. + const [anthropicPromptCaching, setAnthropicPromptCaching] = useState(false); const [saving, setSaving] = useState(false); const [formError, setFormError] = useState(null); const [detecting, setDetecting] = useState(false); @@ -122,6 +129,7 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C setBaseUrl(""); setApiKey(""); setModels(""); + setAnthropicPromptCaching(false); setFormError(null); setDetectError(null); setDetecting(false); @@ -135,6 +143,7 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C setBaseUrl(""); setApiKey(""); setModels(""); + setAnthropicPromptCaching(false); setFormError(null); setDetectError(null); setDetecting(false); @@ -152,6 +161,7 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C // unchanged blank field leaves the stored key untouched on save. setApiKey(""); setModels((provider.models ?? []).map((model) => model.id).join(", ")); + setAnthropicPromptCaching(provider.anthropicPromptCaching === true); setFormError(null); setDetectError(null); setDetecting(false); @@ -240,6 +250,10 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C baseUrl: baseUrl.trim(), ...(apiKey.trim() ? { apiKey: apiKey.trim() } : {}), ...(parsedModels.length > 0 ? { models: parsedModels } : {}), + // FNXC:ProviderAuth 2026-07-08-00:00: only send the caching opt-in for apiTypes where it + // applies (openai-compatible/openai-responses); anthropic-compatible/google-generative-ai + // never surface the checkbox so this is always false for them. + ...(anthropicPromptCaching ? { anthropicPromptCaching: true } : {}), }; setSaving(true); @@ -259,7 +273,7 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C } finally { setSaving(false); } - }, [apiKey, apiType, baseUrl, editingProvider, loadProviders, models, name, resetForm, validateForm, t]); + }, [anthropicPromptCaching, apiKey, apiType, baseUrl, editingProvider, loadProviders, models, name, resetForm, validateForm, t]); const handleDelete = useCallback( async (provider: CustomProvider) => { @@ -415,6 +429,27 @@ export function CustomProvidersSection({ embedded = false, onProviderChange }: C + {apiType === "openai-compatible" || apiType === "openai-responses" ? ( +
+ +

+ {t( + "providers.anthropicPromptCachingHint", + "Enable if this gateway proxies an Anthropic model (e.g. Claude via a custom router). Reduces re-billing the full context every turn.", + )} +

+
+ ) : null} +
+ {apiType === "openai-compatible" || apiType === "openai-responses" ? ( +
+ +

+ {t( + "providers.anthropicPromptCachingHint", + "Enable if this gateway proxies an Anthropic model (e.g. Claude via a custom router). Reduces re-billing the full context every turn.", + )} +

+
+ ) : null} +
{ } } + // FNXC:ProviderAuth 2026-07-08-00:00: + // FN-7689: accept the Anthropic prompt-caching opt-in from the dashboard editor so it survives + // the round trip to registerCustomProviders/reregisterCustomProviders (custom-provider-registry.ts). + if (row.anthropicPromptCaching !== undefined) { + if (typeof row.anthropicPromptCaching !== "boolean") { + throw badRequest("anthropicPromptCaching must be a boolean"); + } + provider.anthropicPromptCaching = row.anthropicPromptCaching; + } + const models = validateModels(row.models); if (models) { provider.models = models; @@ -575,6 +585,12 @@ function parseUpdateBody(body: unknown): Partial> { updates.apiKey = row.apiKey.trim().length > 0 ? row.apiKey : undefined; } } + if (row.anthropicPromptCaching !== undefined) { + if (typeof row.anthropicPromptCaching !== "boolean") { + throw badRequest("anthropicPromptCaching must be a boolean"); + } + updates.anthropicPromptCaching = row.anthropicPromptCaching; + } if (row.models !== undefined) { updates.models = validateModels(row.models); } diff --git a/packages/engine/src/__tests__/pi-create-fn-agent.test.ts b/packages/engine/src/__tests__/pi-create-fn-agent.test.ts index 70218dc6ee..563cd67222 100644 --- a/packages/engine/src/__tests__/pi-create-fn-agent.test.ts +++ b/packages/engine/src/__tests__/pi-create-fn-agent.test.ts @@ -1629,6 +1629,77 @@ describe("createFnAgent", () => { } }); + /* + FNXC:ProviderAuth 2026-07-08-00:00: + FN-7689 regression coverage — registration path B (createFnAgent's inline custom-provider + registration). Path A was already refactored into `buildCustomProviderModels` and reused here, + but this test exists specifically to catch a future re-divergence: if someone inlines a fresh + models.map(...) in createFnAgent again (as it was before this fix), this assertion fails. + */ + it("registers openai-compatible custom providers with compat.cacheControlFormat='anthropic' when opted in (createFnAgent inline path)", async () => { + readCustomProvidersMock.mockReturnValue([ + { + id: "550e8400-e29b-41d4-a716-446655440000", + name: "Custom OpenAI Caching", + apiType: "openai-compatible", + baseUrl: "https://custom.example/v1", + apiKey: "CUSTOM_API_KEY", + anthropicPromptCaching: true, + models: [{ id: "custom-model", name: "Custom Model" }], + }, + { + id: "660e8400-e29b-41d4-a716-446655440001", + name: "Custom OpenAI No Caching", + apiType: "openai-compatible", + baseUrl: "https://nocaching.example/v1", + apiKey: "NOCACHE_API_KEY", + models: [{ id: "nocache-model", name: "No Cache Model" }], + }, + { + id: "770e8400-e29b-41d4-a716-446655440002", + name: "Custom Anthropic Caching Opt-in", + apiType: "anthropic-compatible", + baseUrl: "https://anthropic.example", + apiKey: "ANTHROPIC_API_KEY", + anthropicPromptCaching: true, + models: [{ id: "anthropic-model", name: "Anthropic Model" }], + }, + ] as any); + + const { createFnAgent } = await import("../pi.js"); + + await createFnAgent({ + cwd: "/tmp", + systemPrompt: "test", + tools: "readonly", + defaultProvider: "openai-codex", + defaultModelId: "gpt-5.4", + }); + + // Opted-in openai-compatible provider: cacheControlFormat must be set. + expect(registerProviderMock).toHaveBeenCalledWith("custom-openai-caching", expect.objectContaining({ + api: "openai-completions", + models: [expect.objectContaining({ + id: "custom-model", + compat: expect.objectContaining({ cacheControlFormat: "anthropic" }), + })], + })); + + // Opted-out (default) openai-compatible provider: no forced cache_control marker. + const noCacheCall = registerProviderMock.mock.calls.find(([key]: [string]) => key === "custom-openai-no-caching"); + expect(noCacheCall).toBeDefined(); + const [, noCacheConfig] = noCacheCall as [string, { models: Array<{ compat?: Record }> }]; + expect(noCacheConfig.models[0].compat).not.toHaveProperty("cacheControlFormat"); + + // anthropic-compatible provider: opt-in is a documented no-op — pi-ai's anthropic path already + // auto-caches without this compat flag, and openai-completions-only compat must not leak in. + const anthropicCall = registerProviderMock.mock.calls.find(([key]: [string]) => key === "custom-anthropic-caching-opt-in"); + expect(anthropicCall).toBeDefined(); + const [, anthropicConfig] = anthropicCall as [string, { api: string; models: Array<{ compat?: Record }> }]; + expect(anthropicConfig.api).toBe("anthropic-messages"); + expect(anthropicConfig.models[0].compat).toBeUndefined(); + }); + it("avoids lock-based SettingsManager.create when loading extension providers", async () => { const { createFnAgent } = await import("../pi.js"); diff --git a/packages/engine/src/__tests__/provider-registration.test.ts b/packages/engine/src/__tests__/provider-registration.test.ts index aa6c9e9791..aaa0f44af4 100644 --- a/packages/engine/src/__tests__/provider-registration.test.ts +++ b/packages/engine/src/__tests__/provider-registration.test.ts @@ -1,6 +1,9 @@ -import { describe, expect, it, vi } from "vitest"; -import type { CustomProvider } from "@fusion/core"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { customProviderRegistryKey, type CustomProvider } from "@fusion/core"; +import { AuthStorage, ModelRegistry } from "@earendil-works/pi-coding-agent"; +import { completeSimple } from "@earendil-works/pi-ai/compat"; import { seedDashboardProviders } from "../provider-registration.js"; +import { registerCustomProviders } from "../custom-provider-registry.js"; /* FNXC:ProviderRegistration 2026-07-07-00:00: @@ -210,3 +213,200 @@ describe("seedDashboardProviders", () => { expect(modelRegistry.registerProvider).not.toHaveBeenCalled(); }); }); + +/* +FNXC:ProviderAuth 2026-07-08-00:00: +FN-7689 regression coverage — registration path A (custom-provider-registry.ts `toProviderConfig`, +driven here via `registerCustomProviders`). Asserts the invariant: an opted-in openai-compatible +custom provider's registered pi-ai model config carries `compat.cacheControlFormat === "anthropic"`, +and that an opted-out (default) provider does NOT get it forced — no cache_control markers on +gateways that never asked for them (avoids provider 400s on backends like Together/Fireworks). +*/ +describe("registerCustomProviders anthropicPromptCaching opt-in (FN-7689)", () => { + it("sets compat.cacheControlFormat='anthropic' for an opted-in openai-compatible provider", () => { + const modelRegistry = makeModelRegistry(); + const provider = customProvider({ anthropicPromptCaching: true }); + + registerCustomProviders(modelRegistry, [provider], vi.fn()); + + expect(modelRegistry.registerProvider).toHaveBeenCalledWith( + "acme-ai", + expect.objectContaining({ + api: "openai-completions", + models: [expect.objectContaining({ compat: expect.objectContaining({ cacheControlFormat: "anthropic" }) })], + }), + ); + }); + + it("does NOT set cacheControlFormat for an opted-out (default) openai-compatible provider", () => { + const modelRegistry = makeModelRegistry(); + const provider = customProvider(); + + registerCustomProviders(modelRegistry, [provider], vi.fn()); + + const call = modelRegistry.registerProvider.mock.calls.find(([key]: [string]) => key === "acme-ai"); + expect(call).toBeDefined(); + const [, config] = call as [string, { models: Array<{ compat?: Record }> }]; + expect(config.models[0].compat).not.toHaveProperty("cacheControlFormat"); + }); + + it("leaves the opt-in inert for anthropic-compatible providers (already auto-caches)", () => { + const modelRegistry = makeModelRegistry(); + const provider = customProvider({ apiType: "anthropic-compatible", anthropicPromptCaching: true }); + + registerCustomProviders(modelRegistry, [provider], vi.fn()); + + const call = modelRegistry.registerProvider.mock.calls.find(([key]: [string]) => key === "acme-ai"); + expect(call).toBeDefined(); + const [, config] = call as [string, { api: string; models: Array<{ compat?: Record }> }]; + expect(config.api).toBe("anthropic"); + expect(config.models[0].compat).toBeUndefined(); + }); + + it("leaves the opt-in inert for openai-responses providers (no cache_control concept there)", () => { + const modelRegistry = makeModelRegistry(); + const provider = customProvider({ apiType: "openai-responses", anthropicPromptCaching: true }); + + registerCustomProviders(modelRegistry, [provider], vi.fn()); + + const call = modelRegistry.registerProvider.mock.calls.find(([key]: [string]) => key === "acme-ai"); + expect(call).toBeDefined(); + const [, config] = call as [string, { api: string; models: Array<{ compat?: Record }> }]; + expect(config.api).toBe("openai-responses"); + expect(config.models[0].compat).toBeUndefined(); + }); +}); + +/* +FNXC:ProviderAuth 2026-07-08-00:00: +FN-7689 symptom-based acceptance. The ORIGINAL SYMPTOM: a custom openai-compatible gateway proxying +Anthropic (e.g. usai/claude_4_6_sonnet) got cachedTokens=0/cacheWriteTokens=0 across all 243 runs +because pi-ai's openai-completions request builder never attached `cache_control` — detectCompat() +only auto-enables it for OpenRouter anthropic/* models. This drives the REAL pi-ai request-building +code path (not a reimplementation): register a provider through this module's own +`registerCustomProviders` (path A) into a real pi-ai ModelRegistry, resolve the model, then call +pi-ai's `completeSimple` (compat.ts, which internally calls the same `stream()` → `buildParams()` → +`getCompat()` → `getCompatCacheControl()` → `applyAnthropicCacheControl()` chain grounded during +preflight) with a system prompt, multi-turn messages, and a tool list. The mocked `fetch` seam +captures the exact HTTP request body pi-ai sends, so the assertion proves cache_control markers are +present on the wire where before this fix they were completely absent. Reverting Step 1 makes this +test fail (cache_control undefined everywhere). +*/ +describe("FN-7689 symptom verification: cache_control on the wire for opted-in custom providers", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("emits cache_control on the system message, last conversation message, and last tool", async () => { + const authStorage = AuthStorage.inMemory(); + const modelRegistry = ModelRegistry.inMemory(authStorage); + + const provider = customProvider({ + id: "aa0e8400-e29b-41d4-a716-446655440099", + name: "Usai Gateway", + baseUrl: "https://usai.example.test/v1", + anthropicPromptCaching: true, + models: [{ id: "claude_4_6_sonnet", name: "Claude 4.6 Sonnet" }], + }); + registerCustomProviders(modelRegistry as any, [provider], vi.fn()); + + const registryKey = customProviderRegistryKey(provider, [provider]); + const model = modelRegistry.find(registryKey, "claude_4_6_sonnet"); + expect(model).toBeDefined(); + expect(model?.compat).toEqual(expect.objectContaining({ cacheControlFormat: "anthropic" })); + + let capturedBody: any; + vi.stubGlobal("fetch", vi.fn(async (_url: unknown, init?: { body?: string }) => { + capturedBody = init?.body ? JSON.parse(init.body) : undefined; + return new Response( + "data: {\"id\":\"c\",\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); + })); + + await completeSimple(model!, { + systemPrompt: "You are a helpful coding agent. Follow AGENTS.md conventions.", + messages: [ + { role: "user", content: "Read this file and summarize it.", timestamp: Date.now() }, + { + role: "assistant", + content: [{ type: "text", text: "Here is a summary of the file." }], + api: "openai-completions", + provider: registryKey as any, + model: "claude_4_6_sonnet", + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + stopReason: "stop", + timestamp: Date.now(), + }, + { role: "user", content: "Now make the requested change.", timestamp: Date.now() }, + ], + tools: [{ name: "read_file", description: "Read a file from disk", parameters: { type: "object", properties: {}, required: [] } as any }], + } as any, { apiKey: "test-key" } as any); + + expect(capturedBody).toBeDefined(); + const messages = capturedBody.messages as Array<{ role: string; content: unknown }>; + + // System message carries cache_control on its (converted) text content block. + const systemMessage = messages.find((m) => m.role === "system" || m.role === "developer"); + expect(systemMessage).toBeDefined(); + const systemContent = systemMessage!.content; + expect(Array.isArray(systemContent) ? systemContent : []).toEqual( + expect.arrayContaining([expect.objectContaining({ cache_control: { type: "ephemeral" } })]), + ); + + // Last user/assistant conversation message carries cache_control. + const lastConversationMessage = [...messages].reverse().find((m) => m.role === "user" || m.role === "assistant"); + expect(lastConversationMessage).toBeDefined(); + const lastContent = lastConversationMessage!.content; + expect(Array.isArray(lastContent) ? lastContent : []).toEqual( + expect.arrayContaining([expect.objectContaining({ cache_control: { type: "ephemeral" } })]), + ); + + // Last tool definition carries cache_control. + const tools = capturedBody.tools as Array<{ cache_control?: unknown }> | undefined; + expect(tools).toBeDefined(); + expect(tools![tools!.length - 1].cache_control).toEqual({ type: "ephemeral" }); + }); + + it("emits NO cache_control when the provider did not opt in (negative control)", async () => { + const authStorage = AuthStorage.inMemory(); + const modelRegistry = ModelRegistry.inMemory(authStorage); + + const provider = customProvider({ + id: "bb0e8400-e29b-41d4-a716-446655440098", + name: "No Caching Gateway", + baseUrl: "https://nocache.example.test/v1", + models: [{ id: "some-model", name: "Some Model" }], + }); + registerCustomProviders(modelRegistry as any, [provider], vi.fn()); + + const registryKey = customProviderRegistryKey(provider, [provider]); + const model = modelRegistry.find(registryKey, "some-model"); + expect(model).toBeDefined(); + expect(model?.compat).not.toHaveProperty("cacheControlFormat"); + + let capturedBody: any; + vi.stubGlobal("fetch", vi.fn(async (_url: unknown, init?: { body?: string }) => { + capturedBody = init?.body ? JSON.parse(init.body) : undefined; + return new Response( + "data: {\"id\":\"c\",\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); + })); + + await completeSimple(model!, { + systemPrompt: "System prompt.", + messages: [{ role: "user", content: "Hello.", timestamp: Date.now() }], + } as any, { apiKey: "test-key" } as any); + + expect(capturedBody).toBeDefined(); + const messages = capturedBody.messages as Array<{ role: string; content: unknown }>; + for (const message of messages) { + if (Array.isArray(message.content)) { + for (const part of message.content as Array>) { + expect(part).not.toHaveProperty("cache_control"); + } + } + } + }); +}); diff --git a/packages/engine/src/custom-provider-registry.ts b/packages/engine/src/custom-provider-registry.ts index 1543996d86..0d3156b86f 100644 --- a/packages/engine/src/custom-provider-registry.ts +++ b/packages/engine/src/custom-provider-registry.ts @@ -22,6 +22,7 @@ interface ModelRegistryLike { maxTokens: number; compat?: { supportsDeveloperRole?: boolean; + cacheControlFormat?: "anthropic"; }; }>; }) => void; @@ -51,29 +52,67 @@ export function resolveApiType(apiType: string): string { return "openai-completions"; } +/** + * FNXC:ProviderAuth 2026-07-08-00:00: + * FN-7689: shared model-list builder used by BOTH custom-provider registration paths + * (this module's `toProviderConfig` and pi.ts's `createFnAgent` inline registration) so the + * `compat.cacheControlFormat` opt-in cannot drift between them again. `api` is the pi-ai + * api-registry key resolved by each call site's own resolver — `resolveApiType` here returns + * `"anthropic"` while pi.ts's `resolveCustomProviderApiType` returns `"anthropic-messages"` for + * the same `anthropic-compatible` input (a pre-existing naming drift out of scope for this fix; + * see FN-7689 follow-up). Only `"openai-completions"` gets `compat.cacheControlFormat` — pi-ai's + * anthropic path already auto-caches without any flag, and `openai-responses` uses OpenAI's + * native `prompt_cache_key`/`prompt_cache_retention` mechanism (no `cache_control` marker concept + * per pi-ai's `OpenAIResponsesCompat`), so the opt-in is inert there by construction. + */ +export function buildCustomProviderModels( + provider: CustomProvider, + api: string, +): Array<{ + id: string; + name: string; + reasoning: boolean; + input: ("text" | "image")[]; + cost: { input: number; output: number; cacheRead: number; cacheWrite: number }; + contextWindow: number; + maxTokens: number; + compat?: { supportsDeveloperRole?: boolean; cacheControlFormat?: "anthropic" }; +}> { + const supportsDeveloperRole = provider.supportsDeveloperRole === true; + const anthropicPromptCaching = provider.anthropicPromptCaching === true; + + return (provider.models ?? []).map((model) => ({ + id: model.id, + name: model.name, + reasoning: false, + input: ["text" as const], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + contextWindow: 128000, + maxTokens: 16384, + ...(api === "openai-completions" + ? { + compat: { + supportsDeveloperRole, + ...(anthropicPromptCaching ? { cacheControlFormat: "anthropic" as const } : {}), + }, + } + : {}), + })); +} + function toProviderConfig(provider: CustomProvider) { const api = resolveApiType(provider.apiType); - const supportsDeveloperRole = provider.supportsDeveloperRole === true; return { baseUrl: provider.baseUrl, api, apiKey: provider.apiKey, - models: (provider.models ?? []).map((model) => ({ - id: model.id, - name: model.name, - reasoning: false, - input: ["text" as const], - cost: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 128000, - maxTokens: 16384, - ...(api === "openai-completions" ? { compat: { supportsDeveloperRole } } : {}), - })), + models: buildCustomProviderModels(provider, api), }; } diff --git a/packages/engine/src/pi.ts b/packages/engine/src/pi.ts index ac1a84e302..bb39a7084b 100644 --- a/packages/engine/src/pi.ts +++ b/packages/engine/src/pi.ts @@ -63,6 +63,7 @@ import { applyClaudeAcpEnable } from "./claude-acp-enable.js"; import { createFusionAuthStorage, getModelRegistryModelsPath } from "./auth-storage.js"; import { piLog, extensionsLog } from "./logger.js"; import { readCustomProviders } from "./custom-providers.js"; +import { buildCustomProviderModels } from "./custom-provider-registry.js"; import { buildGateRejection, evaluateAgentActionGate, @@ -2092,24 +2093,20 @@ export async function createFnAgent(options: AgentOptions): Promise for (const provider of customProviders) { try { const registryKey = customProviderRegistryKey(provider, customProviders); + const api = resolveCustomProviderApiType(provider.apiType); + // FNXC:ProviderAuth 2026-07-08-00:00: + // FN-7689: reuse the shared `buildCustomProviderModels` helper (custom-provider-registry.ts) + // instead of building the model list inline here. This is registration path B (the + // `createFnAgent` inline path); path A is `custom-provider-registry.ts`'s `toProviderConfig`. + // Before this fix path B set no `compat` at all, so an opted-in provider's + // `anthropicPromptCaching` flag only took effect via path A and silently dropped here — + // exactly the drift risk called out for FN-7689. Sharing the builder makes that + // impossible to reintroduce. modelRegistry.registerProvider(registryKey, { baseUrl: provider.baseUrl, - api: resolveCustomProviderApiType(provider.apiType), + api, apiKey: provider.apiKey, - models: (provider.models ?? []).map((model) => ({ - id: model.id, - name: model.name, - reasoning: false, - input: ["text" as const], - cost: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 128000, - maxTokens: 16384, - })), + models: buildCustomProviderModels(provider, api), }); piLog.log(`Registered custom provider "${provider.name}" (key=${registryKey}, id=${provider.id})`); } catch (error) {