fix(engine): stop grok-cli fallback preempting the configured primary model

A configured grok-cli fallback with no Fusion-visible GROK_API_KEY was
promoted to primary at session start (FN-7758 seam), so every planning
session silently ran grok-4.5 instead of the configured planning model.
The no-visible-key Grok CLI auto-route now fires only for a grok-cli
primary; a fallback-only grok-cli pair is dropped with a warning and an
audited grokCliFallbackDropped flag, and session:runtime-resolved now
records the post-transform model pair the session actually runs.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-07-22 14:14:36 -07:00
parent b6135f4bd5
commit d36059bdce
3 changed files with 131 additions and 39 deletions

View File

@@ -0,0 +1,7 @@
---
"@runfusion/fusion": patch
---
summary: Fix grok-cli fallback models silently replacing the configured primary model when no GROK_API_KEY is visible.
category: fix
dev: The FN-7758 no-visible-key seam no longer promotes a grok-cli fallback to primary at session start; only a grok-cli primary auto-routes to the Grok CLI runtime. A fallback-only grok-cli pair without a visible key is dropped with a session warning, `grokCliFallbackDropped: true`, and reason `grok-cli-fallback-dropped-no-visible-key` in the `session:runtime-resolved` audit event, which now also records the post-transform provider/model pair the session actually runs.

View File

@@ -342,7 +342,8 @@ describe("Grok CLI runtime routing (FN-7725)", () => {
runtimeHint: "grok", runtimeHint: "grok",
reason: "grok-cli-no-visible-key", reason: "grok-cli-no-visible-key",
provider: "grok-cli", provider: "grok-cli",
modelId: "grok-cli/grok-4.5", // FNXC:ModelResolution 2026-07-22-14:30: audit records the post-transform model the session actually runs (provider prefix stripped for the CLI).
modelId: "grok-4.5",
}), }),
})); }));
}); });
@@ -425,12 +426,19 @@ describe("Grok CLI runtime routing (FN-7725)", () => {
})); }));
}); });
it("auto-routes a grok-cli fallback model to the Grok runtime when no Fusion-visible key exists", async () => { /*
FNXC:GrokCliRouting 2026-07-22-14:30:
A grok-cli FALLBACK behind a healthy non-grok primary must never preempt the primary.
The old FN-7758 behavior routed the whole session onto the Grok CLI runtime and promoted
the fallback model up front, so every planning session silently ran grok-4.5 instead of
the configured primary. The fixed invariant: primary stays on its own runtime with its
configured model; the unusable grok-cli fallback pair is dropped and the drop is audited.
*/
it("keeps the configured primary and drops a grok-cli fallback when no Fusion-visible key exists", async () => {
vi.mocked(fusionCore.isGrokApiKeyFusionVisible).mockReturnValue(false); vi.mocked(fusionCore.isGrokApiKeyFusionVisible).mockReturnValue(false);
const grokRegistration = await createGrokRegistration(); const grokRegistration = await createGrokRegistration();
const pluginRunner = createMockPluginRunner({ const getRuntimeById = vi.fn().mockReturnValue(grokRegistration);
getRuntimeById: vi.fn().mockReturnValue(grokRegistration), const pluginRunner = createMockPluginRunner({ getRuntimeById });
});
const audit = { database: vi.fn().mockResolvedValue(undefined) }; const audit = { database: vi.fn().mockResolvedValue(undefined) };
const result = await createResolvedAgentSession({ const result = await createResolvedAgentSession({
@@ -445,23 +453,26 @@ describe("Grok CLI runtime routing (FN-7725)", () => {
systemPrompt: "fallback-selection-only", systemPrompt: "fallback-selection-only",
}); });
expect(result.runtimeId).toBe("grok"); expect(result.runtimeId).toBe("pi");
expect(result.wasConfigured).toBe(true); expect(mockCreateFnAgent).toHaveBeenCalledWith(expect.objectContaining({
expect(mockCreateFnAgent).not.toHaveBeenCalled(); defaultProvider: "openai",
expect(result.session).toMatchObject({ model: "grok-4.5" }); defaultModelId: "gpt-4o",
fallbackProvider: undefined,
fallbackModelId: undefined,
}));
expect(audit.database).toHaveBeenCalledWith(expect.objectContaining({ expect(audit.database).toHaveBeenCalledWith(expect.objectContaining({
type: "session:runtime-resolved", type: "session:runtime-resolved",
target: "grok", target: "pi",
metadata: expect.objectContaining({ metadata: expect.objectContaining({
runtimeHint: "grok", reason: "grok-cli-fallback-dropped-no-visible-key",
reason: "grok-cli-no-visible-key", grokCliFallbackDropped: true,
provider: "openai", provider: "openai",
modelId: "gpt-4o", modelId: "gpt-4o",
}), }),
})); }));
}); });
it("auto-routes a bare grok-cli fallback model id without adding a provider prefix", async () => { it("drops a bare grok-cli fallback model id while leaving the primary untouched", async () => {
vi.mocked(fusionCore.isGrokApiKeyFusionVisible).mockReturnValue(false); vi.mocked(fusionCore.isGrokApiKeyFusionVisible).mockReturnValue(false);
const grokRegistration = await createGrokRegistration(); const grokRegistration = await createGrokRegistration();
const pluginRunner = createMockPluginRunner({ const pluginRunner = createMockPluginRunner({
@@ -479,8 +490,37 @@ describe("Grok CLI runtime routing (FN-7725)", () => {
systemPrompt: "fallback-bare-model", systemPrompt: "fallback-bare-model",
}); });
expect(result.runtimeId).toBe("grok"); expect(result.runtimeId).toBe("pi");
expect(result.session).toMatchObject({ model: "grok-4.5" }); expect(mockCreateFnAgent).toHaveBeenCalledWith(expect.objectContaining({
defaultProvider: "anthropic",
defaultModelId: "claude-sonnet-4-5",
fallbackProvider: undefined,
fallbackModelId: undefined,
}));
});
it("keeps a grok-cli fallback when a Fusion-visible key exists (pi can resolve it directly)", async () => {
vi.mocked(fusionCore.isGrokApiKeyFusionVisible).mockReturnValue(true);
const pluginRunner = createMockPluginRunner();
const result = await createResolvedAgentSession({
sessionPurpose: "executor",
pluginRunner,
cwd: "/tmp/project",
defaultProvider: "openai",
defaultModelId: "gpt-4o",
fallbackProvider: "grok-cli",
fallbackModelId: "grok-4.5",
systemPrompt: "visible-key-fallback",
});
expect(result.runtimeId).toBe("pi");
expect(mockCreateFnAgent).toHaveBeenCalledWith(expect.objectContaining({
defaultProvider: "openai",
defaultModelId: "gpt-4o",
fallbackProvider: "grok-cli",
fallbackModelId: "grok-4.5",
}));
}); });
it("keeps mock/test-mode provider routing on the mock runtime when grok-cli fallback is configured", async () => { it("keeps mock/test-mode provider routing on the mock runtime when grok-cli fallback is configured", async () => {

View File

@@ -429,12 +429,20 @@ function buildMissingGrokRuntimeError(): Error {
); );
} }
/*
FNXC:GrokCliRouting 2026-07-22-14:30:
The no-visible-key Grok CLI auto-derive fires only when grok-cli is the PRIMARY provider.
FN-7758 used to fire on a grok-cli FALLBACK too and promoted it to primary up front, which
silently replaced a healthy configured primary (e.g. planning openai-codex/gpt-5.6-sol ran
as grok/grok-4.5 on every triage session because the workflow's planningFallback was
grok-cli). A fallback must never preempt a primary that has not failed; the fallback-only
case is handled by dropGrokCliFallbackForNoVisibleKey below.
*/
function deriveGrokRuntimeHintForNoVisibleKey( function deriveGrokRuntimeHintForNoVisibleKey(
runtimeOptions: AgentRuntimeOptions, runtimeOptions: AgentRuntimeOptions,
pluginRunner: PluginRunner | undefined, pluginRunner: PluginRunner | undefined,
): string | undefined { ): string | undefined {
if (runtimeOptions.defaultProvider !== GROK_CLI_PROVIDER_ID if (runtimeOptions.defaultProvider !== GROK_CLI_PROVIDER_ID) return undefined;
&& runtimeOptions.fallbackProvider !== GROK_CLI_PROVIDER_ID) return undefined;
if (isGrokApiKeyFusionVisible()) return undefined; if (isGrokApiKeyFusionVisible()) return undefined;
try { try {
if (pluginRunner?.getRuntimeById("grok")) return "grok"; if (pluginRunner?.getRuntimeById("grok")) return "grok";
@@ -454,23 +462,35 @@ function applyGrokCliNoKeyRuntimeOptions(
}; };
} }
if (runtimeOptions.fallbackProvider === GROK_CLI_PROVIDER_ID) { return runtimeOptions;
return { }
/*
FNXC:GrokCliRouting 2026-07-22-14:30:
A grok-cli fallback behind a non-grok primary is unusable without a Fusion-visible
GROK_API_KEY: pi resolves fallback swaps through the key-requiring provider registry
(the original FN-7758 failure mode), and cross-runtime swaps into the Grok CLI runtime
are not supported mid-session. Drop the fallback pair, keep the configured primary
untouched, and surface the drop via a session warning plus run-audit metadata so the
operator can fix the lane (set GROK_API_KEY or pick a same-runtime fallback).
*/
function dropGrokCliFallbackForNoVisibleKey(
runtimeOptions: AgentRuntimeOptions,
): { options: AgentRuntimeOptions; dropped: boolean } {
if (runtimeOptions.defaultProvider === GROK_CLI_PROVIDER_ID
|| runtimeOptions.fallbackProvider !== GROK_CLI_PROVIDER_ID
|| isGrokApiKeyFusionVisible()) {
return { options: runtimeOptions, dropped: false };
}
return {
options: {
...runtimeOptions, ...runtimeOptions,
defaultProvider: runtimeOptions.fallbackProvider,
defaultModelId: stripGrokCliModelProviderPrefix(runtimeOptions.fallbackModelId),
/*
* FNXC:Settings-ThinkingLevel 2026-07-10-00:00:
* When the no-visible-key Grok CLI fallback is promoted to the primary runtime, promote its fallback thinking level too; the cleared fallback pair must not leave the Grok CLI session using the superseded primary model's thinking level.
*/
defaultThinkingLevel: runtimeOptions.fallbackThinkingLevel ?? runtimeOptions.defaultThinkingLevel,
fallbackProvider: undefined, fallbackProvider: undefined,
fallbackModelId: undefined, fallbackModelId: undefined,
fallbackThinkingLevel: undefined, fallbackThinkingLevel: undefined,
}; },
} dropped: true,
};
return runtimeOptions;
} }
function pickSettingsThenRuntimeModel( function pickSettingsThenRuntimeModel(
@@ -739,10 +759,12 @@ export async function createResolvedAgentSession(
provider routing stays on the mock runtime. Strip only the provider-qualified model prefix so the CLI provider routing stays on the mock runtime. Strip only the provider-qualified model prefix so the CLI
receives the concrete selected model via GrokRuntimeAdapter without changing non-grok sessions. receives the concrete selected model via GrokRuntimeAdapter without changing non-grok sessions.
FNXC:GrokCliRouting 2026-07-09-22:10: FNXC:GrokCliRouting 2026-07-22-14:30:
FN-7758 extends the no-visible-key invariant to configured fallback models. Pi resolves fallback models FN-7758's fallback-promotion behavior is removed: a configured grok-cli FALLBACK no longer selects the
during session creation and prompt-time swaps through the key-requiring provider registry, so a grok-cli Grok CLI runtime or replaces the primary model up front. That promotion silently discarded a healthy
fallback must select the Grok CLI runtime up front and promote the fallback model into the CLI session. configured primary (planning ran grok-4.5 instead of the configured gpt-5.6-sol on every session).
Fallback-only grok-cli selections without a visible key now keep the primary on its own runtime and
drop the unusable fallback pair with a warning + audit flag (see dropGrokCliFallbackForNoVisibleKey).
FNXC:GrokCliRouting 2026-07-09-23:05: FNXC:GrokCliRouting 2026-07-09-23:05:
FN-7761 closes the packaged serve/daemon/dashboard gap: if grok-cli is selected and no Fusion-visible key exists, this seam must never silently fall through to the key-requiring pi/openai-completions runtime when the Grok plugin was not pre-installed. The hosts eagerly install/load the bundled runtime; if that genuinely fails, throw an operator-actionable error naming the two supported remediations. FN-7761 closes the packaged serve/daemon/dashboard gap: if grok-cli is selected and no Fusion-visible key exists, this seam must never silently fall through to the key-requiring pi/openai-completions runtime when the Grok plugin was not pre-installed. The hosts eagerly install/load the bundled runtime; if that genuinely fails, throw an operator-actionable error naming the two supported remediations.
@@ -760,11 +782,25 @@ export async function createResolvedAgentSession(
// selections must fail here instead so pi never attempts registry resolution. // selections must fail here instead so pi never attempts registry resolution.
deriveOmpRuntimeHint(runtimeOptions, pluginRunner); deriveOmpRuntimeHint(runtimeOptions, pluginRunner);
} }
/*
FNXC:GrokCliRouting 2026-07-22-14:30:
When only the fallback is grok-cli with no visible key (and the session is not explicitly
hinted onto the Grok runtime), the fallback is unusable — drop it so the configured primary
keeps running on its own runtime instead of being preempted.
*/
const grokFallbackDrop = !useMockRuntime && !autoGrokRuntimeHint && effectiveRuntimeHint !== "grok"
? dropGrokCliFallbackForNoVisibleKey(effectiveRuntimeOptions)
: { options: effectiveRuntimeOptions, dropped: false };
const effectiveRuntimeOptionsWithModel: AgentRuntimeOptions = autoGrokRuntimeHint const effectiveRuntimeOptionsWithModel: AgentRuntimeOptions = autoGrokRuntimeHint
? applyGrokCliNoKeyRuntimeOptions(effectiveRuntimeOptions) ? applyGrokCliNoKeyRuntimeOptions(effectiveRuntimeOptions)
: usesOmpRuntime : usesOmpRuntime
? applyOmpCliRuntimeOptions(effectiveRuntimeOptions) ? applyOmpCliRuntimeOptions(grokFallbackDrop.options)
: effectiveRuntimeOptions; : grokFallbackDrop.options;
if (grokFallbackDrop.dropped) {
sessionLog.warn(
`[${sessionPurpose}] configured grok-cli fallback "${runtimeOptions.fallbackModelId ?? "unknown"}" dropped: no Fusion-visible GROK_API_KEY and cross-runtime fallback swaps are unsupported; primary "${runtimeOptions.defaultProvider}/${runtimeOptions.defaultModelId}" is unchanged. Set GROK_API_KEY or configure a same-runtime fallback.`,
);
}
const resolved = useMockRuntime const resolved = useMockRuntime
? { ? {
@@ -832,10 +868,18 @@ export async function createResolvedAgentSession(
sessionPurpose, sessionPurpose,
runtimeId: resolved.runtimeId, runtimeId: resolved.runtimeId,
wasConfigured: resolved.wasConfigured, wasConfigured: resolved.wasConfigured,
provider: runtimeOptions.defaultProvider ?? null, /*
modelId: runtimeOptions.defaultModelId ?? null, FNXC:ModelResolution 2026-07-22-14:30:
Audit the POST-transform pair the session actually runs, not the pre-transform
configuration. The old pre-transform values masked the FN-7758 fallback promotion:
run-audit claimed planning used the configured primary while the session ran the
promoted grok fallback.
*/
provider: effectiveRuntimeOptionsWithModel.defaultProvider ?? null,
modelId: effectiveRuntimeOptionsWithModel.defaultModelId ?? null,
mockProviderActive, mockProviderActive,
testModeActive, testModeActive,
...(grokFallbackDrop.dropped ? { grokCliFallbackDropped: true } : {}),
...(noModelResolved ? { noModelResolved: true, runtimeBuiltInFallbackModel } : {}), ...(noModelResolved ? { noModelResolved: true, runtimeBuiltInFallbackModel } : {}),
/* /*
FNXC:FusionToolBridgeDiagnostics 2026-07-20-08:00: FNXC:FusionToolBridgeDiagnostics 2026-07-20-08:00:
@@ -855,7 +899,8 @@ export async function createResolvedAgentSession(
...(effectiveRuntimeHint ? { runtimeHint: effectiveRuntimeHint } : {}), ...(effectiveRuntimeHint ? { runtimeHint: effectiveRuntimeHint } : {}),
...(autoGrokRuntimeHint ? { reason: "grok-cli-no-visible-key" } : {}), ...(autoGrokRuntimeHint ? { reason: "grok-cli-no-visible-key" } : {}),
...(autoOmpRuntimeHint ? { reason: "omp-cli-runtime" } : {}), ...(autoOmpRuntimeHint ? { reason: "omp-cli-runtime" } : {}),
...(!autoGrokRuntimeHint && !autoOmpRuntimeHint && "fallbackReason" in resolved && resolved.fallbackReason ? { reason: resolved.fallbackReason } : {}), ...(grokFallbackDrop.dropped ? { reason: "grok-cli-fallback-dropped-no-visible-key" } : {}),
...(!autoGrokRuntimeHint && !autoOmpRuntimeHint && !grokFallbackDrop.dropped && "fallbackReason" in resolved && resolved.fallbackReason ? { reason: resolved.fallbackReason } : {}),
}, },
}); });
} catch (err) { } catch (err) {