fix(engine): stop grok-cli fallback preempting the configured primary model
A configured grok-cli fallback with no Fusion-visible GROK_API_KEY was promoted to primary at session start (FN-7758 seam), so every planning session silently ran grok-4.5 instead of the configured planning model. The no-visible-key Grok CLI auto-route now fires only for a grok-cli primary; a fallback-only grok-cli pair is dropped with a warning and an audited grokCliFallbackDropped flag, and session:runtime-resolved now records the post-transform model pair the session actually runs. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
7
.changeset/grok-fallback-no-preempt.md
Normal file
7
.changeset/grok-fallback-no-preempt.md
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
---
|
||||||
|
"@runfusion/fusion": patch
|
||||||
|
---
|
||||||
|
|
||||||
|
summary: Fix grok-cli fallback models silently replacing the configured primary model when no GROK_API_KEY is visible.
|
||||||
|
category: fix
|
||||||
|
dev: The FN-7758 no-visible-key seam no longer promotes a grok-cli fallback to primary at session start; only a grok-cli primary auto-routes to the Grok CLI runtime. A fallback-only grok-cli pair without a visible key is dropped with a session warning, `grokCliFallbackDropped: true`, and reason `grok-cli-fallback-dropped-no-visible-key` in the `session:runtime-resolved` audit event, which now also records the post-transform provider/model pair the session actually runs.
|
||||||
@@ -342,7 +342,8 @@ describe("Grok CLI runtime routing (FN-7725)", () => {
|
|||||||
runtimeHint: "grok",
|
runtimeHint: "grok",
|
||||||
reason: "grok-cli-no-visible-key",
|
reason: "grok-cli-no-visible-key",
|
||||||
provider: "grok-cli",
|
provider: "grok-cli",
|
||||||
modelId: "grok-cli/grok-4.5",
|
// FNXC:ModelResolution 2026-07-22-14:30: audit records the post-transform model the session actually runs (provider prefix stripped for the CLI).
|
||||||
|
modelId: "grok-4.5",
|
||||||
}),
|
}),
|
||||||
}));
|
}));
|
||||||
});
|
});
|
||||||
@@ -425,12 +426,19 @@ describe("Grok CLI runtime routing (FN-7725)", () => {
|
|||||||
}));
|
}));
|
||||||
});
|
});
|
||||||
|
|
||||||
it("auto-routes a grok-cli fallback model to the Grok runtime when no Fusion-visible key exists", async () => {
|
/*
|
||||||
|
FNXC:GrokCliRouting 2026-07-22-14:30:
|
||||||
|
A grok-cli FALLBACK behind a healthy non-grok primary must never preempt the primary.
|
||||||
|
The old FN-7758 behavior routed the whole session onto the Grok CLI runtime and promoted
|
||||||
|
the fallback model up front, so every planning session silently ran grok-4.5 instead of
|
||||||
|
the configured primary. The fixed invariant: primary stays on its own runtime with its
|
||||||
|
configured model; the unusable grok-cli fallback pair is dropped and the drop is audited.
|
||||||
|
*/
|
||||||
|
it("keeps the configured primary and drops a grok-cli fallback when no Fusion-visible key exists", async () => {
|
||||||
vi.mocked(fusionCore.isGrokApiKeyFusionVisible).mockReturnValue(false);
|
vi.mocked(fusionCore.isGrokApiKeyFusionVisible).mockReturnValue(false);
|
||||||
const grokRegistration = await createGrokRegistration();
|
const grokRegistration = await createGrokRegistration();
|
||||||
const pluginRunner = createMockPluginRunner({
|
const getRuntimeById = vi.fn().mockReturnValue(grokRegistration);
|
||||||
getRuntimeById: vi.fn().mockReturnValue(grokRegistration),
|
const pluginRunner = createMockPluginRunner({ getRuntimeById });
|
||||||
});
|
|
||||||
const audit = { database: vi.fn().mockResolvedValue(undefined) };
|
const audit = { database: vi.fn().mockResolvedValue(undefined) };
|
||||||
|
|
||||||
const result = await createResolvedAgentSession({
|
const result = await createResolvedAgentSession({
|
||||||
@@ -445,23 +453,26 @@ describe("Grok CLI runtime routing (FN-7725)", () => {
|
|||||||
systemPrompt: "fallback-selection-only",
|
systemPrompt: "fallback-selection-only",
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(result.runtimeId).toBe("grok");
|
expect(result.runtimeId).toBe("pi");
|
||||||
expect(result.wasConfigured).toBe(true);
|
expect(mockCreateFnAgent).toHaveBeenCalledWith(expect.objectContaining({
|
||||||
expect(mockCreateFnAgent).not.toHaveBeenCalled();
|
defaultProvider: "openai",
|
||||||
expect(result.session).toMatchObject({ model: "grok-4.5" });
|
defaultModelId: "gpt-4o",
|
||||||
|
fallbackProvider: undefined,
|
||||||
|
fallbackModelId: undefined,
|
||||||
|
}));
|
||||||
expect(audit.database).toHaveBeenCalledWith(expect.objectContaining({
|
expect(audit.database).toHaveBeenCalledWith(expect.objectContaining({
|
||||||
type: "session:runtime-resolved",
|
type: "session:runtime-resolved",
|
||||||
target: "grok",
|
target: "pi",
|
||||||
metadata: expect.objectContaining({
|
metadata: expect.objectContaining({
|
||||||
runtimeHint: "grok",
|
reason: "grok-cli-fallback-dropped-no-visible-key",
|
||||||
reason: "grok-cli-no-visible-key",
|
grokCliFallbackDropped: true,
|
||||||
provider: "openai",
|
provider: "openai",
|
||||||
modelId: "gpt-4o",
|
modelId: "gpt-4o",
|
||||||
}),
|
}),
|
||||||
}));
|
}));
|
||||||
});
|
});
|
||||||
|
|
||||||
it("auto-routes a bare grok-cli fallback model id without adding a provider prefix", async () => {
|
it("drops a bare grok-cli fallback model id while leaving the primary untouched", async () => {
|
||||||
vi.mocked(fusionCore.isGrokApiKeyFusionVisible).mockReturnValue(false);
|
vi.mocked(fusionCore.isGrokApiKeyFusionVisible).mockReturnValue(false);
|
||||||
const grokRegistration = await createGrokRegistration();
|
const grokRegistration = await createGrokRegistration();
|
||||||
const pluginRunner = createMockPluginRunner({
|
const pluginRunner = createMockPluginRunner({
|
||||||
@@ -479,8 +490,37 @@ describe("Grok CLI runtime routing (FN-7725)", () => {
|
|||||||
systemPrompt: "fallback-bare-model",
|
systemPrompt: "fallback-bare-model",
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(result.runtimeId).toBe("grok");
|
expect(result.runtimeId).toBe("pi");
|
||||||
expect(result.session).toMatchObject({ model: "grok-4.5" });
|
expect(mockCreateFnAgent).toHaveBeenCalledWith(expect.objectContaining({
|
||||||
|
defaultProvider: "anthropic",
|
||||||
|
defaultModelId: "claude-sonnet-4-5",
|
||||||
|
fallbackProvider: undefined,
|
||||||
|
fallbackModelId: undefined,
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps a grok-cli fallback when a Fusion-visible key exists (pi can resolve it directly)", async () => {
|
||||||
|
vi.mocked(fusionCore.isGrokApiKeyFusionVisible).mockReturnValue(true);
|
||||||
|
const pluginRunner = createMockPluginRunner();
|
||||||
|
|
||||||
|
const result = await createResolvedAgentSession({
|
||||||
|
sessionPurpose: "executor",
|
||||||
|
pluginRunner,
|
||||||
|
cwd: "/tmp/project",
|
||||||
|
defaultProvider: "openai",
|
||||||
|
defaultModelId: "gpt-4o",
|
||||||
|
fallbackProvider: "grok-cli",
|
||||||
|
fallbackModelId: "grok-4.5",
|
||||||
|
systemPrompt: "visible-key-fallback",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.runtimeId).toBe("pi");
|
||||||
|
expect(mockCreateFnAgent).toHaveBeenCalledWith(expect.objectContaining({
|
||||||
|
defaultProvider: "openai",
|
||||||
|
defaultModelId: "gpt-4o",
|
||||||
|
fallbackProvider: "grok-cli",
|
||||||
|
fallbackModelId: "grok-4.5",
|
||||||
|
}));
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps mock/test-mode provider routing on the mock runtime when grok-cli fallback is configured", async () => {
|
it("keeps mock/test-mode provider routing on the mock runtime when grok-cli fallback is configured", async () => {
|
||||||
|
|||||||
@@ -429,12 +429,20 @@ function buildMissingGrokRuntimeError(): Error {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
FNXC:GrokCliRouting 2026-07-22-14:30:
|
||||||
|
The no-visible-key Grok CLI auto-derive fires only when grok-cli is the PRIMARY provider.
|
||||||
|
FN-7758 used to fire on a grok-cli FALLBACK too and promoted it to primary up front, which
|
||||||
|
silently replaced a healthy configured primary (e.g. planning openai-codex/gpt-5.6-sol ran
|
||||||
|
as grok/grok-4.5 on every triage session because the workflow's planningFallback was
|
||||||
|
grok-cli). A fallback must never preempt a primary that has not failed; the fallback-only
|
||||||
|
case is handled by dropGrokCliFallbackForNoVisibleKey below.
|
||||||
|
*/
|
||||||
function deriveGrokRuntimeHintForNoVisibleKey(
|
function deriveGrokRuntimeHintForNoVisibleKey(
|
||||||
runtimeOptions: AgentRuntimeOptions,
|
runtimeOptions: AgentRuntimeOptions,
|
||||||
pluginRunner: PluginRunner | undefined,
|
pluginRunner: PluginRunner | undefined,
|
||||||
): string | undefined {
|
): string | undefined {
|
||||||
if (runtimeOptions.defaultProvider !== GROK_CLI_PROVIDER_ID
|
if (runtimeOptions.defaultProvider !== GROK_CLI_PROVIDER_ID) return undefined;
|
||||||
&& runtimeOptions.fallbackProvider !== GROK_CLI_PROVIDER_ID) return undefined;
|
|
||||||
if (isGrokApiKeyFusionVisible()) return undefined;
|
if (isGrokApiKeyFusionVisible()) return undefined;
|
||||||
try {
|
try {
|
||||||
if (pluginRunner?.getRuntimeById("grok")) return "grok";
|
if (pluginRunner?.getRuntimeById("grok")) return "grok";
|
||||||
@@ -454,23 +462,35 @@ function applyGrokCliNoKeyRuntimeOptions(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (runtimeOptions.fallbackProvider === GROK_CLI_PROVIDER_ID) {
|
return runtimeOptions;
|
||||||
return {
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
FNXC:GrokCliRouting 2026-07-22-14:30:
|
||||||
|
A grok-cli fallback behind a non-grok primary is unusable without a Fusion-visible
|
||||||
|
GROK_API_KEY: pi resolves fallback swaps through the key-requiring provider registry
|
||||||
|
(the original FN-7758 failure mode), and cross-runtime swaps into the Grok CLI runtime
|
||||||
|
are not supported mid-session. Drop the fallback pair, keep the configured primary
|
||||||
|
untouched, and surface the drop via a session warning plus run-audit metadata so the
|
||||||
|
operator can fix the lane (set GROK_API_KEY or pick a same-runtime fallback).
|
||||||
|
*/
|
||||||
|
function dropGrokCliFallbackForNoVisibleKey(
|
||||||
|
runtimeOptions: AgentRuntimeOptions,
|
||||||
|
): { options: AgentRuntimeOptions; dropped: boolean } {
|
||||||
|
if (runtimeOptions.defaultProvider === GROK_CLI_PROVIDER_ID
|
||||||
|
|| runtimeOptions.fallbackProvider !== GROK_CLI_PROVIDER_ID
|
||||||
|
|| isGrokApiKeyFusionVisible()) {
|
||||||
|
return { options: runtimeOptions, dropped: false };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
options: {
|
||||||
...runtimeOptions,
|
...runtimeOptions,
|
||||||
defaultProvider: runtimeOptions.fallbackProvider,
|
|
||||||
defaultModelId: stripGrokCliModelProviderPrefix(runtimeOptions.fallbackModelId),
|
|
||||||
/*
|
|
||||||
* FNXC:Settings-ThinkingLevel 2026-07-10-00:00:
|
|
||||||
* When the no-visible-key Grok CLI fallback is promoted to the primary runtime, promote its fallback thinking level too; the cleared fallback pair must not leave the Grok CLI session using the superseded primary model's thinking level.
|
|
||||||
*/
|
|
||||||
defaultThinkingLevel: runtimeOptions.fallbackThinkingLevel ?? runtimeOptions.defaultThinkingLevel,
|
|
||||||
fallbackProvider: undefined,
|
fallbackProvider: undefined,
|
||||||
fallbackModelId: undefined,
|
fallbackModelId: undefined,
|
||||||
fallbackThinkingLevel: undefined,
|
fallbackThinkingLevel: undefined,
|
||||||
};
|
},
|
||||||
}
|
dropped: true,
|
||||||
|
};
|
||||||
return runtimeOptions;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function pickSettingsThenRuntimeModel(
|
function pickSettingsThenRuntimeModel(
|
||||||
@@ -739,10 +759,12 @@ export async function createResolvedAgentSession(
|
|||||||
provider routing stays on the mock runtime. Strip only the provider-qualified model prefix so the CLI
|
provider routing stays on the mock runtime. Strip only the provider-qualified model prefix so the CLI
|
||||||
receives the concrete selected model via GrokRuntimeAdapter without changing non-grok sessions.
|
receives the concrete selected model via GrokRuntimeAdapter without changing non-grok sessions.
|
||||||
|
|
||||||
FNXC:GrokCliRouting 2026-07-09-22:10:
|
FNXC:GrokCliRouting 2026-07-22-14:30:
|
||||||
FN-7758 extends the no-visible-key invariant to configured fallback models. Pi resolves fallback models
|
FN-7758's fallback-promotion behavior is removed: a configured grok-cli FALLBACK no longer selects the
|
||||||
during session creation and prompt-time swaps through the key-requiring provider registry, so a grok-cli
|
Grok CLI runtime or replaces the primary model up front. That promotion silently discarded a healthy
|
||||||
fallback must select the Grok CLI runtime up front and promote the fallback model into the CLI session.
|
configured primary (planning ran grok-4.5 instead of the configured gpt-5.6-sol on every session).
|
||||||
|
Fallback-only grok-cli selections without a visible key now keep the primary on its own runtime and
|
||||||
|
drop the unusable fallback pair with a warning + audit flag (see dropGrokCliFallbackForNoVisibleKey).
|
||||||
|
|
||||||
FNXC:GrokCliRouting 2026-07-09-23:05:
|
FNXC:GrokCliRouting 2026-07-09-23:05:
|
||||||
FN-7761 closes the packaged serve/daemon/dashboard gap: if grok-cli is selected and no Fusion-visible key exists, this seam must never silently fall through to the key-requiring pi/openai-completions runtime when the Grok plugin was not pre-installed. The hosts eagerly install/load the bundled runtime; if that genuinely fails, throw an operator-actionable error naming the two supported remediations.
|
FN-7761 closes the packaged serve/daemon/dashboard gap: if grok-cli is selected and no Fusion-visible key exists, this seam must never silently fall through to the key-requiring pi/openai-completions runtime when the Grok plugin was not pre-installed. The hosts eagerly install/load the bundled runtime; if that genuinely fails, throw an operator-actionable error naming the two supported remediations.
|
||||||
@@ -760,11 +782,25 @@ export async function createResolvedAgentSession(
|
|||||||
// selections must fail here instead so pi never attempts registry resolution.
|
// selections must fail here instead so pi never attempts registry resolution.
|
||||||
deriveOmpRuntimeHint(runtimeOptions, pluginRunner);
|
deriveOmpRuntimeHint(runtimeOptions, pluginRunner);
|
||||||
}
|
}
|
||||||
|
/*
|
||||||
|
FNXC:GrokCliRouting 2026-07-22-14:30:
|
||||||
|
When only the fallback is grok-cli with no visible key (and the session is not explicitly
|
||||||
|
hinted onto the Grok runtime), the fallback is unusable — drop it so the configured primary
|
||||||
|
keeps running on its own runtime instead of being preempted.
|
||||||
|
*/
|
||||||
|
const grokFallbackDrop = !useMockRuntime && !autoGrokRuntimeHint && effectiveRuntimeHint !== "grok"
|
||||||
|
? dropGrokCliFallbackForNoVisibleKey(effectiveRuntimeOptions)
|
||||||
|
: { options: effectiveRuntimeOptions, dropped: false };
|
||||||
const effectiveRuntimeOptionsWithModel: AgentRuntimeOptions = autoGrokRuntimeHint
|
const effectiveRuntimeOptionsWithModel: AgentRuntimeOptions = autoGrokRuntimeHint
|
||||||
? applyGrokCliNoKeyRuntimeOptions(effectiveRuntimeOptions)
|
? applyGrokCliNoKeyRuntimeOptions(effectiveRuntimeOptions)
|
||||||
: usesOmpRuntime
|
: usesOmpRuntime
|
||||||
? applyOmpCliRuntimeOptions(effectiveRuntimeOptions)
|
? applyOmpCliRuntimeOptions(grokFallbackDrop.options)
|
||||||
: effectiveRuntimeOptions;
|
: grokFallbackDrop.options;
|
||||||
|
if (grokFallbackDrop.dropped) {
|
||||||
|
sessionLog.warn(
|
||||||
|
`[${sessionPurpose}] configured grok-cli fallback "${runtimeOptions.fallbackModelId ?? "unknown"}" dropped: no Fusion-visible GROK_API_KEY and cross-runtime fallback swaps are unsupported; primary "${runtimeOptions.defaultProvider}/${runtimeOptions.defaultModelId}" is unchanged. Set GROK_API_KEY or configure a same-runtime fallback.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const resolved = useMockRuntime
|
const resolved = useMockRuntime
|
||||||
? {
|
? {
|
||||||
@@ -832,10 +868,18 @@ export async function createResolvedAgentSession(
|
|||||||
sessionPurpose,
|
sessionPurpose,
|
||||||
runtimeId: resolved.runtimeId,
|
runtimeId: resolved.runtimeId,
|
||||||
wasConfigured: resolved.wasConfigured,
|
wasConfigured: resolved.wasConfigured,
|
||||||
provider: runtimeOptions.defaultProvider ?? null,
|
/*
|
||||||
modelId: runtimeOptions.defaultModelId ?? null,
|
FNXC:ModelResolution 2026-07-22-14:30:
|
||||||
|
Audit the POST-transform pair the session actually runs, not the pre-transform
|
||||||
|
configuration. The old pre-transform values masked the FN-7758 fallback promotion:
|
||||||
|
run-audit claimed planning used the configured primary while the session ran the
|
||||||
|
promoted grok fallback.
|
||||||
|
*/
|
||||||
|
provider: effectiveRuntimeOptionsWithModel.defaultProvider ?? null,
|
||||||
|
modelId: effectiveRuntimeOptionsWithModel.defaultModelId ?? null,
|
||||||
mockProviderActive,
|
mockProviderActive,
|
||||||
testModeActive,
|
testModeActive,
|
||||||
|
...(grokFallbackDrop.dropped ? { grokCliFallbackDropped: true } : {}),
|
||||||
...(noModelResolved ? { noModelResolved: true, runtimeBuiltInFallbackModel } : {}),
|
...(noModelResolved ? { noModelResolved: true, runtimeBuiltInFallbackModel } : {}),
|
||||||
/*
|
/*
|
||||||
FNXC:FusionToolBridgeDiagnostics 2026-07-20-08:00:
|
FNXC:FusionToolBridgeDiagnostics 2026-07-20-08:00:
|
||||||
@@ -855,7 +899,8 @@ export async function createResolvedAgentSession(
|
|||||||
...(effectiveRuntimeHint ? { runtimeHint: effectiveRuntimeHint } : {}),
|
...(effectiveRuntimeHint ? { runtimeHint: effectiveRuntimeHint } : {}),
|
||||||
...(autoGrokRuntimeHint ? { reason: "grok-cli-no-visible-key" } : {}),
|
...(autoGrokRuntimeHint ? { reason: "grok-cli-no-visible-key" } : {}),
|
||||||
...(autoOmpRuntimeHint ? { reason: "omp-cli-runtime" } : {}),
|
...(autoOmpRuntimeHint ? { reason: "omp-cli-runtime" } : {}),
|
||||||
...(!autoGrokRuntimeHint && !autoOmpRuntimeHint && "fallbackReason" in resolved && resolved.fallbackReason ? { reason: resolved.fallbackReason } : {}),
|
...(grokFallbackDrop.dropped ? { reason: "grok-cli-fallback-dropped-no-visible-key" } : {}),
|
||||||
|
...(!autoGrokRuntimeHint && !autoOmpRuntimeHint && !grokFallbackDrop.dropped && "fallbackReason" in resolved && resolved.fallbackReason ? { reason: resolved.fallbackReason } : {}),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|||||||
Reference in New Issue
Block a user