diff --git a/.changeset/fn-7838-cli-timeout-configurable.md b/.changeset/fn-7838-cli-timeout-configurable.md new file mode 100644 index 0000000000..eca301909e --- /dev/null +++ b/.changeset/fn-7838-cli-timeout-configurable.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": minor +--- + +summary: CLI agent cold-start timeouts now default to 2 minutes and are configurable. +category: feature +dev: Grok honors GROK_CLI_FIRST_OUTPUT_TIMEOUT_MS; Droid honors PI_DROID_CLI_FIRST_LINE_TIMEOUT_MS. Defaults raised 60000 → 120000. Inactivity ceilings unchanged. diff --git a/docs/settings-reference.md b/docs/settings-reference.md index 77f2fb19c2..702d9c535f 100644 --- a/docs/settings-reference.md +++ b/docs/settings-reference.md @@ -1192,7 +1192,18 @@ fn plugin install ./plugins/fusion-plugin-openclaw-runtime 3. Assign the agent to tasks that should use this runtime. -For more details, see the [Paperclip Runtime Plugin documentation](../plugins/fusion-plugin-paperclip-runtime/README.md), [Hermes Runtime Plugin documentation](../plugins/fusion-plugin-hermes-runtime/README.md), and [OpenClaw Runtime Plugin documentation](../plugins/fusion-plugin-openclaw-runtime/README.md). +For more details, see the [Paperclip Runtime Plugin documentation](../plugins/fusion-plugin-paperclip-runtime/README.md), [Hermes Runtime Plugin documentation](../plugins/fusion-plugin-hermes-runtime/README.md), [OpenClaw Runtime Plugin documentation](../plugins/fusion-plugin-openclaw-runtime/README.md), [Grok Runtime Plugin documentation](../plugins/fusion-plugin-grok-runtime/README.md), and [Droid Runtime Plugin documentation](../plugins/fusion-plugin-droid-runtime/README.md). + +### CLI Runtime Cold-Start Timeout Configuration + +Grok and Droid have first-output cold-start guards for subprocesses that never emit initial stdout. These guards are configured by environment variable only and resolve as: environment variable → built-in default. Blank, non-numeric, zero, or negative values are ignored and the default remains active. + +| Runtime | Environment Variable | Default if Unset | Description | +|---|---|---|---| +| Grok | `GROK_CLI_FIRST_OUTPUT_TIMEOUT_MS` | `120000` | Fusion-side cold-start / first-stdout-byte kill ceiling for `grok` headless prompts. | +| Droid | `PI_DROID_CLI_FIRST_LINE_TIMEOUT_MS` | `120000` | Fusion-side cold-start / first-stdout-line kill ceiling for `droid` CLI streams. | + +These cold-start guards are distinct from OpenClaw/Hermes `cliTimeoutMs` settings (full-turn subprocess hard kills, default `300000`) and from Grok/Droid's separate 30-minute inactivity safety nets after output has begun. ### OpenClaw Runtime Configuration diff --git a/plugins/fusion-plugin-droid-runtime/README.md b/plugins/fusion-plugin-droid-runtime/README.md index 9326086f07..edc9643324 100644 --- a/plugins/fusion-plugin-droid-runtime/README.md +++ b/plugins/fusion-plugin-droid-runtime/README.md @@ -24,6 +24,14 @@ Core implementation files live in `src/`: - `probe.ts` - prompt/tool/thinking/control helpers +## Configuration + +| Environment variable | Default | Description | +|---|---|---| +| `PI_DROID_CLI_FIRST_LINE_TIMEOUT_MS` | `120000` | Cold-start / first-stdout-line kill ceiling for Droid CLI streams. Blank, non-numeric, zero, or negative values fall back to the default. | + +The first-line guard is separate from the 30-minute inactivity safety net that applies after stdout has begun. + ## Dashboard UI contribution surfaces The plugin registers `uiSlots` for: diff --git a/plugins/fusion-plugin-droid-runtime/src/__tests__/provider.test.ts b/plugins/fusion-plugin-droid-runtime/src/__tests__/provider.test.ts index 76fd2571ed..4390a72bac 100644 --- a/plugins/fusion-plugin-droid-runtime/src/__tests__/provider.test.ts +++ b/plugins/fusion-plugin-droid-runtime/src/__tests__/provider.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it, vi, beforeEach } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { EventEmitter } from "node:events"; import { PassThrough } from "node:stream"; @@ -58,6 +58,12 @@ function makeProc() { describe("streamViaCli", () => { beforeEach(() => { vi.clearAllMocks(); + delete process.env.PI_DROID_CLI_FIRST_LINE_TIMEOUT_MS; + }); + + afterEach(() => { + delete process.env.PI_DROID_CLI_FIRST_LINE_TIMEOUT_MS; + vi.useRealTimers(); }); it("spawns droid and writes prompt", async () => { @@ -112,4 +118,93 @@ describe("streamViaCli", () => { expect.objectContaining({ mcpConfigPath: "/tmp/mcp.json" }), ); }); + + it("kills the subprocess if no stdout line arrives within the default cold-start ceiling", async () => { + vi.useFakeTimers(); + const proc = makeProc(); + mocks.spawnDroid.mockReturnValue(proc); + + const stream = streamViaCli( + { id: "droid-pro", provider: "droid-cli" } as any, + { messages: [{ role: "user", content: "hi" }] } as any, + ) as any; + const push = vi.spyOn(stream, "push"); + + await vi.advanceTimersByTimeAsync(119_999); + expect(mocks.forceKillProcess).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + + expect(mocks.forceKillProcess).toHaveBeenCalledWith(proc); + expect(push).toHaveBeenCalledWith(expect.objectContaining({ + type: "done", + message: expect.objectContaining({ + content: [expect.objectContaining({ + text: expect.stringContaining("Droid CLI produced no output within 120s"), + })], + }), + })); + proc.emit("close", null, "SIGKILL"); + await Promise.resolve(); + }); + + it("uses PI_DROID_CLI_FIRST_LINE_TIMEOUT_MS when it is a positive integer", async () => { + vi.useFakeTimers(); + process.env.PI_DROID_CLI_FIRST_LINE_TIMEOUT_MS = "25"; + const proc = makeProc(); + mocks.spawnDroid.mockReturnValue(proc); + + const stream = streamViaCli( + { id: "droid-pro", provider: "droid-cli" } as any, + { messages: [{ role: "user", content: "hi" }] } as any, + ) as any; + const push = vi.spyOn(stream, "push"); + + await vi.advanceTimersByTimeAsync(24); + expect(mocks.forceKillProcess).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + + expect(mocks.forceKillProcess).toHaveBeenCalledWith(proc); + expect(push).toHaveBeenCalledWith(expect.objectContaining({ + type: "done", + message: expect.objectContaining({ + content: [expect.objectContaining({ + text: expect.stringContaining("Droid CLI produced no output within 0.025s"), + })], + }), + })); + proc.emit("close", null, "SIGKILL"); + await Promise.resolve(); + }); + + it.each(["", " ", "nope", "0", "-1", "1.5"])( + "falls back to the default cold-start ceiling for invalid PI_DROID_CLI_FIRST_LINE_TIMEOUT_MS=%j", + async (value) => { + vi.useFakeTimers(); + process.env.PI_DROID_CLI_FIRST_LINE_TIMEOUT_MS = value; + const proc = makeProc(); + mocks.spawnDroid.mockReturnValue(proc); + + const stream = streamViaCli( + { id: "droid-pro", provider: "droid-cli" } as any, + { messages: [{ role: "user", content: "hi" }] } as any, + ) as any; + const push = vi.spyOn(stream, "push"); + + await vi.advanceTimersByTimeAsync(119_999); + expect(mocks.forceKillProcess).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + + expect(mocks.forceKillProcess).toHaveBeenCalledWith(proc); + expect(push).toHaveBeenCalledWith(expect.objectContaining({ + type: "done", + message: expect.objectContaining({ + content: [expect.objectContaining({ + text: expect.stringContaining("Droid CLI produced no output within 120s"), + })], + }), + })); + proc.emit("close", null, "SIGKILL"); + await Promise.resolve(); + }, + ); }); diff --git a/plugins/fusion-plugin-droid-runtime/src/provider.ts b/plugins/fusion-plugin-droid-runtime/src/provider.ts index c071db8c05..9ead55aa07 100644 --- a/plugins/fusion-plugin-droid-runtime/src/provider.ts +++ b/plugins/fusion-plugin-droid-runtime/src/provider.ts @@ -66,10 +66,25 @@ const INACTIVITY_TIMEOUT_MS = 30 * 60_000; * Cold-start ceiling: kill the subprocess if it hasn't produced a single line * of stdout within this window. Distinct from INACTIVITY_TIMEOUT_MS so a hung * binary (no output ever) is reported with a clear cause instead of being - * indistinguishable from a slow-thinking turn. Observed cold-start on a healthy - * droid is ~20s; 60s gives 3x headroom for slow machines / cold caches. + * indistinguishable from a slow-thinking turn. + * + * FNXC:DroidCli 2026-07-11-00:00: + * FN-7838 raises Droid's cold-start ceiling from 60s to 120s because slow/cold first-token starts were killed prematurely. Operators can override the first-line guard with PI_DROID_CLI_FIRST_LINE_TIMEOUT_MS; invalid values fall back to the default so the guard is never disabled. This mirrors the OpenClaw/Hermes *_CLI_TIMEOUT_MS precedence pattern while keeping the 30-minute inactivity safety net separate. */ -const FIRST_LINE_TIMEOUT_MS = 60_000; +const DEFAULT_FIRST_LINE_TIMEOUT_MS = 120_000; +const PI_DROID_FIRST_LINE_TIMEOUT_ENV = "PI_DROID_CLI_FIRST_LINE_TIMEOUT_MS"; + +function parsePositiveInteger(value: string | undefined): number | undefined { + const trimmed = value?.trim(); + if (!trimmed) return undefined; + const parsed = Number(trimmed); + if (!Number.isSafeInteger(parsed) || parsed <= 0) return undefined; + return parsed; +} + +function resolveFirstLineTimeoutMs(): number { + return parsePositiveInteger(process.env[PI_DROID_FIRST_LINE_TIMEOUT_ENV]) ?? DEFAULT_FIRST_LINE_TIMEOUT_MS; +} function isDebugStreamEnabled(): boolean { return process.env.PI_DROID_CLI_DEBUG === "1"; } @@ -117,6 +132,7 @@ export function streamViaCli( try { const cwd = options?.cwd ?? process.cwd(); + const firstLineTimeoutMs = resolveFirstLineTimeoutMs(); // Resume if pi provides a session ID AND this isn't the first turn. // Pi passes sessionId on every call (including first), but we can only @@ -282,9 +298,9 @@ export function streamViaCli( if (firstLineReceived) return; forceKillProcess(proc!); endStreamWithError( - `Droid CLI produced no output within ${FIRST_LINE_TIMEOUT_MS / 1000}s — likely binary hang or auth failure (try \`droid --version\` and \`droid auth status\`)`, + `Droid CLI produced no output within ${firstLineTimeoutMs / 1000}s — likely binary hang or auth failure (try \`droid --version\` and \`droid auth status\`)`, ); - }, FIRST_LINE_TIMEOUT_MS); + }, firstLineTimeoutMs); proc.on("close", () => clearTimeout(firstLineTimer)); // Process NDJSON lines from stdout using event-based callback diff --git a/plugins/fusion-plugin-grok-runtime/README.md b/plugins/fusion-plugin-grok-runtime/README.md index 58894e86a1..d3d9278f23 100644 --- a/plugins/fusion-plugin-grok-runtime/README.md +++ b/plugins/fusion-plugin-grok-runtime/README.md @@ -39,6 +39,14 @@ grok -p "" --output-format json -m "grok-4.5" --cwd "/path/to/project" - A wrong-binary/wrong-flag run that emits no parseable JSON surfaces a concrete diagnostic instead of a blank assistant response. - **Auth implication:** because the `grok` binary resolves its own credentials for this path, a CLI-routed selection needs **no Fusion-visible `GROK_API_KEY`** — unlike the direct xAI OpenAI-compatible streaming path. +## Configuration + +| Environment variable | Default | Description | +|---|---|---| +| `GROK_CLI_FIRST_OUTPUT_TIMEOUT_MS` | `120000` | Cold-start / first-stdout-byte kill ceiling for headless `grok` prompts. Blank, non-numeric, zero, or negative values fall back to the default. | + +The first-output guard is separate from the 30-minute inactivity safety net that applies after stdout has begun. + See `docs/grok-cli-contract.md` for the full contract, live captures, and the reason Fusion no longer uses the old `grok --prompt --format json` / `step_*` schema or the flaky streaming-json prompt path. ## Routing Grok through the CLI runtime (FN-7725 / FN-7753 / FN-7790) diff --git a/plugins/fusion-plugin-grok-runtime/src/__tests__/runtime-adapter.test.ts b/plugins/fusion-plugin-grok-runtime/src/__tests__/runtime-adapter.test.ts index 1200950622..88582909e0 100644 --- a/plugins/fusion-plugin-grok-runtime/src/__tests__/runtime-adapter.test.ts +++ b/plugins/fusion-plugin-grok-runtime/src/__tests__/runtime-adapter.test.ts @@ -334,23 +334,71 @@ describe("GrokRuntimeAdapter", () => { describe("lifecycle timeouts (fake timers)", () => { beforeEach(() => { vi.useFakeTimers(); + delete process.env.GROK_CLI_FIRST_OUTPUT_TIMEOUT_MS; }); afterEach(() => { + delete process.env.GROK_CLI_FIRST_OUTPUT_TIMEOUT_MS; vi.useRealTimers(); }); - it("kills the subprocess and resolves if no stdout line arrives within the cold-start ceiling", async () => { + it("kills the subprocess and resolves if no stdout line arrives within the default cold-start ceiling", async () => { const { proc, kill } = makeFakeProc(); const spawn = vi.fn().mockReturnValue(proc); const adapter = new GrokRuntimeAdapter({ spawn }); const { session } = await adapter.createSession({}); const promise = adapter.promptWithFallback(session, "hi"); - await vi.advanceTimersByTimeAsync(60_000); + await vi.advanceTimersByTimeAsync(119_999); + expect(kill).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); await promise; expect(kill).toHaveBeenCalledWith("SIGKILL"); + expect(session.state.errorMessage).toBe( + "Grok CLI produced no stdout within 120000ms for a headless prompt; the process was killed.", + ); }); + + it("uses GROK_CLI_FIRST_OUTPUT_TIMEOUT_MS when it is a positive integer", async () => { + process.env.GROK_CLI_FIRST_OUTPUT_TIMEOUT_MS = "25"; + const { proc, kill } = makeFakeProc(); + const spawn = vi.fn().mockReturnValue(proc); + const adapter = new GrokRuntimeAdapter({ spawn }); + const { session } = await adapter.createSession({}); + + const promise = adapter.promptWithFallback(session, "hi"); + await vi.advanceTimersByTimeAsync(24); + expect(kill).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + + await promise; + expect(kill).toHaveBeenCalledWith("SIGKILL"); + expect(session.state.errorMessage).toBe( + "Grok CLI produced no stdout within 25ms for a headless prompt; the process was killed.", + ); + }); + + it.each(["", " ", "nope", "0", "-1", "1.5"])( + "falls back to the default cold-start ceiling for invalid GROK_CLI_FIRST_OUTPUT_TIMEOUT_MS=%j", + async (value) => { + process.env.GROK_CLI_FIRST_OUTPUT_TIMEOUT_MS = value; + const { proc, kill } = makeFakeProc(); + const spawn = vi.fn().mockReturnValue(proc); + const adapter = new GrokRuntimeAdapter({ spawn }); + const { session } = await adapter.createSession({}); + + const promise = adapter.promptWithFallback(session, "hi"); + await vi.advanceTimersByTimeAsync(119_999); + expect(kill).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + + await promise; + expect(kill).toHaveBeenCalledWith("SIGKILL"); + expect(session.state.errorMessage).toBe( + "Grok CLI produced no stdout within 120000ms for a headless prompt; the process was killed.", + ); + }, + ); }); it("resolves without throwing if the injected spawn function throws synchronously and records the diagnostic", async () => { diff --git a/plugins/fusion-plugin-grok-runtime/src/runtime-adapter.ts b/plugins/fusion-plugin-grok-runtime/src/runtime-adapter.ts index a99ea8811e..f87abc9dfa 100644 --- a/plugins/fusion-plugin-grok-runtime/src/runtime-adapter.ts +++ b/plugins/fusion-plugin-grok-runtime/src/runtime-adapter.ts @@ -15,8 +15,24 @@ FN-7753's auto-derived `grok` runtime routing from a `grok-cli/*` model selectio * bytes within this window, treat it as a hung/failed subprocess and resolve * (never reject — mirrors the Droid adapter's resolve-on-error lifecycle so pi * always gets a well-formed, if diagnostic, result instead of an unhandled rejection). + * + * FNXC:GrokCli 2026-07-11-00:00: + * FN-7838 raises Grok's cold-start ceiling from 60s to 120s because slow/cold first-token starts were killed prematurely. Operators can override the first-output guard with GROK_CLI_FIRST_OUTPUT_TIMEOUT_MS; invalid values fall back to the default so the guard is never disabled. This mirrors the OpenClaw/Hermes *_CLI_TIMEOUT_MS precedence pattern while keeping the 30-minute inactivity safety net separate. */ -const FIRST_OUTPUT_TIMEOUT_MS = 60_000; +const DEFAULT_FIRST_OUTPUT_TIMEOUT_MS = 120_000; +const GROK_FIRST_OUTPUT_TIMEOUT_ENV = "GROK_CLI_FIRST_OUTPUT_TIMEOUT_MS"; + +function parsePositiveInteger(value: string | undefined): number | undefined { + const trimmed = value?.trim(); + if (!trimmed) return undefined; + const parsed = Number(trimmed); + if (!Number.isSafeInteger(parsed) || parsed <= 0) return undefined; + return parsed; +} + +function resolveFirstOutputTimeoutMs(): number { + return parsePositiveInteger(process.env[GROK_FIRST_OUTPUT_TIMEOUT_ENV]) ?? DEFAULT_FIRST_OUTPUT_TIMEOUT_MS; +} /** * Inactivity safety net: kill the subprocess if no stdout bytes arrive for @@ -203,6 +219,7 @@ export class GrokRuntimeAdapter implements AgentRuntime { const cwd = options?.cwd; const signal = options?.signal; appendMessage(grokSession, "user", prompt); + const firstOutputTimeoutMs = resolveFirstOutputTimeoutMs(); return new Promise((resolve) => { let proc: GrokStreamProcess; @@ -332,11 +349,11 @@ export class GrokRuntimeAdapter implements AgentRuntime { firstOutputTimer = setTimeout(() => { if (firstOutputReceived) return; setErrorMessage( - `Grok CLI produced no stdout within ${FIRST_OUTPUT_TIMEOUT_MS}ms for a headless prompt; the process was killed.`, + `Grok CLI produced no stdout within ${firstOutputTimeoutMs}ms for a headless prompt; the process was killed.`, ); forceKillGrokStream(proc); finish(); - }, FIRST_OUTPUT_TIMEOUT_MS); + }, firstOutputTimeoutMs); proc.stdout?.on("data", (chunk: Buffer | string) => { const text = chunk.toString();