From 9432339363fe5288b9efb1152fdadbde12a4f534 Mon Sep 17 00:00:00 2001 From: Phil Larson Date: Tue, 30 Jun 2026 23:35:27 -0700 Subject: [PATCH] fix: keep dashboard available after planning failures --- .changeset/dashboard-availability.md | 7 ++ docs/testing.md | 21 ++++ packages/cli/src/bin.ts | 9 +- .../src/commands/__tests__/dashboard.test.ts | 77 ++++++++++++ packages/cli/src/commands/dashboard.ts | 114 ++++++++++++++++++ .../dashboard/src/__tests__/server.test.ts | 40 ++++++ .../__tests__/session-error-recovery.test.ts | 63 ++++++++++ packages/dashboard/src/planning.ts | 1 + 8 files changed, 331 insertions(+), 1 deletion(-) create mode 100644 .changeset/dashboard-availability.md diff --git a/.changeset/dashboard-availability.md b/.changeset/dashboard-availability.md new file mode 100644 index 0000000000..7813f92441 --- /dev/null +++ b/.changeset/dashboard-availability.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Contain planning parse failures as retryable session errors and add `--supervise` dashboard restart mode. +category: fix +dev: Planning sessions that receive non-JSON AI output now persist as retryable error state instead of unpersisting the session. The `/api/health` endpoint remains available during session errors. A new `--supervise` flag on `fn dashboard` runs the dashboard under foreground process supervision with bounded restart attempts and exponential backoff, preventing Tailscale Serve 502s from unexpected dashboard exits. diff --git a/docs/testing.md b/docs/testing.md index e19205157f..4326b93e0b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -67,6 +67,27 @@ Public `@fusion/core` exports consumed by runtime tools should include a literal `packages/engine/src/__tests__/user-configured-command-no-execsync.test.ts` guards user-configured command execution helpers against accidental `execSync` usage or dropped async bounds. Its registry covers verification helpers, `fn_run_verification`, executor configured-command execution, merger post-merge script execution, routine command execution, and the native/bubblewrap/sandbox-exec sandbox backends. Each protected slice must keep the appropriate bounded async safeguard (`timeout`/`timeoutMs`, `maxBuffer`, or `maxLifetimeMs`). The test intentionally slices named function bodies instead of scanning whole files; deterministic git-plumbing `execSync` in merger/self-healing/already-merged/integration/worktree-prune paths and the executor git ancestry check are explicitly out of scope. + +## Dashboard Availability & Supervised Mode + + + +When running the dashboard for extended UX review sessions (e.g., Atlas Notes Jony pass via Tailscale Serve), use the `--supervise` flag to prevent unexpected dashboard exits from leaving the Tailscale endpoint returning 502: + +```bash +fn dashboard --project atlas-notes --port 4040 --supervise +``` + +The supervisor runs the dashboard as a child process with **bounded restart attempts** (one initial run plus up to 3 restarts with exponential backoff: 2s → 4s → 8s). Clean exits (SIGINT, SIGTERM, exit 0) propagate without restart. If the child crashes repeatedly, the supervisor gives up after the retry budget and prints actionable diagnostics including the actual restart command and health-check curl. + +**Key invariants:** +- Planning sessions that receive non-JSON AI output persist as retryable error state (not process exit) +- `/api/health` remains available during planning session errors +- Remote Tailscale 502 means the local listener on `127.0.0.1:4040` is absent — restart the dashboard +- Check local health: `curl http://127.0.0.1:4040/api/health` + +**Process management guardrails still apply:** The supervisor does NOT use `nohup`, shell kill loops, or unbounded retries. It never kills existing processes on port 4040. + ## Dashboard Test Lanes ```bash diff --git a/packages/cli/src/bin.ts b/packages/cli/src/bin.ts index 341819b615..b7dba4d78f 100644 --- a/packages/cli/src/bin.ts +++ b/packages/cli/src/bin.ts @@ -475,6 +475,7 @@ Options: --paused Start with engine paused (automation disabled) --dev Start dashboard in development mode --no-engine Start dashboard only (no AI engine) + --supervise Run with auto-restart on crash (bounded retries) --lang Terminal-UI locale for this run (en, zh-CN, zh-TW, fr, es, ko); the browser dashboard resolves its own language --attach Attach file(s) on task create (repeatable) --depends Declare dependency on task create (repeatable) @@ -828,6 +829,7 @@ async function main() { const noAuth = args.includes("--no-auth"); const dashTokenIdx = args.indexOf("--token"); const token = dashTokenIdx !== -1 && dashTokenIdx + 1 < args.length ? args[dashTokenIdx + 1] : undefined; + const supervise = args.includes("--supervise"); const dashLangIdx = args.indexOf("--lang"); const lang = dashLangIdx !== -1 && dashLangIdx + 1 < args.length ? args[dashLangIdx + 1] : undefined; if (lang !== undefined) { @@ -839,7 +841,12 @@ async function main() { process.exit(1); } } - await runDashboard(port, { paused, dev, noEngine, interactive, host, noAuth, token, lang }); + if (supervise) { + const { runDashboardSupervised } = await import("./commands/dashboard.js"); + await runDashboardSupervised(port); + } else { + await runDashboard(port, { paused, dev, noEngine, interactive, host, noAuth, token, lang }); + } break; } diff --git a/packages/cli/src/commands/__tests__/dashboard.test.ts b/packages/cli/src/commands/__tests__/dashboard.test.ts index 0fa61a3439..e960a5ec1b 100644 --- a/packages/cli/src/commands/__tests__/dashboard.test.ts +++ b/packages/cli/src/commands/__tests__/dashboard.test.ts @@ -31,6 +31,16 @@ const { mockSyncStartupModels, mockShouldUseHybridExecutor, mockHybridExecutorCt }; }), })); + +const { mockSuperviseSpawn } = vi.hoisted(() => ({ + mockSuperviseSpawn: vi.fn(() => ({ + pid: 12345, + pgid: 12345, + child: {}, + kill: vi.fn(), + waitExit: vi.fn().mockResolvedValue({ code: 0, signal: null }), + })), +})); vi.mock("../startup-model-sync.js", () => ({ syncStartupModels: mockSyncStartupModels, })); @@ -243,6 +253,7 @@ vi.mock("@fusion/core", async (importOriginal) => { getToken: vi.fn().mockResolvedValue(undefined), generateToken: vi.fn().mockResolvedValue("fn_test_dashboard_token"), })), + superviseSpawn: mockSuperviseSpawn, getTaskMergeBlocker: vi.fn((task: any) => { if (task.column !== "in-review") return `task is in '${task.column}', must be in 'in-review'`; if (task.paused) return "task is paused"; @@ -3492,3 +3503,69 @@ describe("runDashboard update check wiring", () => { } }); }); + +describe("runDashboardSupervised — bounded restart behavior", () => { + beforeEach(() => { + mockSuperviseSpawn.mockClear(); + }); + + it("spawns the dashboard without inheriting the supervisor flag or a lifetime cap", async () => { + const mod = await import("../dashboard.js"); + const originalArgv = process.argv; + process.argv = [ + originalArgv[0] ?? process.execPath, + "/tmp/fn-entry.mjs", + "dashboard", + "--host", + "127.0.0.1", + "--port", + "4040", + "--supervise", + ]; + + try { + await mod.runDashboardSupervised(4040); + } finally { + process.argv = originalArgv; + } + + expect(mockSuperviseSpawn).toHaveBeenCalledWith( + process.execPath, + ["/tmp/fn-entry.mjs", "dashboard", "--host", "127.0.0.1", "--port", "4040"], + expect.objectContaining({ + stdio: "inherit", + maxLifetimeMs: Number.POSITIVE_INFINITY, + }), + ); + }); + + it("preserves global flags before the dashboard subcommand without duplicating dashboard", async () => { + const mod = await import("../dashboard.js"); + const originalArgv = process.argv; + process.argv = [ + originalArgv[0] ?? process.execPath, + "/tmp/fn-entry.mjs", + "--project", + "atlas-notes", + "dashboard", + "--port", + "4040", + "--supervise", + ]; + + try { + await mod.runDashboardSupervised(4040); + } finally { + process.argv = originalArgv; + } + + expect(mockSuperviseSpawn).toHaveBeenCalledWith( + process.execPath, + ["/tmp/fn-entry.mjs", "--project", "atlas-notes", "dashboard", "--port", "4040"], + expect.objectContaining({ + stdio: "inherit", + maxLifetimeMs: Number.POSITIVE_INFINITY, + }), + ); + }); +}); diff --git a/packages/cli/src/commands/dashboard.ts b/packages/cli/src/commands/dashboard.ts index 8e81be4700..41f4f3008a 100644 --- a/packages/cli/src/commands/dashboard.ts +++ b/packages/cli/src/commands/dashboard.ts @@ -25,6 +25,8 @@ import { registerBuiltInZaiProvider, type WorkflowIrColumn, type TraitFlags, + superviseSpawn, + type SupervisedChild, } from "@fusion/core"; import { createServer, @@ -3059,3 +3061,115 @@ export async function runDashboard(port: number, opts: { paused?: boolean; dev?: return { dispose }; } + +// ── Supervised Dashboard Mode ──────────────────────────────────────────────── + +const SUPERVISE_MAX_RESTARTS = 3; +const SUPERVISE_BASE_DELAY_MS = 2_000; +const SUPERVISE_MAX_DELAY_MS = 16_000; +const SUPERVISE_STALE_RESET_MS = 60_000; + +/** + * Run the dashboard under foreground process supervision with bounded restart + * attempts and exponential backoff. + * + * FNXC:DashboardAvailability 2026-06-30-23:20: + * Long-lived remote dashboard sessions need bounded crash recovery without + * detaching from the operator terminal or terminating unrelated port listeners. + * + * Spawns the current CLI entry point as a child process (minus the --supervise + * flag) and monitors for unexpected exits. If the child exits non-zero, the + * supervisor restarts up to SUPERVISE_MAX_RESTARTS times with exponential + * backoff. Clean exits (SIGINT/SIGTERM/exit 0) propagate without restart. + * + * This does NOT use shell detachment wrappers, shell kill loops, or unbounded retries. + * Port 4040 processes are never killed — the child binds its own port. + */ +export async function runDashboardSupervised( + port: number, + _opts: Parameters[1] = {}, +): Promise { + // Reconstruct child args: same entry point, same flags, minus --supervise + const childArgs = process.argv.slice(2).filter((a) => a !== "--supervise"); + // Ensure "dashboard" is present without duplicating it after global flags. + if (!childArgs.includes("dashboard")) { + const firstOptionIndex = childArgs.findIndex((arg) => arg.startsWith("-")); + childArgs.splice(firstOptionIndex === -1 ? 0 : firstOptionIndex, 0, "dashboard"); + } + + const entryPoint = process.argv[1]; + if (!entryPoint) { + console.error("[dashboard:supervisor] cannot determine entry point for child process"); + process.exit(1); + } + + let restartCount = 0; + let lastExitTime = 0; + const restartCommand = formatSupervisorRestartCommand(process.execPath, entryPoint, childArgs); + + while (true) { + const attemptLabel = `${restartCount + 1}/${SUPERVISE_MAX_RESTARTS + 1}`; + console.log(`[dashboard:supervisor] starting dashboard (attempt ${attemptLabel})`); + + let child: SupervisedChild; + try { + child = superviseSpawn(process.execPath, [entryPoint, ...childArgs], { + stdio: "inherit", + maxLifetimeMs: Number.POSITIVE_INFINITY, + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.error(`[dashboard:supervisor] failed to spawn child: ${message}`); + process.exit(1); + } + + const exitResult = await child.waitExit(); + const exitCode = exitResult.code ?? 1; + const exitSignal = exitResult.signal; + + // Clean exit — propagate without restart + if (exitSignal === "SIGINT" || exitSignal === "SIGTERM" || exitCode === 0) { + return; + } + + // Reset restart counter if the child ran for a long time + const now = Date.now(); + if (now - lastExitTime > SUPERVISE_STALE_RESET_MS) { + restartCount = 0; + } + lastExitTime = now; + + restartCount++; + if (restartCount > SUPERVISE_MAX_RESTARTS) { + console.error( + `\n[dashboard:supervisor] dashboard exited unexpectedly ${SUPERVISE_MAX_RESTARTS + 1} times.\n` + + `Giving up. If using Tailscale Serve, the remote URL will return 502\n` + + `until the dashboard is restarted manually:\n\n` + + ` ${restartCommand}\n\n` + + `To check if a listener is still active:\n` + + ` curl http://127.0.0.1:${port}/api/health\n`, + ); + process.exit(1); + } + + const delay = Math.min( + SUPERVISE_BASE_DELAY_MS * Math.pow(2, restartCount - 1), + SUPERVISE_MAX_DELAY_MS, + ); + console.log( + `[dashboard:supervisor] restarting in ${Math.round(delay / 1000)}s (attempt ${restartCount}/${SUPERVISE_MAX_RESTARTS})`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } +} + +function formatSupervisorRestartCommand(nodePath: string, entryPoint: string, childArgs: readonly string[]): string { + return [nodePath, entryPoint, ...childArgs].map(quoteShellArg).join(" "); +} + +function quoteShellArg(value: string): string { + if (/^[A-Za-z0-9_/:=.,+-]+$/.test(value)) { + return value; + } + return `'${value.replace(/'/g, "'\\''")}'`; +} diff --git a/packages/dashboard/src/__tests__/server.test.ts b/packages/dashboard/src/__tests__/server.test.ts index ff5d5b75e5..a01cc04e0f 100644 --- a/packages/dashboard/src/__tests__/server.test.ts +++ b/packages/dashboard/src/__tests__/server.test.ts @@ -12,6 +12,7 @@ import express from "express"; import { createServer, setupTerminalWebSocket } from "../server.js"; import { toSessionTag } from "../terminal-websocket-diagnostics.js"; import { RATE_LIMITS } from "../rate-limit.js"; +import { AiSessionStore } from "../ai-session-store.js"; import { Database, TaskStore, getRunningAgentCountSource, setRunningAgentCountSource, type CentralCore } from "@fusion/core"; import { get as performGet, request as performRequest } from "../test-request.js"; @@ -1238,6 +1239,45 @@ describe("createServer health and headless mode", () => { expect(Object.keys(headlessStatusBody).sort()).toEqual(["cloudflaredAvailable", "externalTunnel", "lastError", "lastErrorCode", "provider", "restore", "state", "url"]); expect(headlessRoot.status).toBe(404); }); + + it("returns health OK independently of persisted AI session error rows", async () => { + const rootDir = makeTmpDir(); + const db = new Database(join(rootDir, ".fusion")); + db.init(); + const aiSessionStore = new AiSessionStore(db); + + try { + aiSessionStore.upsert({ + id: "planning-error", + type: "planning", + status: "error", + title: "Planning parse failure", + inputPayload: JSON.stringify({ initialPlan: "broken planning response" }), + conversationHistory: JSON.stringify([]), + currentQuestion: null, + result: null, + thinkingOutput: "", + error: "AI response could not be parsed", + projectId: null, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + lockedByTab: null, + lockedAt: null, + }); + + const store = createMockStore(); + const app = createServer(store, { aiSessionStore }); + + const res = await GET(app, "/api/health"); + expect(res.status).toBe(200); + expect((res.body as any).status).toBe("ok"); + expect(aiSessionStore.get("planning-error")?.status).toBe("error"); + } finally { + aiSessionStore.stopScheduledCleanup(); + db.close(); + await rm(rootDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 }); + } + }); }); describe("API Error Handling Middleware", () => { diff --git a/packages/dashboard/src/__tests__/session-error-recovery.test.ts b/packages/dashboard/src/__tests__/session-error-recovery.test.ts index 6251191cf3..c37bf77b4a 100644 --- a/packages/dashboard/src/__tests__/session-error-recovery.test.ts +++ b/packages/dashboard/src/__tests__/session-error-recovery.test.ts @@ -58,6 +58,7 @@ vi.mock("@fusion/engine", () => ({ createChatTaskDocumentTools: vi.fn(() => []), createChatArtifactTools: vi.fn(() => []), // FNXC:DashboardSessionTests 2026-06-17-19:33: planning and mission-interview sessions now request skills through the shared helper; focused engine mocks must return the shaped helper result so lifecycle tests do not crash before createFnAgent is captured. + resolveMcpServersForStore: vi.fn(() => ({ servers: [] })), buildSessionSkillContextSync: vi.fn(() => ({ skillSelectionContext: undefined, resolvedSkillNames: [], @@ -106,6 +107,10 @@ async function waitFor(check: () => boolean, timeoutMs = 2000): Promise { } } +async function flushUnhandledRejectionTurn(): Promise { + await new Promise((resolve) => setImmediate(resolve)); +} + describe("session error recovery", () => { let tmpDir: string; let db: Database; @@ -700,4 +705,62 @@ The actual planning response is: unsubscribe(); }); + + it("does not leak unhandled rejections when non-streaming parse fails", async () => { + const unhandledRejections: unknown[] = []; + const onUnhandled = (reason: unknown) => { unhandledRejections.push(reason); }; + process.on("unhandledRejection", onUnhandled); + + try { + __setCreateFnAgent( + async () => + createMockAgent([ + "Prose only response for unhandled rejection check.", + "Still prose after reformat attempt.", + ]), + ); + + await expect( + createSession("127.0.0.130", "Unhandled rejection test", taskStore, "/tmp/project"), + ).rejects.toThrow("Failed to get first question from AI"); + + await flushUnhandledRejectionTurn(); + + expect(unhandledRejections).toHaveLength(0); + } finally { + process.off("unhandledRejection", onUnhandled); + } + }); + + it("does not leak unhandled rejections when streaming parse fails", async () => { + const unhandledRejections: unknown[] = []; + const onUnhandled = (reason: unknown) => { unhandledRejections.push(reason); }; + process.on("unhandledRejection", onUnhandled); + + try { + __setCreateFnAgent( + async () => + createMockAgent([ + "Streaming prose only — no valid JSON.", + "Still prose after reformat.", + ]), + ); + + const sessionId = await createSessionWithAgent( + "127.0.0.131", + "Streaming unhandled rejection test", + "/tmp/project", + taskStore, + ); + + planningStreamManager.consumeInitialTurn(sessionId)?.(); + + await waitFor(() => aiSessionStore.get(sessionId)?.status === "error"); + await flushUnhandledRejectionTurn(); + + expect(unhandledRejections).toHaveLength(0); + } finally { + process.off("unhandledRejection", onUnhandled); + } + }); }); diff --git a/packages/dashboard/src/planning.ts b/packages/dashboard/src/planning.ts index 67f03b58ae..41a41de05f 100644 --- a/packages/dashboard/src/planning.ts +++ b/packages/dashboard/src/planning.ts @@ -1094,6 +1094,7 @@ async function getFirstQuestionFromAgent( if (!parsed) { const errorMessage = buildRetryableParseErrorMessage(lastError); setSessionError(session, errorMessage); + // Keep the session and persisted error state so retry can reuse the original project context. try { await session.agent.session.dispose?.(); } catch (disposeErr) {