From 14c8793df5bc5ac48c20190e10e5c9b25bbde378 Mon Sep 17 00:00:00 2001 From: flexi767 <96955327+flexi767@users.noreply.github.com> Date: Mon, 10 Aug 2026 03:23:59 +0200 Subject: [PATCH] fix(engine): auto-restart tunnels that exit unexpectedly (#3393) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Symptom A tunnel process that dies unexpectedly (crash, OOM, network hiccup, the 24h `maxLifetimeMs` supervision cutoff) leaves `TunnelProcessManager` in a terminal `failed` state. `handleUnexpectedExit` records the error and stops — the operator's remote-access tunnel silently goes dark until a human notices and restarts it. ## Fix The manager now remembers the desired tunnel (`provider` + `config`) across the start/switch lifecycle and respawns it after an unexpected exit with exponential backoff — 1s base doubling to a 30s cap, both configurable via new `restartBaseDelayMs` / `restartMaxDelayMs` options, and `autoRestart: false` to opt out. The attempt counter resets once the tunnel reports ready. Intentional shutdowns stay shut down: `stop()` and `switchProvider()` clear the desired tunnel and cancel any pending restart timer. Hardening that falls out of restart support: - Exit/spawn-error/readiness-timeout handlers are guarded by process-handle identity, so a stale `close` event from a superseded child cannot clobber the state of the restarted tunnel. - The readiness-timeout path now SIGTERMs the stalled child instead of leaking it while marking the tunnel failed. - A synchronous `superviseSpawn` throw now records a redacted `start_failed` status instead of escaping unhandled. ## Verification - New tests (fake timers, no real waits): restart after unexpected exit for both cloudflare and tailscale providers; backoff progression and delay cap across repeated pre-readiness failures; explicit `stop()` cancels a pending restart. - `pnpm --filter @fusion/engine exec vitest run src/__tests__/tunnel-process-manager.test.ts` — 17/17 pass. - `pnpm verify:fast` — 14 steps green (scoped typecheck/build + CLI build + boot smoke). - Changeset included (`patch`, category `fix`). ## Provenance This is the tunnel half of a fork-side fix (FN-915) that has been running in production since July; the merge-blocker-preservation half of that same fix already landed upstream (present since v0.76.0-beta.0). Ported onto current `main` — the original patch applied cleanly. 🤖 Generated with [Claude Code](https://claude.com/claude-code) ## Summary by CodeRabbit * **New Features** * Tunnels now automatically restart after unexpected crashes. * Restart attempts use increasing delays up to a configurable maximum. * Automatic recovery is enabled by default and can be configured. * **Bug Fixes** * Prevented stale process events from triggering unwanted restarts. * Explicit stops and provider changes now cancel pending restart attempts. * Failed or unready tunnel processes are terminated cleanly. Co-authored-by: v Co-authored-by: Claude Fable 5 --- .changeset/tunnel-auto-restart.md | 7 + .../__tests__/tunnel-process-manager.test.ts | 78 +++++++++++ .../remote-access/tunnel-process-manager.ts | 130 ++++++++++++++++-- 3 files changed, 200 insertions(+), 15 deletions(-) create mode 100644 .changeset/tunnel-auto-restart.md diff --git a/.changeset/tunnel-auto-restart.md b/.changeset/tunnel-auto-restart.md new file mode 100644 index 0000000000..3e624da370 --- /dev/null +++ b/.changeset/tunnel-auto-restart.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Tunnels now restart automatically with backoff after crashing, instead of staying failed until manually restarted. +category: fix +dev: `TunnelProcessManager` gains `autoRestart` (default on), `restartBaseDelayMs`, `restartMaxDelayMs`; explicit stop or provider switch cancels any pending restart; exit/readiness handlers are guarded by process-handle identity so stale events cannot clobber a restarted tunnel. diff --git a/packages/engine/src/__tests__/tunnel-process-manager.test.ts b/packages/engine/src/__tests__/tunnel-process-manager.test.ts index 3f1173208a..dac389281f 100644 --- a/packages/engine/src/__tests__/tunnel-process-manager.test.ts +++ b/packages/engine/src/__tests__/tunnel-process-manager.test.ts @@ -219,6 +219,84 @@ describe("TunnelProcessManager", () => { expect(states).toContain("stopped"); }); + it.each([ + ["cloudflare", cloudflareConfig()], + ["tailscale", { + provider: "tailscale", + executablePath: "tailscale", + args: ["funnel", "4040"], + } satisfies TunnelProviderConfig], + ] as const)("recreates an unexpectedly exited %s tunnel while the manager is active", async (provider, config) => { + vi.useFakeTimers(); + const spawned: FakeChildProcess[] = []; + const manager = new TunnelProcessManager({ + restartBaseDelayMs: 100, + restartMaxDelayMs: 1_000, + spawnImpl: () => { + const child = new FakeChildProcess(++pid); + children.set(child.pid, child); + spawned.push(child); + return child as never; + }, + }); + + await manager.start(provider, config); + spawned[0].close(1); + + expect(manager.getStatus()).toMatchObject({ state: "failed", provider }); + await vi.advanceTimersByTimeAsync(99); + expect(spawned).toHaveLength(1); + await vi.advanceTimersByTimeAsync(1); + expect(spawned).toHaveLength(2); + expect(manager.getStatus()).toMatchObject({ state: "starting", provider }); + }); + + it("backs off repeated pre-readiness failures and caps the retry delay", async () => { + vi.useFakeTimers(); + const spawned: FakeChildProcess[] = []; + const manager = new TunnelProcessManager({ + restartBaseDelayMs: 100, + restartMaxDelayMs: 200, + spawnImpl: () => { + const child = new FakeChildProcess(++pid); + children.set(child.pid, child); + spawned.push(child); + return child as never; + }, + }); + + await manager.start("cloudflare", cloudflareConfig()); + spawned[0].close(1); + await vi.advanceTimersByTimeAsync(100); + spawned[1].close(1); + await vi.advanceTimersByTimeAsync(199); + expect(spawned).toHaveLength(2); + await vi.advanceTimersByTimeAsync(1); + expect(spawned).toHaveLength(3); + + spawned[2].close(1); + await vi.advanceTimersByTimeAsync(200); + expect(spawned).toHaveLength(4); + }); + + it("cancels a pending automatic restart when the tunnel is explicitly stopped", async () => { + vi.useFakeTimers(); + const spawnImpl = vi.fn(() => { + const child = new FakeChildProcess(++pid); + children.set(child.pid, child); + return child as never; + }); + const manager = new TunnelProcessManager({ restartBaseDelayMs: 100, spawnImpl }); + + await manager.start("cloudflare", cloudflareConfig()); + [...children.values()][0].close(1); + await manager.stop(); + await vi.advanceTimersByTimeAsync(1_000); + + expect(spawnImpl).toHaveBeenCalledTimes(1); + expect(manager.getStatus()).toMatchObject({ state: "stopped", provider: null }); + }); + it("falls back to SIGKILL when graceful stop times out", async () => { vi.useFakeTimers(); diff --git a/packages/engine/src/remote-access/tunnel-process-manager.ts b/packages/engine/src/remote-access/tunnel-process-manager.ts index c3789685c9..d5e775a1f4 100644 --- a/packages/engine/src/remote-access/tunnel-process-manager.ts +++ b/packages/engine/src/remote-access/tunnel-process-manager.ts @@ -26,11 +26,16 @@ import type { export interface TunnelProcessManagerOptions { maxLogEntries?: number; stopTimeoutMs?: number; + restartBaseDelayMs?: number; + restartMaxDelayMs?: number; + autoRestart?: boolean; spawnImpl?: typeof spawn; } const DEFAULT_MAX_LOG_ENTRIES = 400; const DEFAULT_STOP_TIMEOUT_MS = 5_000; +const DEFAULT_RESTART_BASE_DELAY_MS = 1_000; +const DEFAULT_RESTART_MAX_DELAY_MS = 30_000; const execFileAsync = promisify(execFile); const execAsync = promisify(exec); @@ -102,6 +107,9 @@ function toStateError(code: TunnelErrorCode, err: unknown): { code: TunnelErrorC export class TunnelProcessManager extends EventEmitter implements TunnelManager { private readonly maxLogEntries: number; private readonly defaultStopTimeoutMs: number; + private readonly restartBaseDelayMs: number; + private readonly restartMaxDelayMs: number; + private readonly autoRestart: boolean; private readonly spawnImpl: typeof spawn; private status: TunnelStatusSnapshot = { @@ -120,14 +128,20 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager private processHandle: ManagedTunnelProcess | null = null; private readinessTimer: NodeJS.Timeout | null = null; private stopTimer: NodeJS.Timeout | null = null; + private restartTimer: NodeJS.Timeout | null = null; private operationChain: Promise = Promise.resolve(); private expectedStop = false; private activeStopPromise: Promise | null = null; + private desiredTunnel: { provider: TunnelProvider; config: TunnelProviderConfig } | null = null; + private restartAttempt = 0; constructor(options: TunnelProcessManagerOptions = {}) { super(); this.maxLogEntries = options.maxLogEntries ?? DEFAULT_MAX_LOG_ENTRIES; this.defaultStopTimeoutMs = options.stopTimeoutMs ?? DEFAULT_STOP_TIMEOUT_MS; + this.restartBaseDelayMs = options.restartBaseDelayMs ?? DEFAULT_RESTART_BASE_DELAY_MS; + this.restartMaxDelayMs = options.restartMaxDelayMs ?? DEFAULT_RESTART_MAX_DELAY_MS; + this.autoRestart = options.autoRestart ?? true; this.spawnImpl = options.spawnImpl ?? spawn; } @@ -155,12 +169,22 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager if (this.processHandle || this.status.state === "starting" || this.status.state === "running") { throw new Error("already_running:tunnel process is already active"); } - await this.startInternal(provider, config); + this.clearRestartTimer(); + this.desiredTunnel = { provider, config }; + try { + await this.startInternal(provider, config); + } catch (error) { + this.desiredTunnel = null; + throw error; + } }); } async stop(): Promise { return this.runExclusive(async () => { + this.desiredTunnel = null; + this.restartAttempt = 0; + this.clearRestartTimer(); await this.stopInternal(); }); } @@ -225,6 +249,8 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager async switchProvider(target: TunnelProvider, config: TunnelProviderConfig): Promise { return this.runExclusive(async () => { + this.clearRestartTimer(); + this.desiredTunnel = { provider: target, config }; const previousProvider = this.status.provider; if (this.processHandle) { await this.stopInternal(); @@ -233,6 +259,7 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager try { await this.startInternal(target, config); } catch (error) { + this.desiredTunnel = null; const stateError = toStateError("switch_failed", error); const redactedMessage = this.redactForProviderConfig(target, config, stateError.message); this.updateStatus({ @@ -307,21 +334,38 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager this.emitLog("info", "manager", `Starting ${provider} tunnel: ${command.redactedPreview}`); - const supervised = superviseSpawn(command.command, command.args, { - cwd: command.cwd, - env: command.env, - shell: false, - stdio: ["ignore", "pipe", "pipe"], - maxLifetimeMs: 24 * 60 * 60 * 1_000, - spawnImpl: this.spawnImpl, - }); + let supervised: ReturnType; + try { + supervised = superviseSpawn(command.command, command.args, { + cwd: command.cwd, + env: command.env, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + maxLifetimeMs: 24 * 60 * 60 * 1_000, + spawnImpl: this.spawnImpl, + }); + } catch (error) { + const stateError = toStateError("start_failed", error); + const redactedMessage = redactTunnelText(stateError.message, command.sensitiveValues); + this.updateStatus({ + provider, + state: "failed", + pid: null, + stoppedAt: nowIso(), + url: null, + lastError: { ...stateError, message: redactedMessage }, + }); + this.emitLog("error", "manager", `Spawn failure for ${provider}: ${redactedMessage}`); + throw error; + } const child = supervised.child; - this.processHandle = { + const managedHandle: ManagedTunnelProcess = { provider, child, command, }; + this.processHandle = managedHandle; this.expectedStop = false; this.updateStatus({ pid: child.pid ?? null }); @@ -342,9 +386,9 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager attachStream(child.stderr, "stderr", stderrBuffer); child.once("error", (error) => { - const maskedMessage = maskSensitive(normalizeError(error).message, this.processHandle); + const maskedMessage = maskSensitive(normalizeError(error).message, managedHandle); this.emitLog("error", "manager", `Spawn failure for ${provider}: ${maskedMessage}`); - this.handleUnexpectedExit("start_failed", `Spawn failure: ${maskedMessage}`); + this.handleUnexpectedExit(managedHandle, "start_failed", `Spawn failure: ${maskedMessage}`); }); child.once("close", (code, signal) => { @@ -355,6 +399,10 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager this.handleOutputLine("stderr", line); } + if (this.processHandle !== managedHandle) { + return; + } + const reason = signal ? `signal ${signal}` : `exit code ${code ?? 0}`; if (this.expectedStop) { this.emitLog("info", "manager", `Tunnel process stopped (${reason})`); @@ -363,13 +411,14 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager } this.emitLog("error", "manager", `Tunnel process exited unexpectedly (${reason})`); - this.handleUnexpectedExit("process_exit", `Process exited unexpectedly (${reason})`); + this.handleUnexpectedExit(managedHandle, "process_exit", `Process exited unexpectedly (${reason})`); }); this.readinessTimer = setTimeout(() => { if (this.status.state === "starting" && this.processHandle?.provider === provider) { this.emitLog("error", "manager", `Readiness timed out after ${command.readinessTimeoutMs}ms`); - this.handleUnexpectedExit("readiness_timeout", `Tunnel readiness timeout after ${command.readinessTimeoutMs}ms`); + this.handleUnexpectedExit(managedHandle, "readiness_timeout", `Tunnel readiness timeout after ${command.readinessTimeoutMs}ms`); + killManagedProcess(managedHandle.child, "SIGTERM"); } }, command.readinessTimeoutMs); this.readinessTimer.unref?.(); @@ -456,10 +505,14 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager startedAt: this.status.startedAt ?? nowIso(), lastError: null, }); + this.restartAttempt = 0; this.emitLog("info", "manager", `${processHandle.provider} tunnel is running`); } - private handleUnexpectedExit(code: TunnelErrorCode, message: string): void { + private handleUnexpectedExit(handle: ManagedTunnelProcess, code: TunnelErrorCode, message: string): void { + if (this.processHandle !== handle) { + return; + } this.clearReadinessTimer(); if (this.stopTimer) { clearTimeout(this.stopTimer); @@ -482,6 +535,46 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager at: nowIso(), }, }); + this.scheduleRestart(); + } + + private scheduleRestart(): void { + if (!this.autoRestart || !this.desiredTunnel || this.restartTimer) { + return; + } + + this.restartAttempt += 1; + const delayMs = Math.min( + this.restartBaseDelayMs * (2 ** Math.max(0, this.restartAttempt - 1)), + this.restartMaxDelayMs, + ); + const provider = this.desiredTunnel.provider; + this.emitLog("warn", "manager", `Scheduling ${provider} tunnel restart in ${delayMs}ms (attempt ${this.restartAttempt})`); + + this.restartTimer = setTimeout(() => { + this.restartTimer = null; + void this.runExclusive(async () => { + const desired = this.desiredTunnel; + if (!desired || this.processHandle) { + return; + } + try { + await this.startInternal(desired.provider, desired.config); + } catch (error) { + const message = this.redactForProviderConfig( + desired.provider, + desired.config, + normalizeError(error).message, + ); + this.emitLog("error", "manager", `Automatic ${desired.provider} tunnel restart failed: ${message}`); + this.scheduleRestart(); + } + }).catch((error: unknown) => { + this.emitLog("error", "manager", `Automatic tunnel restart scheduling failed: ${normalizeError(error).message}`); + this.scheduleRestart(); + }); + }, delayMs); + this.restartTimer.unref?.(); } private finalizeStoppedState(): void { @@ -511,6 +604,13 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager } } + private clearRestartTimer(): void { + if (this.restartTimer) { + clearTimeout(this.restartTimer); + this.restartTimer = null; + } + } + private updateStatus(patch: Partial): void { this.status = { ...this.status,