root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(engine): auto-restart tunnels that exit unexpectedly (#3393)

Browse Source 
## Symptom

A tunnel process that dies unexpectedly (crash, OOM, network hiccup, the
24h `maxLifetimeMs` supervision cutoff) leaves `TunnelProcessManager` in
a terminal `failed` state. `handleUnexpectedExit` records the error and
stops — the operator's remote-access tunnel silently goes dark until a
human notices and restarts it.

## Fix

The manager now remembers the desired tunnel (`provider` + `config`)
across the start/switch lifecycle and respawns it after an unexpected
exit with exponential backoff — 1s base doubling to a 30s cap, both
configurable via new `restartBaseDelayMs` / `restartMaxDelayMs` options,
and `autoRestart: false` to opt out. The attempt counter resets once the
tunnel reports ready.

Intentional shutdowns stay shut down: `stop()` and `switchProvider()`
clear the desired tunnel and cancel any pending restart timer.

Hardening that falls out of restart support:
- Exit/spawn-error/readiness-timeout handlers are guarded by
process-handle identity, so a stale `close` event from a superseded
child cannot clobber the state of the restarted tunnel.
- The readiness-timeout path now SIGTERMs the stalled child instead of
leaking it while marking the tunnel failed.
- A synchronous `superviseSpawn` throw now records a redacted
`start_failed` status instead of escaping unhandled.

## Verification

- New tests (fake timers, no real waits): restart after unexpected exit
for both cloudflare and tailscale providers; backoff progression and
delay cap across repeated pre-readiness failures; explicit `stop()`
cancels a pending restart.
- `pnpm --filter @fusion/engine exec vitest run
src/__tests__/tunnel-process-manager.test.ts` — 17/17 pass.
- `pnpm verify:fast` — 14 steps green (scoped typecheck/build + CLI
build + boot smoke).
- Changeset included (`patch`, category `fix`).

## Provenance

This is the tunnel half of a fork-side fix (FN-915) that has been
running in production since July; the merge-blocker-preservation half of
that same fix already landed upstream (present since v0.76.0-beta.0).
Ported onto current `main` — the original patch applied cleanly.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
  * Tunnels now automatically restart after unexpected crashes.
  * Restart attempts use increasing delays up to a configurable maximum.
  * Automatic recovery is enabled by default and can be configured.

* **Bug Fixes**
  * Prevented stale process events from triggering unwanted restarts.
* Explicit stops and provider changes now cancel pending restart
attempts.
  * Failed or unready tunnel processes are terminated cleanly.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Co-authored-by: v <v@v.speedport.ip>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
flexi767
2026-08-10 03:23:59 +02:00
committed by GitHub
parent b28b6d1053
commit 14c8793df5
3 changed files with 200 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.changeset/tunnel-auto-restart.md Normal file
View File

@@ -0,0 +1,7 @@
---
"@runfusion/fusion": patch
---
summary: Tunnels now restart automatically with backoff after crashing, instead of staying failed until manually restarted.
category: fix
dev: `TunnelProcessManager` gains `autoRestart` (default on), `restartBaseDelayMs`, `restartMaxDelayMs`; explicit stop or provider switch cancels any pending restart; exit/readiness handlers are guarded by process-handle identity so stale events cannot clobber a restarted tunnel.

78
packages/engine/src/__tests__/tunnel-process-manager.test.ts
View File

@@ -219,6 +219,84 @@ describe("TunnelProcessManager", () => {
expect(states).toContain("stopped");
});
it.each([
["cloudflare", cloudflareConfig()],
["tailscale", {
provider: "tailscale",
executablePath: "tailscale",
args: ["funnel", "4040"],
} satisfies TunnelProviderConfig],
] as const)("recreates an unexpectedly exited %s tunnel while the manager is active", async (provider, config) => {
vi.useFakeTimers();
const spawned: FakeChildProcess[] = [];
const manager = new TunnelProcessManager({
restartBaseDelayMs: 100,
restartMaxDelayMs: 1_000,
spawnImpl: () => {
const child = new FakeChildProcess(++pid);
children.set(child.pid, child);
spawned.push(child);
return child as never;
},
});
await manager.start(provider, config);
spawned[0].close(1);
expect(manager.getStatus()).toMatchObject({ state: "failed", provider });
await vi.advanceTimersByTimeAsync(99);
expect(spawned).toHaveLength(1);
await vi.advanceTimersByTimeAsync(1);
expect(spawned).toHaveLength(2);
expect(manager.getStatus()).toMatchObject({ state: "starting", provider });
});
it("backs off repeated pre-readiness failures and caps the retry delay", async () => {
vi.useFakeTimers();
const spawned: FakeChildProcess[] = [];
const manager = new TunnelProcessManager({
restartBaseDelayMs: 100,
restartMaxDelayMs: 200,
spawnImpl: () => {
const child = new FakeChildProcess(++pid);
children.set(child.pid, child);
spawned.push(child);
return child as never;
},
});
await manager.start("cloudflare", cloudflareConfig());
spawned[0].close(1);
await vi.advanceTimersByTimeAsync(100);
spawned[1].close(1);
await vi.advanceTimersByTimeAsync(199);
expect(spawned).toHaveLength(2);
await vi.advanceTimersByTimeAsync(1);
expect(spawned).toHaveLength(3);
spawned[2].close(1);
await vi.advanceTimersByTimeAsync(200);
expect(spawned).toHaveLength(4);
});
it("cancels a pending automatic restart when the tunnel is explicitly stopped", async () => {
vi.useFakeTimers();
const spawnImpl = vi.fn(() => {
const child = new FakeChildProcess(++pid);
children.set(child.pid, child);
return child as never;
});
const manager = new TunnelProcessManager({ restartBaseDelayMs: 100, spawnImpl });
await manager.start("cloudflare", cloudflareConfig());
[...children.values()][0].close(1);
await manager.stop();
await vi.advanceTimersByTimeAsync(1_000);
expect(spawnImpl).toHaveBeenCalledTimes(1);
expect(manager.getStatus()).toMatchObject({ state: "stopped", provider: null });
});
it("falls back to SIGKILL when graceful stop times out", async () => {
vi.useFakeTimers();

130
packages/engine/src/remote-access/tunnel-process-manager.ts
View File

@@ -26,11 +26,16 @@ import type {
export interface TunnelProcessManagerOptions {
maxLogEntries?: number;
stopTimeoutMs?: number;
restartBaseDelayMs?: number;
restartMaxDelayMs?: number;
autoRestart?: boolean;
spawnImpl?: typeof spawn;
}
const DEFAULT_MAX_LOG_ENTRIES = 400;
const DEFAULT_STOP_TIMEOUT_MS = 5_000;
const DEFAULT_RESTART_BASE_DELAY_MS = 1_000;
const DEFAULT_RESTART_MAX_DELAY_MS = 30_000;
const execFileAsync = promisify(execFile);
const execAsync = promisify(exec);
@@ -102,6 +107,9 @@ function toStateError(code: TunnelErrorCode, err: unknown): { code: TunnelErrorC
export class TunnelProcessManager extends EventEmitter implements TunnelManager {
private readonly maxLogEntries: number;
private readonly defaultStopTimeoutMs: number;
private readonly restartBaseDelayMs: number;
private readonly restartMaxDelayMs: number;
private readonly autoRestart: boolean;
private readonly spawnImpl: typeof spawn;
private status: TunnelStatusSnapshot = {
@@ -120,14 +128,20 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager
private processHandle: ManagedTunnelProcess | null = null;
private readinessTimer: NodeJS.Timeout | null = null;
private stopTimer: NodeJS.Timeout | null = null;
private restartTimer: NodeJS.Timeout | null = null;
private operationChain: Promise<void> = Promise.resolve();
private expectedStop = false;
private activeStopPromise: Promise<void> | null = null;
private desiredTunnel: { provider: TunnelProvider; config: TunnelProviderConfig } | null = null;
private restartAttempt = 0;
constructor(options: TunnelProcessManagerOptions = {}) {
super();
this.maxLogEntries = options.maxLogEntries ?? DEFAULT_MAX_LOG_ENTRIES;
this.defaultStopTimeoutMs = options.stopTimeoutMs ?? DEFAULT_STOP_TIMEOUT_MS;
this.restartBaseDelayMs = options.restartBaseDelayMs ?? DEFAULT_RESTART_BASE_DELAY_MS;
this.restartMaxDelayMs = options.restartMaxDelayMs ?? DEFAULT_RESTART_MAX_DELAY_MS;
this.autoRestart = options.autoRestart ?? true;
this.spawnImpl = options.spawnImpl ?? spawn;
}
@@ -155,12 +169,22 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager
if (this.processHandle || this.status.state === "starting" || this.status.state === "running") {
throw new Error("already_running:tunnel process is already active");
}
await this.startInternal(provider, config);
this.clearRestartTimer();
this.desiredTunnel = { provider, config };
try {
await this.startInternal(provider, config);
} catch (error) {
this.desiredTunnel = null;
throw error;
}
});
}
async stop(): Promise<void> {
return this.runExclusive(async () => {
this.desiredTunnel = null;
this.restartAttempt = 0;
this.clearRestartTimer();
await this.stopInternal();
});
}
@@ -225,6 +249,8 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager
async switchProvider(target: TunnelProvider, config: TunnelProviderConfig): Promise<void> {
return this.runExclusive(async () => {
this.clearRestartTimer();
this.desiredTunnel = { provider: target, config };
const previousProvider = this.status.provider;
if (this.processHandle) {
await this.stopInternal();
@@ -233,6 +259,7 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager
try {
await this.startInternal(target, config);
} catch (error) {
this.desiredTunnel = null;
const stateError = toStateError("switch_failed", error);
const redactedMessage = this.redactForProviderConfig(target, config, stateError.message);
this.updateStatus({
@@ -307,21 +334,38 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager
this.emitLog("info", "manager", `Starting ${provider} tunnel: ${command.redactedPreview}`);
const supervised = superviseSpawn(command.command, command.args, {
cwd: command.cwd,
env: command.env,
shell: false,
stdio: ["ignore", "pipe", "pipe"],
maxLifetimeMs: 24 * 60 * 60 * 1_000,
spawnImpl: this.spawnImpl,
});
let supervised: ReturnType<typeof superviseSpawn>;
try {
supervised = superviseSpawn(command.command, command.args, {
cwd: command.cwd,
env: command.env,
shell: false,
stdio: ["ignore", "pipe", "pipe"],
maxLifetimeMs: 24 * 60 * 60 * 1_000,
spawnImpl: this.spawnImpl,
});
} catch (error) {
const stateError = toStateError("start_failed", error);
const redactedMessage = redactTunnelText(stateError.message, command.sensitiveValues);
this.updateStatus({
provider,
state: "failed",
pid: null,
stoppedAt: nowIso(),
url: null,
lastError: { ...stateError, message: redactedMessage },
});
this.emitLog("error", "manager", `Spawn failure for ${provider}: ${redactedMessage}`);
throw error;
}
const child = supervised.child;
this.processHandle = {
const managedHandle: ManagedTunnelProcess = {
provider,
child,
command,
};
this.processHandle = managedHandle;
this.expectedStop = false;
this.updateStatus({ pid: child.pid ?? null });
@@ -342,9 +386,9 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager
attachStream(child.stderr, "stderr", stderrBuffer);
child.once("error", (error) => {
const maskedMessage = maskSensitive(normalizeError(error).message, this.processHandle);
const maskedMessage = maskSensitive(normalizeError(error).message, managedHandle);
this.emitLog("error", "manager", `Spawn failure for ${provider}: ${maskedMessage}`);
this.handleUnexpectedExit("start_failed", `Spawn failure: ${maskedMessage}`);
this.handleUnexpectedExit(managedHandle, "start_failed", `Spawn failure: ${maskedMessage}`);
});
child.once("close", (code, signal) => {
@@ -355,6 +399,10 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager
this.handleOutputLine("stderr", line);
}
if (this.processHandle !== managedHandle) {
return;
}
const reason = signal ? `signal ${signal}` : `exit code ${code ?? 0}`;
if (this.expectedStop) {
this.emitLog("info", "manager", `Tunnel process stopped (${reason})`);
@@ -363,13 +411,14 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager
}
this.emitLog("error", "manager", `Tunnel process exited unexpectedly (${reason})`);
this.handleUnexpectedExit("process_exit", `Process exited unexpectedly (${reason})`);
this.handleUnexpectedExit(managedHandle, "process_exit", `Process exited unexpectedly (${reason})`);
});
this.readinessTimer = setTimeout(() => {
if (this.status.state === "starting" && this.processHandle?.provider === provider) {
this.emitLog("error", "manager", `Readiness timed out after ${command.readinessTimeoutMs}ms`);
this.handleUnexpectedExit("readiness_timeout", `Tunnel readiness timeout after ${command.readinessTimeoutMs}ms`);
this.handleUnexpectedExit(managedHandle, "readiness_timeout", `Tunnel readiness timeout after ${command.readinessTimeoutMs}ms`);
killManagedProcess(managedHandle.child, "SIGTERM");
}
}, command.readinessTimeoutMs);
this.readinessTimer.unref?.();
@@ -456,10 +505,14 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager
startedAt: this.status.startedAt ?? nowIso(),
lastError: null,
});
this.restartAttempt = 0;
this.emitLog("info", "manager", `${processHandle.provider} tunnel is running`);
}
private handleUnexpectedExit(code: TunnelErrorCode, message: string): void {
private handleUnexpectedExit(handle: ManagedTunnelProcess, code: TunnelErrorCode, message: string): void {
if (this.processHandle !== handle) {
return;
}
this.clearReadinessTimer();
if (this.stopTimer) {
clearTimeout(this.stopTimer);
@@ -482,6 +535,46 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager
at: nowIso(),
},
});
this.scheduleRestart();
}
private scheduleRestart(): void {
if (!this.autoRestart || !this.desiredTunnel || this.restartTimer) {
return;
}
this.restartAttempt += 1;
const delayMs = Math.min(
this.restartBaseDelayMs * (2 ** Math.max(0, this.restartAttempt - 1)),
this.restartMaxDelayMs,
);
const provider = this.desiredTunnel.provider;
this.emitLog("warn", "manager", `Scheduling ${provider} tunnel restart in ${delayMs}ms (attempt ${this.restartAttempt})`);
this.restartTimer = setTimeout(() => {
this.restartTimer = null;
void this.runExclusive(async () => {
const desired = this.desiredTunnel;
if (!desired || this.processHandle) {
return;
}
try {
await this.startInternal(desired.provider, desired.config);
} catch (error) {
const message = this.redactForProviderConfig(
desired.provider,
desired.config,
normalizeError(error).message,
);
this.emitLog("error", "manager", `Automatic ${desired.provider} tunnel restart failed: ${message}`);
this.scheduleRestart();
}
}).catch((error: unknown) => {
this.emitLog("error", "manager", `Automatic tunnel restart scheduling failed: ${normalizeError(error).message}`);
this.scheduleRestart();
});
}, delayMs);
this.restartTimer.unref?.();
}
private finalizeStoppedState(): void {
@@ -511,6 +604,13 @@ export class TunnelProcessManager extends EventEmitter implements TunnelManager
}
}
private clearRestartTimer(): void {
if (this.restartTimer) {
clearTimeout(this.restartTimer);
this.restartTimer = null;
}
}
private updateStatus(patch: Partial<TunnelStatusSnapshot>): void {
this.status = {
...this.status,