From e16204deef5ba80e5ba0192a0ed325e2e78e693f Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Wed, 22 Jul 2026 22:43:51 -0700 Subject: [PATCH] FN-8522: fix Windows embedded PostgreSQL crash recovery Harden Windows embedded PostgreSQL startup and recover once from owned-cluster DLL initialization crashes. - Provide child-only native PATH hardening and non-blocking runner-log monitoring. - Detect the ordered 0xC0000142 shutdown sequence and restart only owned clusters on their resolved port. - Add lifecycle coverage and operator diagnostics for recovery behavior. Files changed: .../fn-8522-windows-embedded-postgres-recovery.md | 7 ++ docs/diagnostics.md | 10 ++ docs/storage.md | 6 + .../__tests__/postgres/embedded-lifecycle.test.ts | 87 ++++++++++++++ .../postgres/embedded-windows-elevated.test.ts | 49 ++++++++ packages/core/src/postgres/embedded-lifecycle.ts | 128 ++++++++++++++++++--- .../core/src/postgres/embedded-windows-elevated.ts | 119 +++++++++++++++++-- 7 files changed, 382 insertions(+), 24 deletions(-) Fusion-Task-Id: FN-8522 Fusion-Task-Lineage: f256e421-d82e-4afb-9f81-b5c18f1c7100 Co-authored-by: Fusion (runfusion.ai) --- ...8522-windows-embedded-postgres-recovery.md | 7 + docs/diagnostics.md | 10 ++ docs/storage.md | 6 + .../postgres/embedded-lifecycle.test.ts | 87 ++++++++++++ .../embedded-windows-elevated.test.ts | 49 +++++++ .../core/src/postgres/embedded-lifecycle.ts | 128 ++++++++++++++++-- .../src/postgres/embedded-windows-elevated.ts | 119 ++++++++++++++-- 7 files changed, 382 insertions(+), 24 deletions(-) create mode 100644 .changeset/fn-8522-windows-embedded-postgres-recovery.md diff --git a/.changeset/fn-8522-windows-embedded-postgres-recovery.md b/.changeset/fn-8522-windows-embedded-postgres-recovery.md new file mode 100644 index 0000000000..7ce8507037 --- /dev/null +++ b/.changeset/fn-8522-windows-embedded-postgres-recovery.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Prevent Windows embedded PostgreSQL log contention and recover once from DLL initialization crashes. +category: fix +dev: Harden native PATH, runner-log observation, and bounded owned-cluster restart behavior. diff --git a/docs/diagnostics.md b/docs/diagnostics.md index 31c9b5c98c..4cd3985795 100644 --- a/docs/diagnostics.md +++ b/docs/diagnostics.md @@ -77,6 +77,16 @@ Operator interpretation: - `ageBucket: "aging"` → review blocker progress. - `ageBucket: "stale"` → emerging stall; escalate/unblock blocker. +## Windows embedded PostgreSQL recovery (`[postgres-embedded]`, FN-8522) + +When a Fusion-owned Windows embedded cluster reports the exact `0xC0000142` backend DLL-initialization failure followed by PostgreSQL's shutdown chain, the existing startup/System diagnostic sink records: + +- `detected Windows DLL initialization shutdown; attempting one owned-cluster recovery` +- `Windows owned-cluster recovery completed; existing pools may reconnect`, or +- `Windows DLL initialization recovery failed after one retry; restart Fusion and inspect the System log` + +The recovery budget is one per lifecycle and applies only to a post-readiness cluster Fusion started. It never restarts a joined cluster. On the terminal message, restart Fusion; if it repeats, retain the System log and bundled-runtime version for support rather than deleting the data directory. + ## Process supervisor (`[process-supervisor]`) The process supervisor logs when it registers a supervised child, starts teardown, expires the grace window, escalates to `SIGKILL`, or observes a natural child exit. diff --git a/docs/storage.md b/docs/storage.md index 5256e508b7..0c6e76440e 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -11,6 +11,12 @@ See the [2026-07-14 PostgreSQL runtime cutover review](./postgres-migration-revi ## Embedded PostgreSQL startup resources +### Windows owned-cluster recovery (FN-8522) + +- Windows readiness uses bounded TCP probes. Runner-log reads are diagnostic open/read/close snapshots only; Fusion keeps no persistent runner-log handle, so PostgreSQL can retain its own `pg_ctl` log without a Fusion-induced sharing-violation retry. +- Each Windows PostgreSQL child gets the bundled native `bin` directory prepended to its own case-insensitive `PATH`; Fusion does not mutate the dashboard process environment. +- After readiness, an **owned** cluster that logs the complete `0xC0000142` backend exception plus PostgreSQL shutdown sequence gets one lifecycle-scoped restart on the same initialized data directory and port. Joiners are never restarted or stopped. A second incident, shutdown, or failed recovery is terminal and leaves the original data directory intact. + - The zero-config embedded PostgreSQL lifecycle uses mmap-backed primary shared memory to avoid exhausted SysV shared-memory IDs on constrained hosts. - The supported, tested constrained-host floor is **64MB `/dev/shm`**. Both `fn serve` and boot smoke inherit this lifecycle default; an explicit later PostgreSQL `-c shared_memory_type=…` flag remains an operator override. diff --git a/packages/core/src/__tests__/postgres/embedded-lifecycle.test.ts b/packages/core/src/__tests__/postgres/embedded-lifecycle.test.ts index c89c6c6102..56a8391aa6 100644 --- a/packages/core/src/__tests__/postgres/embedded-lifecycle.test.ts +++ b/packages/core/src/__tests__/postgres/embedded-lifecycle.test.ts @@ -1048,6 +1048,93 @@ describe("embedded-lifecycle: startup timeout (P1 #24)", () => { }); }); +describe("embedded-lifecycle: Windows fatal recovery", () => { + it("restarts an owned cluster on its resolved port when no port was configured", async () => { + const dataDir = makeDataDir(); + writeFileSync(join(dataDir, "PG_VERSION"), "15\n"); + const records: Record[] = []; + const logs: string[] = []; + class RecordingEmbeddedPostgres { + constructor(options: Record) { + records.push(options); + } + initialise = vi.fn(async () => {}); + start = vi.fn(async () => {}); + stop = vi.fn(async () => {}); + } + __setEmbeddedPostgresCtorForTests(RecordingEmbeddedPostgres as never); + const lifecycle = new EmbeddedPostgresLifecycle({ + ...baseOptions(dataDir), + startTimeoutMs: 100, + onLog: (message) => logs.push(message), + }); + const internal = lifecycle as unknown as { + running: boolean; + ownsProcess: boolean; + resolvedPort: number; + pg: { stop: () => Promise }; + ensureDatabase: () => Promise; + recoverWindowsFatalOnce: () => Promise; + }; + internal.running = true; + internal.ownsProcess = true; + internal.resolvedPort = 55491; + internal.pg = { stop: vi.fn(async () => {}) }; + internal.ensureDatabase = async () => {}; + + try { + await internal.recoverWindowsFatalOnce(); + expect(records).toHaveLength(1); + expect(records[0]?.port).toBe(55491); + expect(lifecycle.getConnectionUrl()).toContain(":55491/"); + expect(logs).toContain("embedded postgres: Windows owned-cluster recovery completed; existing pools may reconnect"); + } finally { + await lifecycle.stop(); + rmSync(dataDir, { recursive: true, force: true }); + } + }); + + it("uses the normal startup timeout to report a stalled recovery", async () => { + vi.useFakeTimers(); + const dataDir = makeDataDir(); + writeFileSync(join(dataDir, "PG_VERSION"), "15\n"); + const errors: string[] = []; + class StalledEmbeddedPostgres { + initialise = vi.fn(async () => {}); + start = vi.fn(async () => new Promise(() => {})); + stop = vi.fn(async () => {}); + } + __setEmbeddedPostgresCtorForTests(StalledEmbeddedPostgres as never); + const lifecycle = new EmbeddedPostgresLifecycle({ + ...baseOptions(dataDir), + startTimeoutMs: 25, + onError: (message) => errors.push(String(message)), + }); + const internal = lifecycle as unknown as { + running: boolean; + ownsProcess: boolean; + resolvedPort: number; + pg: { stop: () => Promise }; + recoverWindowsFatalOnce: () => Promise; + }; + internal.running = true; + internal.ownsProcess = true; + internal.resolvedPort = 55492; + internal.pg = { stop: vi.fn(async () => {}) }; + + try { + const recovery = internal.recoverWindowsFatalOnce(); + await vi.advanceTimersByTimeAsync(25); + await recovery; + expect(errors).toHaveLength(1); + expect(errors[0]).toMatch(/recovery failed after one retry/i); + expect(errors[0]).toMatch(/start timed out after 25ms/i); + } finally { + rmSync(dataDir, { recursive: true, force: true }); + } + }); +}); + describe("embedded-lifecycle: readPortFromPostmasterPid (P1 code-review fix)", () => { it("reads the TCP port from PostgreSQL's real line 4 (index 3) postmaster.pid layout", () => { const dir = mkdtempSync(join(tmpdir(), "fusion-embedded-pid-")); diff --git a/packages/core/src/__tests__/postgres/embedded-windows-elevated.test.ts b/packages/core/src/__tests__/postgres/embedded-windows-elevated.test.ts index 5f924e0812..655b19f234 100644 --- a/packages/core/src/__tests__/postgres/embedded-windows-elevated.test.ts +++ b/packages/core/src/__tests__/postgres/embedded-windows-elevated.test.ts @@ -4,6 +4,8 @@ import { buildPgCtlOptionsString, buildPgCtlStartArgs, sanitizePostgresFlags, + withWindowsNativeBinPath, + WindowsPostgresFatalDetector, } from "../../postgres/embedded-windows-elevated.js"; /* @@ -34,6 +36,53 @@ describe("sanitizePostgresFlags", () => { * open, causing EBUSY). These assertions pin the pg_ctl invariants: no-wait * launch, per-launch log file, port + flags carried in the -o option string. */ +describe("Windows fatal shutdown detector", () => { + it("recognizes the exact fatal sequence across chunks once", () => { + const detector = new WindowsPostgresFatalDetector(); + expect(detector.push("server process (PID 7696) was terminated by exception 0xC0000142\\nterminating any other active")).toBe(false); + expect(detector.push(" server processes\\nshutting down due to startup process failure\\ndatabase system is shut down")).toBe(true); + expect(detector.push("0xC0000142 database system is shut down")).toBe(false); + }); + + it("does not restart for an unordered exception followed by an ordinary shutdown", () => { + const detector = new WindowsPostgresFatalDetector(); + expect(detector.push("server process (PID 7696) was terminated by exception 0xC0000142")).toBe(false); + expect(detector.push("database system is shut down\\nterminating any other active server processes\\nshutting down due to startup process failure")).toBe(false); + }); + + it("does not treat unrelated Windows exceptions as a restart signal", () => { + const detector = new WindowsPostgresFatalDetector(); + expect(detector.push("exception 0xC0000005\\ndatabase system is shut down")).toBe(false); + expect(detector.push("exception 0xC0000142 but database remains ready")).toBe(false); + }); +}); + +describe("Windows child PATH hardening", () => { + it("prefixes the native bin while preserving a case-insensitive inherited Path", () => { + const environment = { Path: "C:\\Windows;C:\\pg\\bin", KEEP: "yes" }; + const result = withWindowsNativeBinPath(environment, "C:\\Fusion Runtime\\native", "win32"); + + expect(result).toEqual({ + Path: "C:\\Fusion Runtime\\native\\bin;C:\\Windows;C:\\pg\\bin", + KEEP: "yes", + }); + }); + + it("uses PATH when missing and removes duplicate native-bin entries", () => { + const result = withWindowsNativeBinPath( + { PATH: "c:\\fusion\\native\\bin;C:\\Windows;c:\\FUSION\\NATIVE\\BIN" }, + "C:\\Fusion\\native", + "win32", + ); + expect(result.PATH).toBe("C:\\Fusion\\native\\bin;C:\\Windows"); + }); + + it("leaves non-Windows launch environments untouched", () => { + const environment = { PATH: "/usr/bin", KEEP: "yes" }; + expect(withWindowsNativeBinPath(environment, "/runtime/native", "linux")).toBe(environment); + }); +}); + describe("pg_ctl elevated launch composition", () => { it("carries the port and sanitized flags in the -o option string", () => { expect(buildPgCtlOptionsString(55499, ["-c", "shared_memory_type=sysv"])).toBe( diff --git a/packages/core/src/postgres/embedded-lifecycle.ts b/packages/core/src/postgres/embedded-lifecycle.ts index 6fe7a55c12..50ecdbf90a 100644 --- a/packages/core/src/postgres/embedded-lifecycle.ts +++ b/packages/core/src/postgres/embedded-lifecycle.ts @@ -70,6 +70,8 @@ import type { ResolvedBackend } from "./backend-resolver.js"; import { isWindowsElevatedAdmin, startServerElevatedRestricted, + WindowsPostgresFatalDetector, + withWindowsNativeBinPath, type ElevatedServerHandle, type ElevatedStartOptions, } from "./embedded-windows-elevated.js"; @@ -455,6 +457,38 @@ let electronAsarNativePathPatchRestore: (() => void) | null = null; type MutableSpawnModule = { spawn: (...args: unknown[]) => unknown; }; + +type SpawnOptionsLike = { env?: NodeJS.ProcessEnv; windowsHide?: boolean }; + +function isWindowsEmbeddedPostgresBinary(command: unknown): command is string { + return process.platform === "win32" && + typeof command === "string" && + /[\\/]bin[\\/](?:postgres|initdb|pg_ctl)\.exe$/i.test(command); +} + +/** + * Add the sibling bundled bin directory only to a native PostgreSQL spawn. + * + * FNXC:PostgresEmbedded 2026-07-22-16:10: + * `embedded-postgres` copies process.env when it spawns normal Windows + * postmasters. Patch that narrow spawn seam rather than mutating process.env, + * so npm/pnpm, standalone runtime-bin, and Electron materialized payloads all + * give descendants their DLL directory without changing unrelated children. + */ +function withWindowsPostgresSpawnEnvironment(command: unknown, rest: unknown[]): unknown[] { + if (!isWindowsEmbeddedPostgresBinary(command)) return rest; + const args = [...rest]; + const optionIndex = args.length - 1; + const existing = args[optionIndex] as SpawnOptionsLike | undefined; + if (!existing || typeof existing !== "object" || Array.isArray(existing)) return args; + const binDir = dirname(command); + const nativeRoot = dirname(binDir); + args[optionIndex] = { + ...existing, + env: withWindowsNativeBinPath(existing.env ?? process.env, nativeRoot), + } satisfies SpawnOptionsLike; + return args; +} type MutableFsPromisesModule = { stat: (...args: unknown[]) => unknown; chmod: (...args: unknown[]) => unknown; @@ -500,7 +534,10 @@ export function installElectronAsarNativePathPatch(): void { childProcessMod.spawn = (command: unknown, ...rest: unknown[]) => { const fixedCommand = typeof command === "string" ? resolveElectronAsarUnpackedPath(command) : command; - return originalSpawn(fixedCommand, ...rest); + return originalSpawn( + fixedCommand, + ...withWindowsPostgresSpawnEnvironment(fixedCommand, rest), + ); }; const fsPromisesMod = require("fs/promises") as MutableFsPromisesModule; @@ -1122,6 +1159,10 @@ export class EmbeddedPostgresLifecycle { * on a failure that is handled before the timeout fires. */ private startTimer: NodeJS.Timeout | null = null; + private readonly windowsFatalDetector = new WindowsPostgresFatalDetector(); + private recoveryAttempts = 0; + private recoveryInFlight: Promise | null = null; + private stopRequested = false; constructor(opts: EmbeddedLifecycleOptions) { this.options = { @@ -1139,6 +1180,16 @@ export class EmbeddedPostgresLifecycle { }; } + /** + * Forward native process output to the existing sink, then schedule recovery + * only for a confirmed owned Windows fatal shutdown sequence. + */ + private forwardPostgresLog = (message: string): void => { + this.options.onLog(message); + if (process.platform !== "win32" || !this.running || !this.ownsProcess) return; + if (this.windowsFatalDetector.push(message)) void this.recoverWindowsFatalOnce(); + }; + /** The configured or discovered port. Undefined until assigned (explicit or discovered in `start()`). */ getPort(): number | undefined { return this.options.port ?? this.resolvedPort; @@ -1256,11 +1307,20 @@ export class EmbeddedPostgresLifecycle { migrationUrlOverridden: false, }; } + return this.startBounded(); + } + + /** + * Start an owned postmaster with the same cancellation and timeout contract + * used by public startup. Recovery calls this directly because it must not + * join a stale pid file or allocate a new endpoint between pool reconnects. + */ + private async startBounded(preferredPort?: number): Promise { if (this.options.startTimeoutMs <= 0) { - return this.startInternal(); + return this.startInternal(undefined, preferredPort); } const controller = new AbortController(); - const startAttempt = this.startInternal(controller.signal); + const startAttempt = this.startInternal(controller.signal, preferredPort); let timer: NodeJS.Timeout | undefined; const timeout = new Promise((_resolve, reject) => { timer = setTimeout(() => { @@ -1272,7 +1332,6 @@ export class EmbeddedPostgresLifecycle { ), ); }, this.options.startTimeoutMs); - // Unref so the timer alone does not keep the event loop alive. if (timer && typeof timer.unref === "function") timer.unref(); }); this.startTimer = timer ?? null; @@ -1280,8 +1339,6 @@ export class EmbeddedPostgresLifecycle { return await Promise.race([startAttempt, timeout]); } catch (err) { controller.abort(); - // On timeout (or any failure), best-effort clean up the partial state so - // a retry starts fresh. stop() is safe to call even when not fully running. await this.stop().catch(() => undefined); throw err; } finally { @@ -1290,12 +1347,15 @@ export class EmbeddedPostgresLifecycle { } } - /** - * The actual start sequence, with no timeout wrapper. Called by {@link start} - * either directly (timeout disabled) or via Promise.race with the timeout. - */ - private async startInternal(signal?: AbortSignal): Promise { - const port = this.options.port ?? (await findFreePort()); + /** The actual start sequence, invoked only through {@link startBounded}. */ + private async startInternal(signal?: AbortSignal, preferredPort?: number): Promise { + /* + FNXC:PostgresEmbedded 2026-07-22-23:05: + A Windows crash recovery must retain the originally resolved endpoint even + when no explicit port was configured. Reallocating here strands existing + task-store pools on the dead port, so only a first launch may find a port. + */ + const port = this.options.port ?? preferredPort ?? this.resolvedPort ?? (await findFreePort()); if (signal?.aborted) throw new EmbeddedStartCancelledError(this.options.dataDir); this.resolvedPort = port; @@ -1312,7 +1372,7 @@ export class EmbeddedPostgresLifecycle { authMethod: "password", initdbFlags: [...this.options.initdbFlags], postgresFlags: [...this.options.postgresFlags], - onLog: this.options.onLog, + onLog: this.forwardPostgresLog, onError: this.options.onError, }); this.pg = pg; @@ -1369,7 +1429,7 @@ export class EmbeddedPostgresLifecycle { dataDir: this.options.dataDir, port, postgresFlags: this.options.postgresFlags, - onLog: this.options.onLog, + onLog: this.forwardPostgresLog, onError: this.options.onError, startTimeoutMs: this.options.startTimeoutMs, signal, @@ -1487,6 +1547,44 @@ export class EmbeddedPostgresLifecycle { }; } + /** + * FNXC:PostgresEmbedded 2026-07-22-16:25: + * A 0xC0000142 backend crash shuts down its whole PostgreSQL cluster. One + * lifecycle-owned retry reuses the initialized directory and same resolved + * port; joiners, stop/detach, and a second incident are deliberately inert. + */ + private async recoverWindowsFatalOnce(): Promise { + if (this.recoveryInFlight || this.recoveryAttempts >= 1 || this.stopRequested || !this.ownsProcess) return; + this.recoveryAttempts += 1; + this.recoveryInFlight = (async () => { + this.options.onLog("embedded postgres: detected Windows DLL initialization shutdown; attempting one owned-cluster recovery"); + try { + if (this.nonAdminHandle) await this.nonAdminHandle.stop(); + else await this.pg?.stop(); + this.pg = null; + this.nonAdminHandle = null; + this.running = false; + runningInstances.delete(this.options.dataDir); + if (this.stopRequested || !this.ownsProcess) return; + const recoveryPort = this.resolvedPort; + if (recoveryPort === undefined) { + throw new Error("embedded postgres: recovery lost its resolved port"); + } + await this.startBounded(recoveryPort); + this.options.onLog("embedded postgres: Windows owned-cluster recovery completed; existing pools may reconnect"); + } catch (error) { + this.running = false; + runningInstances.delete(this.options.dataDir); + this.options.onError( + `embedded postgres: Windows DLL initialization recovery failed after one retry; restart Fusion and inspect the System log. ${error instanceof Error ? error.message : String(error)}`, + ); + } finally { + this.recoveryInFlight = null; + } + })(); + await this.recoveryInFlight; + } + private async settleCancelledStart(pg: EmbeddedPostgresInstance): Promise { // FNXC:WindowsDesktopPackaging 2026-07-15-05:20: // Prefer stopping a non-admin handle (if already assigned) before asking @@ -1649,6 +1747,7 @@ export class EmbeddedPostgresLifecycle { */ detachWithoutStop(): void { this.uninstallShutdownHook(); + this.nonAdminHandle?.stopMonitoring(); this.pg = null; this.nonAdminHandle = null; this.running = false; @@ -1657,6 +1756,7 @@ export class EmbeddedPostgresLifecycle { } async stop(): Promise { + this.stopRequested = true; this.uninstallShutdownHook(); // FNXC:PostgresCutover 2026-06-27-11:10: diff --git a/packages/core/src/postgres/embedded-windows-elevated.ts b/packages/core/src/postgres/embedded-windows-elevated.ts index b1f572ed5f..5fe0e0d317 100644 --- a/packages/core/src/postgres/embedded-windows-elevated.ts +++ b/packages/core/src/postgres/embedded-windows-elevated.ts @@ -42,6 +42,8 @@ export interface ElevatedServerHandle { * starter; a caller that lost a startup race must use {@link stopWrapperOnly}. */ stop(): Promise; + /** Stop bounded runner-log observation without affecting PostgreSQL. */ + stopMonitoring(): void; /** * Reap only what this handle launched, never the postmaster named by the * shared data dir. @@ -56,6 +58,40 @@ export interface ElevatedServerHandle { stopWrapperOnly(): Promise; } +/** + * Stateful, bounded detector for PostgreSQL's Windows DLL-init shutdown chain. + * + * FNXC:PostgresEmbedded 2026-07-22-16:25: + * Restart only after the exact ordered PostgreSQL backend-crash shutdown chain. + * A lone Windows exception, an ordinary shutdown following an earlier exception, + * or a repeated log snapshot is not permission to restart a live cluster. + * + * FNXC:PostgresEmbedded 2026-07-22-22:21: + * Issue #2411 recovery is reserved for PostgreSQL's ordered DLL-init failure: + * backend 0xC0000142, peer termination, startup-process failure, then shutdown. + * Do not infer a cluster crash from unordered snippets because an operator or an + * external owner can shut down a cluster after an unrelated Windows exception. + */ +export class WindowsPostgresFatalDetector { + private buffer = ""; + private matched = false; + + push(chunk: string): boolean { + if (this.matched || !chunk) return false; + this.buffer = (this.buffer + chunk).slice(-16_384); + const text = this.buffer.toLowerCase(); + if ( + /server process(?:\s+\(pid\s+\d+\))?\s+was terminated by exception\s+0xc0000142[\s\S]*?terminating any other active server processes[\s\S]*?shutting down due to startup process failure[\s\S]*?database system is shut down/.test( + text, + ) + ) { + this.matched = true; + return true; + } + return false; + } +} + export interface ElevatedStartOptions { /** .../native dir containing bin/postgres.exe + bin/pg_ctl.exe + lib + share. */ readonly nativeRoot: string; @@ -152,6 +188,33 @@ export function buildPgCtlStartArgs(dataDir: string, logFile: string, optionsStr return ["-D", dataDir, "-o", optionsString, "-l", logFile, "-W", "start"]; } +/** + * Build a child-only Windows environment with the bundled PostgreSQL bin first. + * + * FNXC:PostgresEmbedded 2026-07-22-16:10: + * PostgreSQL backend children inherit the environment used for `pg_ctl`/`postgres`. + * Keep the resolved native `bin` directory first in that child PATH so DLL lookup + * cannot depend on the dashboard's install shape. Windows PATH keys are + * case-insensitive; preserve the caller's spelling and every other variable. + */ +export function withWindowsNativeBinPath( + environment: NodeJS.ProcessEnv, + nativeRoot: string, + platform: NodeJS.Platform = process.platform, +): NodeJS.ProcessEnv { + if (platform !== "win32") return environment; + const pathKey = Object.keys(environment).find((key) => key.toLowerCase() === "path") ?? "PATH"; + // `node:path` follows the host platform, while tests and cross-compiled + // packaging can construct a Windows child environment from a non-Windows host. + const bin = nativeRoot.replace(/[\\/]+$/, "") + "\\bin"; + const normalize = (value: string) => value.replace(/\//g, "\\").toLowerCase(); + const inherited = environment[pathKey] ?? ""; + const segments = inherited.split(";").filter(Boolean); + const normalizedBin = normalize(bin); + const deduplicated = segments.filter((segment) => normalize(segment) !== normalizedBin); + return { ...environment, [pathKey]: [bin, ...deduplicated].join(";") }; +} + function readPostgresPid(dataDir: string): number | null { try { const lines = readFileSync(join(dataDir, "postmaster.pid"), "utf-8").split("\n"); @@ -231,7 +294,13 @@ export async function startServerElevatedRestricted( ); let stopped = false; + let logMonitor: NodeJS.Timeout | null = null; + const stopMonitoring = (): void => { + if (logMonitor) clearInterval(logMonitor); + logMonitor = null; + }; const killAll = (): void => { + stopMonitoring(); if (stopped) return; stopped = true; const r = spawnSync(pgCtl, ["-D", opts.dataDir, "-m", "fast", "-t", "30", "-w", "stop"], { @@ -273,9 +342,11 @@ export async function startServerElevatedRestricted( killAll(); await waitForDown(); }, + stopMonitoring, async stopWrapperOnly() { // No wrapper exists on this path; a lock-race loser postmaster exits on // its own. Only mark stopped so a later stop() cannot kill a race winner. + stopMonitoring(); stopped = true; }, }; @@ -296,7 +367,12 @@ export async function startServerElevatedRestricted( // until the outer timeout). Output captured before exit is still reported. const launch = await new Promise<{ status: number | null; output: () => string }>( (resolve, reject) => { - const child = spawn(pgCtl, args, { windowsHide: true }); + const child = spawn(pgCtl, args, { + windowsHide: true, + // Do not mutate process.env: sibling dashboard/engine children must + // retain their inherited environment unchanged. + env: withWindowsNativeBinPath(process.env, opts.nativeRoot), + }); let output = ""; child.stdout.on("data", (d: Buffer) => (output += d.toString())); child.stderr.on("data", (d: Buffer) => (output += d.toString())); @@ -325,18 +401,18 @@ export async function startServerElevatedRestricted( killAll(); throw new Error("embedded postgres: elevated launch cancelled before ready."); } + // TCP is the readiness authority. The old reader inspected pg_ctl's `-l` + // file on every poll and made that implementation detail part of startup. + // Diagnostic snapshots are still bounded/open-read-close only; they never + // retain a descriptor or delay readiness when Windows reports EBUSY. + if (await probeTcpPort(opts.port, 500)) { + ready = true; + break; + } const tail = readTail(logFile, 3000); if (tail !== lastSnapshot) { lastSnapshot = tail; - opts.onLog(`elevated poll pg={${tail.slice(-400)}}`); - } - if (/database system is ready to accept connections/.test(tail)) { - // Log readiness alone is not enough: confirm TCP accept on 127.0.0.1 so - // ensureDatabase cannot hang on a connect that never completes. - if (await probeTcpPort(opts.port, 500)) { - ready = true; - break; - } + opts.onLog(`elevated diagnostic pg={${tail.slice(-400)}}`); } if (/\bFATAL\b|\bPANIC\b|could not (bind|start|create|access|connect|load)|not permitted|Permission denied|is not the owner/i.test(tail)) { killAll(); @@ -371,6 +447,29 @@ export async function startServerElevatedRestricted( opts.onLog( `embedded postgres: elevated server ready on 127.0.0.1:${opts.port} (pid ${postgresPid ?? 0}, restricted token)`, ); + + /* + FNXC:PostgresEmbedded 2026-07-22-23:05: + pg_ctl exits before its restricted-token postmaster, so readiness cannot end + observation of its bounded runner log. Poll open/read/close snapshots only + while this owned handle exists; stop, detach, and race cleanup release the + timer. This forwards a post-ready 0xC0000142 shutdown chain to the lifecycle + without retaining an exclusive Windows file handle. + */ + let lastLogSnapshot = ""; + const observePostReadyLog = (): void => { + if (stopped) return; + const snapshot = readTail(logFile, 16_384); + if (snapshot === "(no log file)" || snapshot === lastLogSnapshot) return; + const next = snapshot.startsWith(lastLogSnapshot) + ? snapshot.slice(lastLogSnapshot.length) + : snapshot; + lastLogSnapshot = snapshot; + if (next) opts.onLog(next); + }; + observePostReadyLog(); + logMonitor = setInterval(observePostReadyLog, 250); + if (typeof logMonitor.unref === "function") logMonitor.unref(); return handle; }