From de1638e262eb62ad42b9b065af14416a9e333e92 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Thu, 16 Jul 2026 06:00:12 -0700 Subject: [PATCH] FN-8090: use mmap shared memory for embedded PostgreSQL Enable constrained-host embedded PostgreSQL startup without SysV shared-memory exhaustion. - Default embedded lifecycle flags to mmap-backed shared memory while preserving caller overrides - Cover normal and elevated Windows launch paths with deterministic flag propagation tests - Document the 64MB /dev/shm support floor and add a patch changeset Files changed: .changeset/fn-8090-embedded-pg-shm.md | 7 ++ docs/postgres-migration-review-2026-07-14.md | 4 + docs/storage.md | 5 ++ .../__tests__/postgres/embedded-lifecycle.test.ts | 88 ++++++++++++++++++++++ .../postgres/embedded-windows-admin.test.ts | 17 +++++ packages/core/src/postgres/embedded-lifecycle.ts | 50 +++++++++++- .../core/src/postgres/embedded-windows-admin.ts | 2 +- 7 files changed, 168 insertions(+), 5 deletions(-) Fusion-Task-Id: FN-8090 Fusion-Task-Lineage: ac175843-69ba-4c9c-9692-aff095fc351f Co-authored-by: Fusion (runfusion.ai) --- .changeset/fn-8090-embedded-pg-shm.md | 7 ++ docs/postgres-migration-review-2026-07-14.md | 4 + docs/storage.md | 5 ++ .../postgres/embedded-lifecycle.test.ts | 88 +++++++++++++++++++ .../postgres/embedded-windows-admin.test.ts | 17 ++++ .../core/src/postgres/embedded-lifecycle.ts | 50 ++++++++++- .../src/postgres/embedded-windows-admin.ts | 2 +- 7 files changed, 168 insertions(+), 5 deletions(-) create mode 100644 .changeset/fn-8090-embedded-pg-shm.md create mode 100644 packages/core/src/__tests__/postgres/embedded-windows-admin.test.ts diff --git a/.changeset/fn-8090-embedded-pg-shm.md b/.changeset/fn-8090-embedded-pg-shm.md new file mode 100644 index 0000000000..d67e232dcd --- /dev/null +++ b/.changeset/fn-8090-embedded-pg-shm.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Embedded PostgreSQL now boots on hosts with a 64MB /dev/shm. +category: fix +dev: Defaults the embedded lifecycle to mmap-backed primary shared memory while preserving later caller flag overrides. diff --git a/docs/postgres-migration-review-2026-07-14.md b/docs/postgres-migration-review-2026-07-14.md index fa3c87959f..9ca4417438 100644 --- a/docs/postgres-migration-review-2026-07-14.md +++ b/docs/postgres-migration-review-2026-07-14.md @@ -12,6 +12,10 @@ Fusion no longer supports SQLite as a live runtime backend. Startup selects eith Legacy `fusion.db`, `archive.db`, and `fusion-central.db` files remain readable only at controlled identity-discovery and one-time migration/import seams. They are never a supported write target or runtime fallback. `.fusion/project.json` is the local project identity marker after cutover. +### Embedded PostgreSQL resource floor + +The zero-config embedded PostgreSQL lifecycle uses mmap-backed primary shared memory so hosts with constrained SysV shared-memory IDs can boot without operator tuning. The supported, tested constrained-host floor is **64MB `/dev/shm`**; `fn serve` and the built boot smoke inherit this default. Operators can still provide a later PostgreSQL `-c shared_memory_type=…` flag when a deployment needs an explicit override. + ## PostgreSQL-authoritative inventory | Surface | PostgreSQL authority | diff --git a/docs/storage.md b/docs/storage.md index 992e784629..8af040b7b9 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -4,6 +4,11 @@ See the [2026-07-14 PostgreSQL runtime cutover review](./postgres-migration-review-2026-07-14.md) for the audited authority inventory, exact authorized legacy readers, and deployment/rollback checklist. +## Embedded PostgreSQL startup resources + +- The zero-config embedded PostgreSQL lifecycle uses mmap-backed primary shared memory to avoid exhausted SysV shared-memory IDs on constrained hosts. +- The supported, tested constrained-host floor is **64MB `/dev/shm`**. Both `fn serve` and boot smoke inherit this lifecycle default; an explicit later PostgreSQL `-c shared_memory_type=…` flag remains an operator override. + ## Task-ID allocator authority and compatibility - `distributed_task_id_state` is the authoritative local task-ID allocator state. `nextSequence` is the active high-water mark used for local ID reservations. diff --git a/packages/core/src/__tests__/postgres/embedded-lifecycle.test.ts b/packages/core/src/__tests__/postgres/embedded-lifecycle.test.ts index dc21898d33..69583f020f 100644 --- a/packages/core/src/__tests__/postgres/embedded-lifecycle.test.ts +++ b/packages/core/src/__tests__/postgres/embedded-lifecycle.test.ts @@ -33,11 +33,15 @@ import { EmbeddedPostgresLifecycle, EmbeddedStartTimeoutError, DEFAULT_START_TIMEOUT_MS, + DEFAULT_EMBEDDED_POSTGRES_FLAGS, isDataDirInitialized, isWindowsElevatedAdmin, normalizeMacosEmbeddedPostgresDylibSymlinks, readPortFromPostmasterPid, __setEmbeddedPostgresCtorForTests, + __setWindowsElevatedAdminForTests, + __setWindowsEmbeddedPostgresNativeRootForTests, + __setWindowsLauncherForTests, resolveElectronAsarUnpackedPath, fingerprintEmbeddedPostgresNativeRoot, buildEmbeddedPostgresMaterializationMarker, @@ -60,6 +64,9 @@ const tracked: Array<{ afterEach(async () => { __setEmbeddedPostgresCtorForTests(null); + __setWindowsElevatedAdminForTests(null); + __setWindowsEmbeddedPostgresNativeRootForTests(null); + __setWindowsLauncherForTests(null); vi.useRealTimers(); while (tracked.length > 0) { const { lifecycle, dataDir } = tracked.pop()!; @@ -1094,6 +1101,87 @@ describe("embedded-lifecycle: readPortFromPostmasterPid (P1 code-review fix)", ( }); }); +/* + * FNXC:PostgresEmbedded 2026-07-16-12:45: + * Assert the shared-memory floor at the lifecycle boundary, which is used by + * both startup-factory boot and direct lifecycle callers. Mock starts reject + * before database work so these tests stay deterministic and process-free. + */ +describe("embedded-lifecycle: shared-memory-safe postgres flags", () => { + const sentinel = new Error("mock postgres start complete"); + + function installCtorRecorder(records: Record[]): void { + class RecordingEmbeddedPostgres { + constructor(options: Record) { + records.push(options); + } + initialise = vi.fn(async () => {}); + async start() { + throw sentinel; + } + stop = vi.fn(async () => {}); + } + __setEmbeddedPostgresCtorForTests(RecordingEmbeddedPostgres as never); + } + + it.each([ + ["omitted", undefined, [...DEFAULT_EMBEDDED_POSTGRES_FLAGS]], + ["empty", [], [...DEFAULT_EMBEDDED_POSTGRES_FLAGS]], + [ + "caller override after the default", + ["-c", "shared_memory_type=sysv"], + [...DEFAULT_EMBEDDED_POSTGRES_FLAGS, "-c", "shared_memory_type=sysv"], + ], + ])("passes %s flags to the normal embedded-postgres constructor", async (_state, postgresFlags, expected) => { + const dataDir = makeDataDir(); + const records: Record[] = []; + installCtorRecorder(records); + try { + writeFileSync(join(dataDir, "PG_VERSION"), "15\n"); + const lifecycle = new EmbeddedPostgresLifecycle({ ...baseOptions(dataDir), postgresFlags }); + + await expect(lifecycle.start()).rejects.toBe(sentinel); + expect(records).toHaveLength(1); + expect(records[0]?.postgresFlags).toEqual(expected); + } finally { + rmSync(dataDir, { recursive: true, force: true }); + } + }); + + it("passes the ordered defaults and caller override through the elevated Windows launcher", async () => { + const dataDir = makeDataDir(); + const records: Record[] = []; + const launcherSentinel = new Error("mock Windows launcher reached"); + let launcherOptions: Record | undefined; + installCtorRecorder(records); + __setWindowsElevatedAdminForTests(true); + __setWindowsEmbeddedPostgresNativeRootForTests("/test/embedded-postgres/native"); + __setWindowsLauncherForTests(async (options) => { + launcherOptions = options; + throw launcherSentinel; + }); + + try { + // Reuse skips real initdb; the sentinel rejects before ensureDatabase. + writeFileSync(join(dataDir, "PG_VERSION"), "15\n"); + const lifecycle = new EmbeddedPostgresLifecycle({ + ...baseOptions(dataDir), + postgresFlags: ["-c", "shared_memory_type=sysv"], + }); + + await expect(lifecycle.start()).rejects.toBe(launcherSentinel); + expect(records).toHaveLength(1); + expect(launcherOptions?.postgresFlags).toEqual([ + ...DEFAULT_EMBEDDED_POSTGRES_FLAGS, + "-c", + "shared_memory_type=sysv", + ]); + } finally { + rmSync(dataDir, { recursive: true, force: true }); + } + }); +}); + describe("embedded-lifecycle: signal re-raise (P1 #23)", () => { it("boundShutdown re-raises real signals via process.kill (unit, no process)", async () => { // Verify the signal re-raise logic without a real cluster: construct a diff --git a/packages/core/src/__tests__/postgres/embedded-windows-admin.test.ts b/packages/core/src/__tests__/postgres/embedded-windows-admin.test.ts new file mode 100644 index 0000000000..56094f7bf9 --- /dev/null +++ b/packages/core/src/__tests__/postgres/embedded-windows-admin.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from "vitest"; +import { DEFAULT_EMBEDDED_POSTGRES_FLAGS } from "../../postgres/embedded-lifecycle.js"; +import { sanitizePostgresFlags } from "../../postgres/embedded-windows-admin.js"; + +/* + * FNXC:PostgresEmbedded 2026-07-16-12:45: + * The constrained-host shared-memory default must retain its exact `-c` form + * through the Windows cmd.exe launcher sanitizer. This is pure validation + * coverage; it does not require an elevated process or a Windows binary. + */ +describe("sanitizePostgresFlags", () => { + it("preserves the shared-memory default and a caller override unchanged", () => { + const flags = [...DEFAULT_EMBEDDED_POSTGRES_FLAGS, "-c", "shared_memory_type=sysv"]; + + expect(sanitizePostgresFlags(flags)).toEqual(flags); + }); +}); diff --git a/packages/core/src/postgres/embedded-lifecycle.ts b/packages/core/src/postgres/embedded-lifecycle.ts index f549c217bb..ac14444aea 100644 --- a/packages/core/src/postgres/embedded-lifecycle.ts +++ b/packages/core/src/postgres/embedded-lifecycle.ts @@ -71,6 +71,7 @@ import { isWindowsElevatedAdmin, startServerAsNonAdminUser, type NonAdminServerHandle, + type NonAdminStartOptions, } from "./embedded-windows-admin.js"; // FNXC:WindowsDesktopPackaging 2026-07-14-22:53: // Static import so tsup/esbuild bundles postgres.js into packages/cli/dist/bin.js. @@ -469,6 +470,33 @@ export function __setEmbeddedPostgresCtorForTests(ctor: EmbeddedPostgresCtor | n embeddedPostgresCtorIsTestOverride = ctor !== null; } +/* + * FNXC:PostgresEmbedded 2026-07-16-12:45: + * Cross-platform tests must exercise the elevated Windows launcher without an + * elevated token, Windows binaries, or a running database. These narrow seams + * replace only the branch dependencies during a test; production always calls + * the imported implementations. + */ +let windowsElevatedAdminForTests: boolean | null = null; +let windowsNativeRootForTests: string | null = null; +let windowsLauncherForTests: + | ((opts: NonAdminStartOptions) => Promise) + | null = null; + +export function __setWindowsElevatedAdminForTests(value: boolean | null): void { + windowsElevatedAdminForTests = value; +} + +export function __setWindowsEmbeddedPostgresNativeRootForTests(value: string | null): void { + windowsNativeRootForTests = value; +} + +export function __setWindowsLauncherForTests( + launcher: ((opts: NonAdminStartOptions) => Promise) | null, +): void { + windowsLauncherForTests = launcher; +} + function getEmbeddedPostgresCtor(): EmbeddedPostgresCtor { if (embeddedPostgresCtorCache) return embeddedPostgresCtorCache; // FNXC:DesktopEmbeddedPostgres 2026-07-14-18:30: @@ -490,6 +518,17 @@ export const DEFAULT_EMBEDDED_PASSWORD = "password"; /** Default application database name created/ensured on the embedded cluster. */ export const DEFAULT_EMBEDDED_DATABASE = "fusion"; +/* + * FNXC:PostgresEmbedded 2026-07-16-12:45: + * Embedded PostgreSQL 15's primary postmaster allocation used SysV shmget and + * failed with `could not create shared memory segment: No space left on device` + * when host SHMMNI/SHMALL was constrained. Use mmap-backed primary shared memory + * so the zero-config cluster has been boot-smoke tested with a 64MB /dev/shm + * lower bound. Defaults precede caller flags because PostgreSQL applies repeated + * `-c key=value` settings last-wins, preserving an operator's explicit override. + */ +export const DEFAULT_EMBEDDED_POSTGRES_FLAGS = ["-c", "shared_memory_type=mmap"] as const; + /** * FNXC:PostgresEmbedded 2026-06-24-09:05: * Default data directory location for the embedded cluster. Mirrors the @@ -914,7 +953,7 @@ export class EmbeddedPostgresLifecycle { user: opts.user ?? DEFAULT_EMBEDDED_USER, password: opts.password ?? DEFAULT_EMBEDDED_PASSWORD, initdbFlags: opts.initdbFlags ?? [], - postgresFlags: opts.postgresFlags ?? [], + postgresFlags: [...DEFAULT_EMBEDDED_POSTGRES_FLAGS, ...(opts.postgresFlags ?? [])], startTimeoutMs: opts.startTimeoutMs ?? DEFAULT_START_TIMEOUT_MS, onLog: opts.onLog ?? ((msg: string) => log.log(msg)), onError: @@ -1128,8 +1167,11 @@ export class EmbeddedPostgresLifecycle { // Skip the real non-admin path when tests inject a mock ctor so delayed // start/cancellation coverage exercises pg.start() even on elevated CI. try { - if (isWindowsElevatedAdmin() && !embeddedPostgresCtorIsTestOverride) { - const nativeRoot = resolveWindowsEmbeddedPostgresNativeRoot(); + const isElevatedWindows = windowsElevatedAdminForTests ?? isWindowsElevatedAdmin(); + // A launcher seam intentionally coexists with the ctor seam so this branch + // can be covered off Windows; without that seam, ctor mocks retain normal-path behavior. + if (isElevatedWindows && (!embeddedPostgresCtorIsTestOverride || windowsLauncherForTests)) { + const nativeRoot = windowsNativeRootForTests ?? resolveWindowsEmbeddedPostgresNativeRoot(); if (!nativeRoot) { throw new Error( "embedded postgres: the process is running elevated on Windows, where " + @@ -1139,7 +1181,7 @@ export class EmbeddedPostgresLifecycle { "non-elevated, or ensure the embedded-postgres platform package is installed.", ); } - this.nonAdminHandle = await startServerAsNonAdminUser({ + this.nonAdminHandle = await (windowsLauncherForTests ?? startServerAsNonAdminUser)({ nativeRoot, dataDir: this.options.dataDir, port, diff --git a/packages/core/src/postgres/embedded-windows-admin.ts b/packages/core/src/postgres/embedded-windows-admin.ts index 521791eb31..8a36bc9d68 100644 --- a/packages/core/src/postgres/embedded-windows-admin.ts +++ b/packages/core/src/postgres/embedded-windows-admin.ts @@ -281,7 +281,7 @@ function readTail(file: string, max: number): string { * Reject postgresFlags that would break cmd.exe quoting or enable injection * when embedded into launch.bat (review: arbitrary flags with % " & | etc.). */ -function sanitizePostgresFlags(flags: readonly string[]): string[] { +export function sanitizePostgresFlags(flags: readonly string[]): string[] { const safe: string[] = []; for (const flag of flags) { if (typeof flag !== "string" || flag.length === 0) {