FN-8090: use mmap shared memory for embedded PostgreSQL
Enable constrained-host embedded PostgreSQL startup without SysV shared-memory exhaustion. - Default embedded lifecycle flags to mmap-backed shared memory while preserving caller overrides - Cover normal and elevated Windows launch paths with deterministic flag propagation tests - Document the 64MB /dev/shm support floor and add a patch changeset Files changed: .changeset/fn-8090-embedded-pg-shm.md | 7 ++ docs/postgres-migration-review-2026-07-14.md | 4 + docs/storage.md | 5 ++ .../__tests__/postgres/embedded-lifecycle.test.ts | 88 ++++++++++++++++++++++ .../postgres/embedded-windows-admin.test.ts | 17 +++++ packages/core/src/postgres/embedded-lifecycle.ts | 50 +++++++++++- .../core/src/postgres/embedded-windows-admin.ts | 2 +- 7 files changed, 168 insertions(+), 5 deletions(-) Fusion-Task-Id: FN-8090 Fusion-Task-Lineage: ac175843-69ba-4c9c-9692-aff095fc351f Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
This commit is contained in:
7
.changeset/fn-8090-embedded-pg-shm.md
Normal file
7
.changeset/fn-8090-embedded-pg-shm.md
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
---
|
||||||
|
"@runfusion/fusion": patch
|
||||||
|
---
|
||||||
|
|
||||||
|
summary: Embedded PostgreSQL now boots on hosts with a 64MB /dev/shm.
|
||||||
|
category: fix
|
||||||
|
dev: Defaults the embedded lifecycle to mmap-backed primary shared memory while preserving later caller flag overrides.
|
||||||
@@ -12,6 +12,10 @@ Fusion no longer supports SQLite as a live runtime backend. Startup selects eith
|
|||||||
|
|
||||||
Legacy `fusion.db`, `archive.db`, and `fusion-central.db` files remain readable only at controlled identity-discovery and one-time migration/import seams. They are never a supported write target or runtime fallback. `.fusion/project.json` is the local project identity marker after cutover.
|
Legacy `fusion.db`, `archive.db`, and `fusion-central.db` files remain readable only at controlled identity-discovery and one-time migration/import seams. They are never a supported write target or runtime fallback. `.fusion/project.json` is the local project identity marker after cutover.
|
||||||
|
|
||||||
|
### Embedded PostgreSQL resource floor
|
||||||
|
|
||||||
|
The zero-config embedded PostgreSQL lifecycle uses mmap-backed primary shared memory so hosts with constrained SysV shared-memory IDs can boot without operator tuning. The supported, tested constrained-host floor is **64MB `/dev/shm`**; `fn serve` and the built boot smoke inherit this default. Operators can still provide a later PostgreSQL `-c shared_memory_type=…` flag when a deployment needs an explicit override.
|
||||||
|
|
||||||
## PostgreSQL-authoritative inventory
|
## PostgreSQL-authoritative inventory
|
||||||
|
|
||||||
| Surface | PostgreSQL authority |
|
| Surface | PostgreSQL authority |
|
||||||
|
|||||||
@@ -4,6 +4,11 @@
|
|||||||
|
|
||||||
See the [2026-07-14 PostgreSQL runtime cutover review](./postgres-migration-review-2026-07-14.md) for the audited authority inventory, exact authorized legacy readers, and deployment/rollback checklist.
|
See the [2026-07-14 PostgreSQL runtime cutover review](./postgres-migration-review-2026-07-14.md) for the audited authority inventory, exact authorized legacy readers, and deployment/rollback checklist.
|
||||||
|
|
||||||
|
## Embedded PostgreSQL startup resources
|
||||||
|
|
||||||
|
- The zero-config embedded PostgreSQL lifecycle uses mmap-backed primary shared memory to avoid exhausted SysV shared-memory IDs on constrained hosts.
|
||||||
|
- The supported, tested constrained-host floor is **64MB `/dev/shm`**. Both `fn serve` and boot smoke inherit this lifecycle default; an explicit later PostgreSQL `-c shared_memory_type=…` flag remains an operator override.
|
||||||
|
|
||||||
## Task-ID allocator authority and compatibility
|
## Task-ID allocator authority and compatibility
|
||||||
|
|
||||||
- `distributed_task_id_state` is the authoritative local task-ID allocator state. `nextSequence` is the active high-water mark used for local ID reservations.
|
- `distributed_task_id_state` is the authoritative local task-ID allocator state. `nextSequence` is the active high-water mark used for local ID reservations.
|
||||||
|
|||||||
@@ -33,11 +33,15 @@ import {
|
|||||||
EmbeddedPostgresLifecycle,
|
EmbeddedPostgresLifecycle,
|
||||||
EmbeddedStartTimeoutError,
|
EmbeddedStartTimeoutError,
|
||||||
DEFAULT_START_TIMEOUT_MS,
|
DEFAULT_START_TIMEOUT_MS,
|
||||||
|
DEFAULT_EMBEDDED_POSTGRES_FLAGS,
|
||||||
isDataDirInitialized,
|
isDataDirInitialized,
|
||||||
isWindowsElevatedAdmin,
|
isWindowsElevatedAdmin,
|
||||||
normalizeMacosEmbeddedPostgresDylibSymlinks,
|
normalizeMacosEmbeddedPostgresDylibSymlinks,
|
||||||
readPortFromPostmasterPid,
|
readPortFromPostmasterPid,
|
||||||
__setEmbeddedPostgresCtorForTests,
|
__setEmbeddedPostgresCtorForTests,
|
||||||
|
__setWindowsElevatedAdminForTests,
|
||||||
|
__setWindowsEmbeddedPostgresNativeRootForTests,
|
||||||
|
__setWindowsLauncherForTests,
|
||||||
resolveElectronAsarUnpackedPath,
|
resolveElectronAsarUnpackedPath,
|
||||||
fingerprintEmbeddedPostgresNativeRoot,
|
fingerprintEmbeddedPostgresNativeRoot,
|
||||||
buildEmbeddedPostgresMaterializationMarker,
|
buildEmbeddedPostgresMaterializationMarker,
|
||||||
@@ -60,6 +64,9 @@ const tracked: Array<{
|
|||||||
|
|
||||||
afterEach(async () => {
|
afterEach(async () => {
|
||||||
__setEmbeddedPostgresCtorForTests(null);
|
__setEmbeddedPostgresCtorForTests(null);
|
||||||
|
__setWindowsElevatedAdminForTests(null);
|
||||||
|
__setWindowsEmbeddedPostgresNativeRootForTests(null);
|
||||||
|
__setWindowsLauncherForTests(null);
|
||||||
vi.useRealTimers();
|
vi.useRealTimers();
|
||||||
while (tracked.length > 0) {
|
while (tracked.length > 0) {
|
||||||
const { lifecycle, dataDir } = tracked.pop()!;
|
const { lifecycle, dataDir } = tracked.pop()!;
|
||||||
@@ -1094,6 +1101,87 @@ describe("embedded-lifecycle: readPortFromPostmasterPid (P1 code-review fix)", (
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
/*
|
||||||
|
* FNXC:PostgresEmbedded 2026-07-16-12:45:
|
||||||
|
* Assert the shared-memory floor at the lifecycle boundary, which is used by
|
||||||
|
* both startup-factory boot and direct lifecycle callers. Mock starts reject
|
||||||
|
* before database work so these tests stay deterministic and process-free.
|
||||||
|
*/
|
||||||
|
describe("embedded-lifecycle: shared-memory-safe postgres flags", () => {
|
||||||
|
const sentinel = new Error("mock postgres start complete");
|
||||||
|
|
||||||
|
function installCtorRecorder(records: Record<string, unknown>[]): void {
|
||||||
|
class RecordingEmbeddedPostgres {
|
||||||
|
constructor(options: Record<string, unknown>) {
|
||||||
|
records.push(options);
|
||||||
|
}
|
||||||
|
initialise = vi.fn(async () => {});
|
||||||
|
async start() {
|
||||||
|
throw sentinel;
|
||||||
|
}
|
||||||
|
stop = vi.fn(async () => {});
|
||||||
|
}
|
||||||
|
__setEmbeddedPostgresCtorForTests(RecordingEmbeddedPostgres as never);
|
||||||
|
}
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
["omitted", undefined, [...DEFAULT_EMBEDDED_POSTGRES_FLAGS]],
|
||||||
|
["empty", [], [...DEFAULT_EMBEDDED_POSTGRES_FLAGS]],
|
||||||
|
[
|
||||||
|
"caller override after the default",
|
||||||
|
["-c", "shared_memory_type=sysv"],
|
||||||
|
[...DEFAULT_EMBEDDED_POSTGRES_FLAGS, "-c", "shared_memory_type=sysv"],
|
||||||
|
],
|
||||||
|
])("passes %s flags to the normal embedded-postgres constructor", async (_state, postgresFlags, expected) => {
|
||||||
|
const dataDir = makeDataDir();
|
||||||
|
const records: Record<string, unknown>[] = [];
|
||||||
|
installCtorRecorder(records);
|
||||||
|
try {
|
||||||
|
writeFileSync(join(dataDir, "PG_VERSION"), "15\n");
|
||||||
|
const lifecycle = new EmbeddedPostgresLifecycle({ ...baseOptions(dataDir), postgresFlags });
|
||||||
|
|
||||||
|
await expect(lifecycle.start()).rejects.toBe(sentinel);
|
||||||
|
expect(records).toHaveLength(1);
|
||||||
|
expect(records[0]?.postgresFlags).toEqual(expected);
|
||||||
|
} finally {
|
||||||
|
rmSync(dataDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("passes the ordered defaults and caller override through the elevated Windows launcher", async () => {
|
||||||
|
const dataDir = makeDataDir();
|
||||||
|
const records: Record<string, unknown>[] = [];
|
||||||
|
const launcherSentinel = new Error("mock Windows launcher reached");
|
||||||
|
let launcherOptions: Record<string, unknown> | undefined;
|
||||||
|
installCtorRecorder(records);
|
||||||
|
__setWindowsElevatedAdminForTests(true);
|
||||||
|
__setWindowsEmbeddedPostgresNativeRootForTests("/test/embedded-postgres/native");
|
||||||
|
__setWindowsLauncherForTests(async (options) => {
|
||||||
|
launcherOptions = options;
|
||||||
|
throw launcherSentinel;
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Reuse skips real initdb; the sentinel rejects before ensureDatabase.
|
||||||
|
writeFileSync(join(dataDir, "PG_VERSION"), "15\n");
|
||||||
|
const lifecycle = new EmbeddedPostgresLifecycle({
|
||||||
|
...baseOptions(dataDir),
|
||||||
|
postgresFlags: ["-c", "shared_memory_type=sysv"],
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(lifecycle.start()).rejects.toBe(launcherSentinel);
|
||||||
|
expect(records).toHaveLength(1);
|
||||||
|
expect(launcherOptions?.postgresFlags).toEqual([
|
||||||
|
...DEFAULT_EMBEDDED_POSTGRES_FLAGS,
|
||||||
|
"-c",
|
||||||
|
"shared_memory_type=sysv",
|
||||||
|
]);
|
||||||
|
} finally {
|
||||||
|
rmSync(dataDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("embedded-lifecycle: signal re-raise (P1 #23)", () => {
|
describe("embedded-lifecycle: signal re-raise (P1 #23)", () => {
|
||||||
it("boundShutdown re-raises real signals via process.kill (unit, no process)", async () => {
|
it("boundShutdown re-raises real signals via process.kill (unit, no process)", async () => {
|
||||||
// Verify the signal re-raise logic without a real cluster: construct a
|
// Verify the signal re-raise logic without a real cluster: construct a
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { DEFAULT_EMBEDDED_POSTGRES_FLAGS } from "../../postgres/embedded-lifecycle.js";
|
||||||
|
import { sanitizePostgresFlags } from "../../postgres/embedded-windows-admin.js";
|
||||||
|
|
||||||
|
/*
|
||||||
|
* FNXC:PostgresEmbedded 2026-07-16-12:45:
|
||||||
|
* The constrained-host shared-memory default must retain its exact `-c` form
|
||||||
|
* through the Windows cmd.exe launcher sanitizer. This is pure validation
|
||||||
|
* coverage; it does not require an elevated process or a Windows binary.
|
||||||
|
*/
|
||||||
|
describe("sanitizePostgresFlags", () => {
|
||||||
|
it("preserves the shared-memory default and a caller override unchanged", () => {
|
||||||
|
const flags = [...DEFAULT_EMBEDDED_POSTGRES_FLAGS, "-c", "shared_memory_type=sysv"];
|
||||||
|
|
||||||
|
expect(sanitizePostgresFlags(flags)).toEqual(flags);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -71,6 +71,7 @@ import {
|
|||||||
isWindowsElevatedAdmin,
|
isWindowsElevatedAdmin,
|
||||||
startServerAsNonAdminUser,
|
startServerAsNonAdminUser,
|
||||||
type NonAdminServerHandle,
|
type NonAdminServerHandle,
|
||||||
|
type NonAdminStartOptions,
|
||||||
} from "./embedded-windows-admin.js";
|
} from "./embedded-windows-admin.js";
|
||||||
// FNXC:WindowsDesktopPackaging 2026-07-14-22:53:
|
// FNXC:WindowsDesktopPackaging 2026-07-14-22:53:
|
||||||
// Static import so tsup/esbuild bundles postgres.js into packages/cli/dist/bin.js.
|
// Static import so tsup/esbuild bundles postgres.js into packages/cli/dist/bin.js.
|
||||||
@@ -469,6 +470,33 @@ export function __setEmbeddedPostgresCtorForTests(ctor: EmbeddedPostgresCtor | n
|
|||||||
embeddedPostgresCtorIsTestOverride = ctor !== null;
|
embeddedPostgresCtorIsTestOverride = ctor !== null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* FNXC:PostgresEmbedded 2026-07-16-12:45:
|
||||||
|
* Cross-platform tests must exercise the elevated Windows launcher without an
|
||||||
|
* elevated token, Windows binaries, or a running database. These narrow seams
|
||||||
|
* replace only the branch dependencies during a test; production always calls
|
||||||
|
* the imported implementations.
|
||||||
|
*/
|
||||||
|
let windowsElevatedAdminForTests: boolean | null = null;
|
||||||
|
let windowsNativeRootForTests: string | null = null;
|
||||||
|
let windowsLauncherForTests:
|
||||||
|
| ((opts: NonAdminStartOptions) => Promise<NonAdminServerHandle>)
|
||||||
|
| null = null;
|
||||||
|
|
||||||
|
export function __setWindowsElevatedAdminForTests(value: boolean | null): void {
|
||||||
|
windowsElevatedAdminForTests = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function __setWindowsEmbeddedPostgresNativeRootForTests(value: string | null): void {
|
||||||
|
windowsNativeRootForTests = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function __setWindowsLauncherForTests(
|
||||||
|
launcher: ((opts: NonAdminStartOptions) => Promise<NonAdminServerHandle>) | null,
|
||||||
|
): void {
|
||||||
|
windowsLauncherForTests = launcher;
|
||||||
|
}
|
||||||
|
|
||||||
function getEmbeddedPostgresCtor(): EmbeddedPostgresCtor {
|
function getEmbeddedPostgresCtor(): EmbeddedPostgresCtor {
|
||||||
if (embeddedPostgresCtorCache) return embeddedPostgresCtorCache;
|
if (embeddedPostgresCtorCache) return embeddedPostgresCtorCache;
|
||||||
// FNXC:DesktopEmbeddedPostgres 2026-07-14-18:30:
|
// FNXC:DesktopEmbeddedPostgres 2026-07-14-18:30:
|
||||||
@@ -490,6 +518,17 @@ export const DEFAULT_EMBEDDED_PASSWORD = "password";
|
|||||||
/** Default application database name created/ensured on the embedded cluster. */
|
/** Default application database name created/ensured on the embedded cluster. */
|
||||||
export const DEFAULT_EMBEDDED_DATABASE = "fusion";
|
export const DEFAULT_EMBEDDED_DATABASE = "fusion";
|
||||||
|
|
||||||
|
/*
|
||||||
|
* FNXC:PostgresEmbedded 2026-07-16-12:45:
|
||||||
|
* Embedded PostgreSQL 15's primary postmaster allocation used SysV shmget and
|
||||||
|
* failed with `could not create shared memory segment: No space left on device`
|
||||||
|
* when host SHMMNI/SHMALL was constrained. Use mmap-backed primary shared memory
|
||||||
|
* so the zero-config cluster has been boot-smoke tested with a 64MB /dev/shm
|
||||||
|
* lower bound. Defaults precede caller flags because PostgreSQL applies repeated
|
||||||
|
* `-c key=value` settings last-wins, preserving an operator's explicit override.
|
||||||
|
*/
|
||||||
|
export const DEFAULT_EMBEDDED_POSTGRES_FLAGS = ["-c", "shared_memory_type=mmap"] as const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* FNXC:PostgresEmbedded 2026-06-24-09:05:
|
* FNXC:PostgresEmbedded 2026-06-24-09:05:
|
||||||
* Default data directory location for the embedded cluster. Mirrors the
|
* Default data directory location for the embedded cluster. Mirrors the
|
||||||
@@ -914,7 +953,7 @@ export class EmbeddedPostgresLifecycle {
|
|||||||
user: opts.user ?? DEFAULT_EMBEDDED_USER,
|
user: opts.user ?? DEFAULT_EMBEDDED_USER,
|
||||||
password: opts.password ?? DEFAULT_EMBEDDED_PASSWORD,
|
password: opts.password ?? DEFAULT_EMBEDDED_PASSWORD,
|
||||||
initdbFlags: opts.initdbFlags ?? [],
|
initdbFlags: opts.initdbFlags ?? [],
|
||||||
postgresFlags: opts.postgresFlags ?? [],
|
postgresFlags: [...DEFAULT_EMBEDDED_POSTGRES_FLAGS, ...(opts.postgresFlags ?? [])],
|
||||||
startTimeoutMs: opts.startTimeoutMs ?? DEFAULT_START_TIMEOUT_MS,
|
startTimeoutMs: opts.startTimeoutMs ?? DEFAULT_START_TIMEOUT_MS,
|
||||||
onLog: opts.onLog ?? ((msg: string) => log.log(msg)),
|
onLog: opts.onLog ?? ((msg: string) => log.log(msg)),
|
||||||
onError:
|
onError:
|
||||||
@@ -1128,8 +1167,11 @@ export class EmbeddedPostgresLifecycle {
|
|||||||
// Skip the real non-admin path when tests inject a mock ctor so delayed
|
// Skip the real non-admin path when tests inject a mock ctor so delayed
|
||||||
// start/cancellation coverage exercises pg.start() even on elevated CI.
|
// start/cancellation coverage exercises pg.start() even on elevated CI.
|
||||||
try {
|
try {
|
||||||
if (isWindowsElevatedAdmin() && !embeddedPostgresCtorIsTestOverride) {
|
const isElevatedWindows = windowsElevatedAdminForTests ?? isWindowsElevatedAdmin();
|
||||||
const nativeRoot = resolveWindowsEmbeddedPostgresNativeRoot();
|
// A launcher seam intentionally coexists with the ctor seam so this branch
|
||||||
|
// can be covered off Windows; without that seam, ctor mocks retain normal-path behavior.
|
||||||
|
if (isElevatedWindows && (!embeddedPostgresCtorIsTestOverride || windowsLauncherForTests)) {
|
||||||
|
const nativeRoot = windowsNativeRootForTests ?? resolveWindowsEmbeddedPostgresNativeRoot();
|
||||||
if (!nativeRoot) {
|
if (!nativeRoot) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
"embedded postgres: the process is running elevated on Windows, where " +
|
"embedded postgres: the process is running elevated on Windows, where " +
|
||||||
@@ -1139,7 +1181,7 @@ export class EmbeddedPostgresLifecycle {
|
|||||||
"non-elevated, or ensure the embedded-postgres platform package is installed.",
|
"non-elevated, or ensure the embedded-postgres platform package is installed.",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
this.nonAdminHandle = await startServerAsNonAdminUser({
|
this.nonAdminHandle = await (windowsLauncherForTests ?? startServerAsNonAdminUser)({
|
||||||
nativeRoot,
|
nativeRoot,
|
||||||
dataDir: this.options.dataDir,
|
dataDir: this.options.dataDir,
|
||||||
port,
|
port,
|
||||||
|
|||||||
@@ -281,7 +281,7 @@ function readTail(file: string, max: number): string {
|
|||||||
* Reject postgresFlags that would break cmd.exe quoting or enable injection
|
* Reject postgresFlags that would break cmd.exe quoting or enable injection
|
||||||
* when embedded into launch.bat (review: arbitrary flags with % " & | etc.).
|
* when embedded into launch.bat (review: arbitrary flags with % " & | etc.).
|
||||||
*/
|
*/
|
||||||
function sanitizePostgresFlags(flags: readonly string[]): string[] {
|
export function sanitizePostgresFlags(flags: readonly string[]): string[] {
|
||||||
const safe: string[] = [];
|
const safe: string[] = [];
|
||||||
for (const flag of flags) {
|
for (const flag of flags) {
|
||||||
if (typeof flag !== "string" || flag.length === 0) {
|
if (typeof flag !== "string" || flag.length === 0) {
|
||||||
|
|||||||
Reference in New Issue
Block a user