FN-8090: use mmap shared memory for embedded PostgreSQL
Enable constrained-host embedded PostgreSQL startup without SysV shared-memory exhaustion. - Default embedded lifecycle flags to mmap-backed shared memory while preserving caller overrides - Cover normal and elevated Windows launch paths with deterministic flag propagation tests - Document the 64MB /dev/shm support floor and add a patch changeset Files changed: .changeset/fn-8090-embedded-pg-shm.md | 7 ++ docs/postgres-migration-review-2026-07-14.md | 4 + docs/storage.md | 5 ++ .../__tests__/postgres/embedded-lifecycle.test.ts | 88 ++++++++++++++++++++++ .../postgres/embedded-windows-admin.test.ts | 17 +++++ packages/core/src/postgres/embedded-lifecycle.ts | 50 +++++++++++- .../core/src/postgres/embedded-windows-admin.ts | 2 +- 7 files changed, 168 insertions(+), 5 deletions(-) Fusion-Task-Id: FN-8090 Fusion-Task-Lineage: ac175843-69ba-4c9c-9692-aff095fc351f Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
This commit is contained in:
7
.changeset/fn-8090-embedded-pg-shm.md
Normal file
7
.changeset/fn-8090-embedded-pg-shm.md
Normal file
@@ -0,0 +1,7 @@
|
||||
---
|
||||
"@runfusion/fusion": patch
|
||||
---
|
||||
|
||||
summary: Embedded PostgreSQL now boots on hosts with a 64MB /dev/shm.
|
||||
category: fix
|
||||
dev: Defaults the embedded lifecycle to mmap-backed primary shared memory while preserving later caller flag overrides.
|
||||
@@ -12,6 +12,10 @@ Fusion no longer supports SQLite as a live runtime backend. Startup selects eith
|
||||
|
||||
Legacy `fusion.db`, `archive.db`, and `fusion-central.db` files remain readable only at controlled identity-discovery and one-time migration/import seams. They are never a supported write target or runtime fallback. `.fusion/project.json` is the local project identity marker after cutover.
|
||||
|
||||
### Embedded PostgreSQL resource floor
|
||||
|
||||
The zero-config embedded PostgreSQL lifecycle uses mmap-backed primary shared memory so hosts with constrained SysV shared-memory IDs can boot without operator tuning. The supported, tested constrained-host floor is **64MB `/dev/shm`**; `fn serve` and the built boot smoke inherit this default. Operators can still provide a later PostgreSQL `-c shared_memory_type=…` flag when a deployment needs an explicit override.
|
||||
|
||||
## PostgreSQL-authoritative inventory
|
||||
|
||||
| Surface | PostgreSQL authority |
|
||||
|
||||
@@ -4,6 +4,11 @@
|
||||
|
||||
See the [2026-07-14 PostgreSQL runtime cutover review](./postgres-migration-review-2026-07-14.md) for the audited authority inventory, exact authorized legacy readers, and deployment/rollback checklist.
|
||||
|
||||
## Embedded PostgreSQL startup resources
|
||||
|
||||
- The zero-config embedded PostgreSQL lifecycle uses mmap-backed primary shared memory to avoid exhausted SysV shared-memory IDs on constrained hosts.
|
||||
- The supported, tested constrained-host floor is **64MB `/dev/shm`**. Both `fn serve` and boot smoke inherit this lifecycle default; an explicit later PostgreSQL `-c shared_memory_type=…` flag remains an operator override.
|
||||
|
||||
## Task-ID allocator authority and compatibility
|
||||
|
||||
- `distributed_task_id_state` is the authoritative local task-ID allocator state. `nextSequence` is the active high-water mark used for local ID reservations.
|
||||
|
||||
@@ -33,11 +33,15 @@ import {
|
||||
EmbeddedPostgresLifecycle,
|
||||
EmbeddedStartTimeoutError,
|
||||
DEFAULT_START_TIMEOUT_MS,
|
||||
DEFAULT_EMBEDDED_POSTGRES_FLAGS,
|
||||
isDataDirInitialized,
|
||||
isWindowsElevatedAdmin,
|
||||
normalizeMacosEmbeddedPostgresDylibSymlinks,
|
||||
readPortFromPostmasterPid,
|
||||
__setEmbeddedPostgresCtorForTests,
|
||||
__setWindowsElevatedAdminForTests,
|
||||
__setWindowsEmbeddedPostgresNativeRootForTests,
|
||||
__setWindowsLauncherForTests,
|
||||
resolveElectronAsarUnpackedPath,
|
||||
fingerprintEmbeddedPostgresNativeRoot,
|
||||
buildEmbeddedPostgresMaterializationMarker,
|
||||
@@ -60,6 +64,9 @@ const tracked: Array<{
|
||||
|
||||
afterEach(async () => {
|
||||
__setEmbeddedPostgresCtorForTests(null);
|
||||
__setWindowsElevatedAdminForTests(null);
|
||||
__setWindowsEmbeddedPostgresNativeRootForTests(null);
|
||||
__setWindowsLauncherForTests(null);
|
||||
vi.useRealTimers();
|
||||
while (tracked.length > 0) {
|
||||
const { lifecycle, dataDir } = tracked.pop()!;
|
||||
@@ -1094,6 +1101,87 @@ describe("embedded-lifecycle: readPortFromPostmasterPid (P1 code-review fix)", (
|
||||
});
|
||||
});
|
||||
|
||||
/*
|
||||
* FNXC:PostgresEmbedded 2026-07-16-12:45:
|
||||
* Assert the shared-memory floor at the lifecycle boundary, which is used by
|
||||
* both startup-factory boot and direct lifecycle callers. Mock starts reject
|
||||
* before database work so these tests stay deterministic and process-free.
|
||||
*/
|
||||
describe("embedded-lifecycle: shared-memory-safe postgres flags", () => {
|
||||
const sentinel = new Error("mock postgres start complete");
|
||||
|
||||
function installCtorRecorder(records: Record<string, unknown>[]): void {
|
||||
class RecordingEmbeddedPostgres {
|
||||
constructor(options: Record<string, unknown>) {
|
||||
records.push(options);
|
||||
}
|
||||
initialise = vi.fn(async () => {});
|
||||
async start() {
|
||||
throw sentinel;
|
||||
}
|
||||
stop = vi.fn(async () => {});
|
||||
}
|
||||
__setEmbeddedPostgresCtorForTests(RecordingEmbeddedPostgres as never);
|
||||
}
|
||||
|
||||
it.each([
|
||||
["omitted", undefined, [...DEFAULT_EMBEDDED_POSTGRES_FLAGS]],
|
||||
["empty", [], [...DEFAULT_EMBEDDED_POSTGRES_FLAGS]],
|
||||
[
|
||||
"caller override after the default",
|
||||
["-c", "shared_memory_type=sysv"],
|
||||
[...DEFAULT_EMBEDDED_POSTGRES_FLAGS, "-c", "shared_memory_type=sysv"],
|
||||
],
|
||||
])("passes %s flags to the normal embedded-postgres constructor", async (_state, postgresFlags, expected) => {
|
||||
const dataDir = makeDataDir();
|
||||
const records: Record<string, unknown>[] = [];
|
||||
installCtorRecorder(records);
|
||||
try {
|
||||
writeFileSync(join(dataDir, "PG_VERSION"), "15\n");
|
||||
const lifecycle = new EmbeddedPostgresLifecycle({ ...baseOptions(dataDir), postgresFlags });
|
||||
|
||||
await expect(lifecycle.start()).rejects.toBe(sentinel);
|
||||
expect(records).toHaveLength(1);
|
||||
expect(records[0]?.postgresFlags).toEqual(expected);
|
||||
} finally {
|
||||
rmSync(dataDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("passes the ordered defaults and caller override through the elevated Windows launcher", async () => {
|
||||
const dataDir = makeDataDir();
|
||||
const records: Record<string, unknown>[] = [];
|
||||
const launcherSentinel = new Error("mock Windows launcher reached");
|
||||
let launcherOptions: Record<string, unknown> | undefined;
|
||||
installCtorRecorder(records);
|
||||
__setWindowsElevatedAdminForTests(true);
|
||||
__setWindowsEmbeddedPostgresNativeRootForTests("/test/embedded-postgres/native");
|
||||
__setWindowsLauncherForTests(async (options) => {
|
||||
launcherOptions = options;
|
||||
throw launcherSentinel;
|
||||
});
|
||||
|
||||
try {
|
||||
// Reuse skips real initdb; the sentinel rejects before ensureDatabase.
|
||||
writeFileSync(join(dataDir, "PG_VERSION"), "15\n");
|
||||
const lifecycle = new EmbeddedPostgresLifecycle({
|
||||
...baseOptions(dataDir),
|
||||
postgresFlags: ["-c", "shared_memory_type=sysv"],
|
||||
});
|
||||
|
||||
await expect(lifecycle.start()).rejects.toBe(launcherSentinel);
|
||||
expect(records).toHaveLength(1);
|
||||
expect(launcherOptions?.postgresFlags).toEqual([
|
||||
...DEFAULT_EMBEDDED_POSTGRES_FLAGS,
|
||||
"-c",
|
||||
"shared_memory_type=sysv",
|
||||
]);
|
||||
} finally {
|
||||
rmSync(dataDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("embedded-lifecycle: signal re-raise (P1 #23)", () => {
|
||||
it("boundShutdown re-raises real signals via process.kill (unit, no process)", async () => {
|
||||
// Verify the signal re-raise logic without a real cluster: construct a
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { DEFAULT_EMBEDDED_POSTGRES_FLAGS } from "../../postgres/embedded-lifecycle.js";
|
||||
import { sanitizePostgresFlags } from "../../postgres/embedded-windows-admin.js";
|
||||
|
||||
/*
|
||||
* FNXC:PostgresEmbedded 2026-07-16-12:45:
|
||||
* The constrained-host shared-memory default must retain its exact `-c` form
|
||||
* through the Windows cmd.exe launcher sanitizer. This is pure validation
|
||||
* coverage; it does not require an elevated process or a Windows binary.
|
||||
*/
|
||||
describe("sanitizePostgresFlags", () => {
|
||||
it("preserves the shared-memory default and a caller override unchanged", () => {
|
||||
const flags = [...DEFAULT_EMBEDDED_POSTGRES_FLAGS, "-c", "shared_memory_type=sysv"];
|
||||
|
||||
expect(sanitizePostgresFlags(flags)).toEqual(flags);
|
||||
});
|
||||
});
|
||||
@@ -71,6 +71,7 @@ import {
|
||||
isWindowsElevatedAdmin,
|
||||
startServerAsNonAdminUser,
|
||||
type NonAdminServerHandle,
|
||||
type NonAdminStartOptions,
|
||||
} from "./embedded-windows-admin.js";
|
||||
// FNXC:WindowsDesktopPackaging 2026-07-14-22:53:
|
||||
// Static import so tsup/esbuild bundles postgres.js into packages/cli/dist/bin.js.
|
||||
@@ -469,6 +470,33 @@ export function __setEmbeddedPostgresCtorForTests(ctor: EmbeddedPostgresCtor | n
|
||||
embeddedPostgresCtorIsTestOverride = ctor !== null;
|
||||
}
|
||||
|
||||
/*
|
||||
* FNXC:PostgresEmbedded 2026-07-16-12:45:
|
||||
* Cross-platform tests must exercise the elevated Windows launcher without an
|
||||
* elevated token, Windows binaries, or a running database. These narrow seams
|
||||
* replace only the branch dependencies during a test; production always calls
|
||||
* the imported implementations.
|
||||
*/
|
||||
let windowsElevatedAdminForTests: boolean | null = null;
|
||||
let windowsNativeRootForTests: string | null = null;
|
||||
let windowsLauncherForTests:
|
||||
| ((opts: NonAdminStartOptions) => Promise<NonAdminServerHandle>)
|
||||
| null = null;
|
||||
|
||||
export function __setWindowsElevatedAdminForTests(value: boolean | null): void {
|
||||
windowsElevatedAdminForTests = value;
|
||||
}
|
||||
|
||||
export function __setWindowsEmbeddedPostgresNativeRootForTests(value: string | null): void {
|
||||
windowsNativeRootForTests = value;
|
||||
}
|
||||
|
||||
export function __setWindowsLauncherForTests(
|
||||
launcher: ((opts: NonAdminStartOptions) => Promise<NonAdminServerHandle>) | null,
|
||||
): void {
|
||||
windowsLauncherForTests = launcher;
|
||||
}
|
||||
|
||||
function getEmbeddedPostgresCtor(): EmbeddedPostgresCtor {
|
||||
if (embeddedPostgresCtorCache) return embeddedPostgresCtorCache;
|
||||
// FNXC:DesktopEmbeddedPostgres 2026-07-14-18:30:
|
||||
@@ -490,6 +518,17 @@ export const DEFAULT_EMBEDDED_PASSWORD = "password";
|
||||
/** Default application database name created/ensured on the embedded cluster. */
|
||||
export const DEFAULT_EMBEDDED_DATABASE = "fusion";
|
||||
|
||||
/*
|
||||
* FNXC:PostgresEmbedded 2026-07-16-12:45:
|
||||
* Embedded PostgreSQL 15's primary postmaster allocation used SysV shmget and
|
||||
* failed with `could not create shared memory segment: No space left on device`
|
||||
* when host SHMMNI/SHMALL was constrained. Use mmap-backed primary shared memory
|
||||
* so the zero-config cluster has been boot-smoke tested with a 64MB /dev/shm
|
||||
* lower bound. Defaults precede caller flags because PostgreSQL applies repeated
|
||||
* `-c key=value` settings last-wins, preserving an operator's explicit override.
|
||||
*/
|
||||
export const DEFAULT_EMBEDDED_POSTGRES_FLAGS = ["-c", "shared_memory_type=mmap"] as const;
|
||||
|
||||
/**
|
||||
* FNXC:PostgresEmbedded 2026-06-24-09:05:
|
||||
* Default data directory location for the embedded cluster. Mirrors the
|
||||
@@ -914,7 +953,7 @@ export class EmbeddedPostgresLifecycle {
|
||||
user: opts.user ?? DEFAULT_EMBEDDED_USER,
|
||||
password: opts.password ?? DEFAULT_EMBEDDED_PASSWORD,
|
||||
initdbFlags: opts.initdbFlags ?? [],
|
||||
postgresFlags: opts.postgresFlags ?? [],
|
||||
postgresFlags: [...DEFAULT_EMBEDDED_POSTGRES_FLAGS, ...(opts.postgresFlags ?? [])],
|
||||
startTimeoutMs: opts.startTimeoutMs ?? DEFAULT_START_TIMEOUT_MS,
|
||||
onLog: opts.onLog ?? ((msg: string) => log.log(msg)),
|
||||
onError:
|
||||
@@ -1128,8 +1167,11 @@ export class EmbeddedPostgresLifecycle {
|
||||
// Skip the real non-admin path when tests inject a mock ctor so delayed
|
||||
// start/cancellation coverage exercises pg.start() even on elevated CI.
|
||||
try {
|
||||
if (isWindowsElevatedAdmin() && !embeddedPostgresCtorIsTestOverride) {
|
||||
const nativeRoot = resolveWindowsEmbeddedPostgresNativeRoot();
|
||||
const isElevatedWindows = windowsElevatedAdminForTests ?? isWindowsElevatedAdmin();
|
||||
// A launcher seam intentionally coexists with the ctor seam so this branch
|
||||
// can be covered off Windows; without that seam, ctor mocks retain normal-path behavior.
|
||||
if (isElevatedWindows && (!embeddedPostgresCtorIsTestOverride || windowsLauncherForTests)) {
|
||||
const nativeRoot = windowsNativeRootForTests ?? resolveWindowsEmbeddedPostgresNativeRoot();
|
||||
if (!nativeRoot) {
|
||||
throw new Error(
|
||||
"embedded postgres: the process is running elevated on Windows, where " +
|
||||
@@ -1139,7 +1181,7 @@ export class EmbeddedPostgresLifecycle {
|
||||
"non-elevated, or ensure the embedded-postgres platform package is installed.",
|
||||
);
|
||||
}
|
||||
this.nonAdminHandle = await startServerAsNonAdminUser({
|
||||
this.nonAdminHandle = await (windowsLauncherForTests ?? startServerAsNonAdminUser)({
|
||||
nativeRoot,
|
||||
dataDir: this.options.dataDir,
|
||||
port,
|
||||
|
||||
@@ -281,7 +281,7 @@ function readTail(file: string, max: number): string {
|
||||
* Reject postgresFlags that would break cmd.exe quoting or enable injection
|
||||
* when embedded into launch.bat (review: arbitrary flags with % " & | etc.).
|
||||
*/
|
||||
function sanitizePostgresFlags(flags: readonly string[]): string[] {
|
||||
export function sanitizePostgresFlags(flags: readonly string[]): string[] {
|
||||
const safe: string[] = [];
|
||||
for (const flag of flags) {
|
||||
if (typeof flag !== "string" || flag.length === 0) {
|
||||
|
||||
Reference in New Issue
Block a user