From 95e011f890711b9b1d44b70806fad0331d10cf73 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Fri, 17 Jul 2026 23:18:40 -0700 Subject: [PATCH] fix(core): auto-repair empty non-UTF-8 embedded Postgres clusters on boot (#2286) Users whose embedded cluster was initdb'd with an OS-locale encoding by a pre-fix version now self-heal with zero manual steps: on the encoding-conversion schema failure the startup factory proves the cluster is non-UTF-8 AND empty (the baseline transaction never applied, so no schema or migrated data can exist) and that this process owns the postmaster, then deletes the data dir and reboots once with the UTF-8 initdb defaults. Joined instances and unproven states keep the manual re-init hint; one retry ever, so no loops. Verified on the elevated windows-latest runner: CI seeds a real WIN1252 cluster via initdb and proves a stock 'fn serve' auto-recovers it to a healthy /api/health (run 29633351848, all jobs green). Also caps the desktop-windows embedded-PG smoke at 30 min and adds a skip input. Co-Authored-By: Claude Fable 5 --- .../embedded-pg-encoding-auto-reinit.md | 7 + .github/workflows/desktop-windows.yml | 13 ++ .../workflows/verify-elevated-restricted.yml | 7 +- .../postgres/startup-factory.test.ts | 29 ++++ packages/core/src/postgres/startup-factory.ts | 109 ++++++++++++++- scripts/verify-windows-encoding-recovery.mjs | 127 ++++++++++++++++++ 6 files changed, 289 insertions(+), 3 deletions(-) create mode 100644 .changeset/embedded-pg-encoding-auto-reinit.md create mode 100644 scripts/verify-windows-encoding-recovery.mjs diff --git a/.changeset/embedded-pg-encoding-auto-reinit.md b/.changeset/embedded-pg-encoding-auto-reinit.md new file mode 100644 index 0000000000..83fb94b7e7 --- /dev/null +++ b/.changeset/embedded-pg-encoding-auto-reinit.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Fusion now auto-repairs embedded PostgreSQL clusters left in the non-UTF-8 encoding state by earlier versions. +category: fix +dev: "Issue #2286 follow-up: on the encoding-conversion schema failure, the startup factory proves the embedded cluster is non-UTF-8 AND empty (no tables in project/central/archive, no recorded migrations — guaranteed for affected installs since the baseline never applied) and that this process owns the postmaster, then deletes the data dir and re-boots once with the UTF-8 initdb defaults. Joined instances and any non-proven state keep the manual re-init hint." diff --git a/.github/workflows/desktop-windows.yml b/.github/workflows/desktop-windows.yml index 1ea053858e..3723e9fb30 100644 --- a/.github/workflows/desktop-windows.yml +++ b/.github/workflows/desktop-windows.yml @@ -2,6 +2,11 @@ name: Desktop Windows Build on: workflow_dispatch: + inputs: + skip_pg_smoke: + description: "Skip the embedded-PG smoke (already covered by verify-elevated-restricted.yml)" + type: boolean + default: false jobs: build-windows-exe: @@ -29,7 +34,13 @@ jobs: # Create the user and warm its profile once here, outside any test window; # the launcher resets the user's password before each run, but the warmed # profile persists, so every later launch is ~0.5s. + # FNXC:WindowsDesktopPackaging 2026-07-18-01:40: + # The smoke wedged a runner for 45+ min with Start-Process -Wait and no + # step timeout (baseline ~18 min). Cap it, and allow skipping it via + # dispatch input when the dedicated verify-elevated-restricted workflow + # already proves embedded PG on this ref — packaging does not depend on it. - name: Prewarm embedded-PG helper user profile + if: ${{ !inputs.skip_pg_smoke }} shell: pwsh run: | $user = "fusion-pg" @@ -53,6 +64,8 @@ jobs: # the normal embedded-postgres path — no in-launcher Start-Process # -Credential / staging / process-kill races. - name: Smoke embedded Postgres on Windows + if: ${{ !inputs.skip_pg_smoke }} + timeout-minutes: 30 shell: pwsh run: | $user = "fusion-pg" diff --git a/.github/workflows/verify-elevated-restricted.yml b/.github/workflows/verify-elevated-restricted.yml index 9f56b64755..e2f2e4e7b8 100644 --- a/.github/workflows/verify-elevated-restricted.yml +++ b/.github/workflows/verify-elevated-restricted.yml @@ -9,7 +9,7 @@ name: Verify Elevated Restricted-Token Postgres on: workflow_dispatch: push: - branches: [feature/win-elevated-no-user] + branches: [feature/win-elevated-no-user, fix/embedded-pg-encoding-auto-reinit] jobs: verify-elevated-restricted: @@ -75,6 +75,11 @@ jobs: - name: Boot smoke (fn --help + serve /api/health, elevated) run: pnpm smoke:boot + # Issue #2286 auto-recovery: seed a real non-UTF-8 cluster, then prove a + # full `fn serve` self-heals it to UTF-8 and reaches a healthy state. + - name: Encoding auto-recovery (seeded WIN1252 cluster) + run: node scripts/verify-windows-encoding-recovery.mjs + - name: Assert no fusion-pg account was created shell: pwsh run: | diff --git a/packages/core/src/__tests__/postgres/startup-factory.test.ts b/packages/core/src/__tests__/postgres/startup-factory.test.ts index 8694d08328..daf133278b 100644 --- a/packages/core/src/__tests__/postgres/startup-factory.test.ts +++ b/packages/core/src/__tests__/postgres/startup-factory.test.ts @@ -25,6 +25,7 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; import { createTaskStoreForBackend, + isEncodingConversionError, shouldUsePostgresBackend, isEmbeddedPgRequested, isEmbeddedPgOptedOut, @@ -196,3 +197,31 @@ describe("startup-factory: backend descriptor propagation", () => { ).rejects.toThrow(); }); }); + +/* +FNXC:PostgresEmbedded 2026-07-18-01:10: +Issue #2286 auto-recovery trigger. The classifier must catch PostgreSQL's +encoding-conversion failure raised when a non-UTF-8 cluster (WIN1252/WIN1254 +from a pre-fix initdb on a non-UTF-8 OS locale) receives the UTF-8 schema +SQL — and nothing else, so ordinary schema errors never delete a data dir. +*/ +describe("isEncodingConversionError (#2286 recovery trigger)", () => { + it("matches the encoding-conversion failure from a non-UTF-8 cluster", () => { + expect( + isEncodingConversionError( + 'character with byte sequence 0xe2 0x86 0x92 in encoding "UTF8" has no equivalent in encoding "WIN1254"', + ), + ).toBe(true); + expect( + isEncodingConversionError( + 'Failed query: CREATE TABLE ... params: caused by: character with byte sequence 0xe2 0x86 0x92 in encoding "UTF8" has no equivalent in encoding "WIN1252"', + ), + ).toBe(true); + }); + + it("does not match unrelated schema or connection errors", () => { + expect(isEncodingConversionError('syntax error at or near "CREATE"')).toBe(false); + expect(isEncodingConversionError("connection refused")).toBe(false); + expect(isEncodingConversionError('FATAL: invalid value for parameter "shared_memory_type"')).toBe(false); + }); +}); diff --git a/packages/core/src/postgres/startup-factory.ts b/packages/core/src/postgres/startup-factory.ts index 7355dcdaec..83de84227e 100644 --- a/packages/core/src/postgres/startup-factory.ts +++ b/packages/core/src/postgres/startup-factory.ts @@ -37,7 +37,7 @@ */ import { join, resolve } from "node:path"; -import { existsSync } from "node:fs"; +import { existsSync, rmSync } from "node:fs"; import { createHash } from "node:crypto"; import { sql } from "drizzle-orm"; import { isValidSqliteDatabaseFile } from "../sqlite-validation.js"; @@ -53,7 +53,8 @@ import { createConnectionSetFromUrl, type PostgresConnections, } from "./connection.js"; -import { applySchemaBaseline } from "./schema-applier.js"; +import { applySchemaBaseline, MIGRATION_BOOKKEEPING_TABLE } from "./schema-applier.js"; +import type { PostgresJsDatabase } from "drizzle-orm/postgres-js"; import { createAsyncDataLayer, type AsyncDataLayer } from "./data-layer.js"; import { invalidateEmbeddedRuntimeUrl, @@ -234,9 +235,91 @@ async function stopEmbeddedRuntime( * Callers retain ownership of the returned resources and may replace the * administrative pool with an RLS-bound runtime pool after migration. */ +/* +FNXC:PostgresEmbedded 2026-07-18-01:10: +Issue #2286 auto-recovery. An embedded cluster initdb'd by an earlier version +on a non-UTF-8 OS locale (WIN1252/WIN1254) rejects the UTF-8 schema SQL with +`character ... has no equivalent in encoding`. Such a cluster can NEVER have +completed a boot: the whole baseline runs in one transaction whose query +string fails encoding conversion before execution, so no schema was applied +and the SQLite auto-migration (which runs after schema boot) never moved any +data. Deleting the data dir and re-initdb'ing (now forced UTF-8) is therefore +lossless — but only when proven, so recovery requires ALL of: + 1. embedded mode (Fusion owns the data dir), + 2. the encoding-conversion error signature, + 3. the live cluster confirming a non-UTF-8 server_encoding, + 4. zero tables in project/central/archive and no recorded migration + versions (the never-successfully-booted state), + 5. this process owns the postmaster (never delete under a foreign owner). +One retry only; a second failure surfaces the manual re-init hint. +*/ +class NonUtf8EmbeddedClusterError extends Error { + constructor(readonly dataDir: string, override readonly cause: unknown) { + super(`embedded cluster at ${dataDir} has a non-UTF-8 encoding and is empty; auto re-initializing`); + } +} + +/** Matches PostgreSQL's encoding-conversion failure raised by a non-UTF-8 cluster. */ +export function isEncodingConversionError(chainText: string): boolean { + return /has no equivalent in encoding/i.test(chainText); +} + +/** + * Prove the cluster is in the recoverable state: non-UTF-8 server encoding AND + * no applied schema (no tables in Fusion's schemas, no migration versions). + * Any query failure means "not proven" — the caller keeps the original error. + */ +async function isEmptyNonUtf8Cluster(db: PostgresJsDatabase>): Promise { + const encodingRows = (await db.execute( + sql`SELECT current_setting('server_encoding') AS encoding`, + )) as unknown as Array<{ encoding: string }>; + const encoding = (encodingRows[0]?.encoding ?? "").toUpperCase(); + if (encoding === "UTF8" || encoding === "") return false; + + const tableRows = (await db.execute(sql` + SELECT count(*)::int AS tables + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname IN ('project', 'central', 'archive') AND c.relkind = 'r' + `)) as unknown as Array<{ tables: number }>; + if ((tableRows[0]?.tables ?? 1) !== 0) return false; + + const versionRows = (await db.execute(sql` + SELECT count(*)::int AS versions + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relname = ${MIGRATION_BOOKKEEPING_TABLE} AND c.relkind = 'r' + `)) as unknown as Array<{ versions: number }>; + if ((versionRows[0]?.versions ?? 0) !== 0) { + const applied = (await db.execute(sql.raw( + `SELECT count(*)::int AS applied FROM public.${MIGRATION_BOOKKEEPING_TABLE}`, + ))) as unknown as Array<{ applied: number }>; + if ((applied[0]?.applied ?? 1) !== 0) return false; + } + return true; +} + async function bootSchemaBackend( options: Pick, bypassProjectIsolation = false, +): Promise { + try { + return await bootSchemaBackendOnce(options, bypassProjectIsolation); + } catch (error) { + if (!(error instanceof NonUtf8EmbeddedClusterError)) throw error; + log.warn( + `startup-factory: embedded cluster at ${error.dataDir} was created with a non-UTF-8 OS-locale ` + + `encoding by an earlier version and never completed a boot (issue #2286). ` + + `Re-initializing it as UTF-8 and retrying once.`, + ); + rmSync(error.dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); + return await bootSchemaBackendOnce(options, bypassProjectIsolation); + } +} + +async function bootSchemaBackendOnce( + options: Pick, + bypassProjectIsolation = false, ): Promise { const env = options.env ?? process.env; const backend = options.backend ?? resolveBackend(env); @@ -252,10 +335,12 @@ async function bootSchemaBackend( let embeddedRuntimeUrl: string | null = null; let embeddedOwnsProcess = false; let resolvedBackend = backend; + let embeddedDataDir: string | null = null; if (backend.mode === "embedded") { const { EmbeddedPostgresLifecycle, defaultEmbeddedDataDir, DEFAULT_EMBEDDED_DATABASE } = await import("./embedded-lifecycle.js"); const dataDir = resolve(options.embeddedDataDir ?? defaultEmbeddedDataDir()); + embeddedDataDir = dataDir; log.log(`startup-factory: starting embedded PostgreSQL (data dir ${dataDir})`); embeddedLifecycle = new EmbeddedPostgresLifecycle({ dataDir, @@ -308,6 +393,23 @@ async function bootSchemaBackend( embeddedOwnsProcess, }; } catch (error) { + /* + FNXC:PostgresEmbedded 2026-07-18-01:10: + Classify the #2286 non-UTF-8-cluster state while the connection is still + open (the proof queries need it), then tear down before signalling the + retry wrapper. Never recover a joined instance (embeddedOwnsProcess false): + deleting a data dir under a foreign postmaster is not ours to do. + */ + let recoverable = false; + if ( + embeddedLifecycle !== null && + embeddedOwnsProcess && + connections !== undefined && + embeddedDataDir !== null && + isEncodingConversionError(error instanceof Error ? `${error.message} ${String(error.cause ?? "")}` : String(error)) + ) { + recoverable = await isEmptyNonUtf8Cluster(connections.migration).catch(() => false); + } await connections?.close().catch(() => undefined); await stopEmbeddedRuntime( embeddedLifecycle, @@ -315,6 +417,9 @@ async function bootSchemaBackend( embeddedRuntimeUrl, embeddedOwnsProcess, ).catch(() => undefined); + if (recoverable && embeddedDataDir !== null) { + throw new NonUtf8EmbeddedClusterError(embeddedDataDir, error); + } throw error; } } diff --git a/scripts/verify-windows-encoding-recovery.mjs b/scripts/verify-windows-encoding-recovery.mjs new file mode 100644 index 0000000000..a5399aeb8a --- /dev/null +++ b/scripts/verify-windows-encoding-recovery.mjs @@ -0,0 +1,127 @@ +// FNXC:PostgresEmbedded 2026-07-18-01:10: +// End-to-end proof of the issue #2286 auto-recovery. Simulates a machine hit +// by the old bug: creates a real embedded cluster with a non-UTF-8 server +// encoding (initdb --encoding=WIN1252, the runner's own OS-locale outcome +// pre-fix) under a throwaway HOME, then boots the full `fn serve` against it. +// The startup factory must detect the encoding-conversion failure, prove the +// cluster is empty, delete and re-initdb it as UTF-8, and reach a healthy +// /api/health — with no operator intervention. +import { spawn, spawnSync } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const repoRoot = resolve(scriptDir, ".."); +const cliBin = join(repoRoot, "packages", "cli", "bin.mjs"); +const PORT = 55497; +const HEALTH_TIMEOUT_MS = 240_000; + +function sh(cmd, args) { + const r = spawnSync(cmd, args, { encoding: "utf8" }); + return { status: r.status, out: `${r.stdout || ""}${r.stderr || ""}`.trim() }; +} + +const home = mkdtempSync(join(tmpdir(), "fusion-encoding-recovery-home-")); +const project = mkdtempSync(join(tmpdir(), "fusion-encoding-recovery-project-")); +const dataDir = join(home, ".fusion", "embedded-postgres", "default"); +mkdirSync(dirname(dataDir), { recursive: true }); + +// Build the bad cluster with the REAL lifecycle: caller initdb flags are +// appended after the UTF-8 defaults and initdb takes the last occurrence, so +// this recreates exactly what pre-fix initdb produced on a WIN1252 locale. +const { EmbeddedPostgresLifecycle } = await import( + pathToFileURL(join(repoRoot, "packages", "core", "dist", "postgres", "embedded-lifecycle.js")).href +); +const seed = new EmbeddedPostgresLifecycle({ + dataDir, + database: "fusion", + user: "postgres", + password: "password", + port: PORT, + initdbFlags: ["--encoding=WIN1252", "--locale=C"], + onLog: (m) => console.log(`[seed] ${m}`), + onError: (e) => console.error(`[seed:err] ${String(e)}`), +}); +await seed.start(); +await seed.stop(); +console.log(`recovery-verify: seeded non-UTF-8 cluster at ${dataDir}`); + +let output = ""; +const child = spawn( + process.execPath, + [cliBin, "serve", "--port", String(PORT), "--host", "127.0.0.1", "--paused"], + { + cwd: project, + env: { + ...process.env, + HOME: home, + USERPROFILE: home, + FUSION_SKIP_ONBOARDING: "1", + DATABASE_URL: undefined, + FUSION_NO_EMBEDDED_PG: undefined, + PORT: undefined, + }, + stdio: ["ignore", "pipe", "pipe"], + }, +); +child.stdout.on("data", (d) => (output += d)); +child.stderr.on("data", (d) => (output += d)); +let exited = false; +child.once("exit", () => (exited = true)); + +async function pollHealth() { + const deadline = Date.now() + HEALTH_TIMEOUT_MS; + while (Date.now() < deadline) { + if (exited) throw new Error(`serve exited before becoming healthy.\n${output.slice(-4000)}`); + try { + const controller = new AbortController(); + const abortTimer = setTimeout(() => controller.abort(), 2000); + const res = await fetch(`http://127.0.0.1:${PORT}/api/health`, { signal: controller.signal }); + clearTimeout(abortTimer); + if (res.ok) return; + } catch { + // not up yet + } + await new Promise((r) => setTimeout(r, 500)); + } + throw new Error(`serve did not become healthy in ${HEALTH_TIMEOUT_MS}ms.\n${output.slice(-4000)}`); +} + +let failed = false; +try { + await pollHealth(); + console.log(`recovery-verify: /api/health OK on :${PORT} after auto-recovery`); + if (!/Re-initializing it as UTF-8 and retrying once/.test(output)) { + throw new Error( + `serve became healthy but the auto-recovery log line is missing — the seed cluster may not have triggered the #2286 path.\n${output.slice(-4000)}`, + ); + } + console.log("recovery-verify: auto-recovery log line confirmed"); +} catch (err) { + console.error(`recovery-verify: FAIL — ${err instanceof Error ? err.message : String(err)}`); + failed = true; +} finally { + try { + child.kill("SIGKILL"); + } catch { + // already gone + } + try { + const pid = parseInt(readFileSync(join(dataDir, "postmaster.pid"), "utf8").split("\n")[0], 10); + if (Number.isFinite(pid) && pid > 0) sh("taskkill", ["/pid", String(pid), "/f", "/t"]); + } catch { + // no postmaster left + } + for (const dir of [home, project]) { + try { + rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); + } catch (cleanupErr) { + console.error(`recovery-verify: cleanup warning: ${String(cleanupErr)}`); + } + } +} + +if (failed) process.exit(1); +console.log("recovery-verify: PASS — non-UTF-8 cluster auto-recovered to a healthy boot");