diff --git a/.changeset/embedded-pg-encoding-auto-reinit.md b/.changeset/embedded-pg-encoding-auto-reinit.md new file mode 100644 index 0000000000..83fb94b7e7 --- /dev/null +++ b/.changeset/embedded-pg-encoding-auto-reinit.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Fusion now auto-repairs embedded PostgreSQL clusters left in the non-UTF-8 encoding state by earlier versions. +category: fix +dev: "Issue #2286 follow-up: on the encoding-conversion schema failure, the startup factory proves the embedded cluster is non-UTF-8 AND empty (no tables in project/central/archive, no recorded migrations — guaranteed for affected installs since the baseline never applied) and that this process owns the postmaster, then deletes the data dir and re-boots once with the UTF-8 initdb defaults. Joined instances and any non-proven state keep the manual re-init hint." diff --git a/.github/workflows/desktop-windows.yml b/.github/workflows/desktop-windows.yml index 1ea053858e..3723e9fb30 100644 --- a/.github/workflows/desktop-windows.yml +++ b/.github/workflows/desktop-windows.yml @@ -2,6 +2,11 @@ name: Desktop Windows Build on: workflow_dispatch: + inputs: + skip_pg_smoke: + description: "Skip the embedded-PG smoke (already covered by verify-elevated-restricted.yml)" + type: boolean + default: false jobs: build-windows-exe: @@ -29,7 +34,13 @@ jobs: # Create the user and warm its profile once here, outside any test window; # the launcher resets the user's password before each run, but the warmed # profile persists, so every later launch is ~0.5s. + # FNXC:WindowsDesktopPackaging 2026-07-18-01:40: + # The smoke wedged a runner for 45+ min with Start-Process -Wait and no + # step timeout (baseline ~18 min). Cap it, and allow skipping it via + # dispatch input when the dedicated verify-elevated-restricted workflow + # already proves embedded PG on this ref — packaging does not depend on it. - name: Prewarm embedded-PG helper user profile + if: ${{ !inputs.skip_pg_smoke }} shell: pwsh run: | $user = "fusion-pg" @@ -53,6 +64,8 @@ jobs: # the normal embedded-postgres path — no in-launcher Start-Process # -Credential / staging / process-kill races. - name: Smoke embedded Postgres on Windows + if: ${{ !inputs.skip_pg_smoke }} + timeout-minutes: 30 shell: pwsh run: | $user = "fusion-pg" diff --git a/.github/workflows/verify-elevated-restricted.yml b/.github/workflows/verify-elevated-restricted.yml index 9f56b64755..e2f2e4e7b8 100644 --- a/.github/workflows/verify-elevated-restricted.yml +++ b/.github/workflows/verify-elevated-restricted.yml @@ -9,7 +9,7 @@ name: Verify Elevated Restricted-Token Postgres on: workflow_dispatch: push: - branches: [feature/win-elevated-no-user] + branches: [feature/win-elevated-no-user, fix/embedded-pg-encoding-auto-reinit] jobs: verify-elevated-restricted: @@ -75,6 +75,11 @@ jobs: - name: Boot smoke (fn --help + serve /api/health, elevated) run: pnpm smoke:boot + # Issue #2286 auto-recovery: seed a real non-UTF-8 cluster, then prove a + # full `fn serve` self-heals it to UTF-8 and reaches a healthy state. + - name: Encoding auto-recovery (seeded WIN1252 cluster) + run: node scripts/verify-windows-encoding-recovery.mjs + - name: Assert no fusion-pg account was created shell: pwsh run: | diff --git a/packages/core/src/__tests__/postgres/startup-factory.test.ts b/packages/core/src/__tests__/postgres/startup-factory.test.ts index 8694d08328..daf133278b 100644 --- a/packages/core/src/__tests__/postgres/startup-factory.test.ts +++ b/packages/core/src/__tests__/postgres/startup-factory.test.ts @@ -25,6 +25,7 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; import { createTaskStoreForBackend, + isEncodingConversionError, shouldUsePostgresBackend, isEmbeddedPgRequested, isEmbeddedPgOptedOut, @@ -196,3 +197,31 @@ describe("startup-factory: backend descriptor propagation", () => { ).rejects.toThrow(); }); }); + +/* +FNXC:PostgresEmbedded 2026-07-18-01:10: +Issue #2286 auto-recovery trigger. The classifier must catch PostgreSQL's +encoding-conversion failure raised when a non-UTF-8 cluster (WIN1252/WIN1254 +from a pre-fix initdb on a non-UTF-8 OS locale) receives the UTF-8 schema +SQL — and nothing else, so ordinary schema errors never delete a data dir. +*/ +describe("isEncodingConversionError (#2286 recovery trigger)", () => { + it("matches the encoding-conversion failure from a non-UTF-8 cluster", () => { + expect( + isEncodingConversionError( + 'character with byte sequence 0xe2 0x86 0x92 in encoding "UTF8" has no equivalent in encoding "WIN1254"', + ), + ).toBe(true); + expect( + isEncodingConversionError( + 'Failed query: CREATE TABLE ... params: caused by: character with byte sequence 0xe2 0x86 0x92 in encoding "UTF8" has no equivalent in encoding "WIN1252"', + ), + ).toBe(true); + }); + + it("does not match unrelated schema or connection errors", () => { + expect(isEncodingConversionError('syntax error at or near "CREATE"')).toBe(false); + expect(isEncodingConversionError("connection refused")).toBe(false); + expect(isEncodingConversionError('FATAL: invalid value for parameter "shared_memory_type"')).toBe(false); + }); +}); diff --git a/packages/core/src/postgres/startup-factory.ts b/packages/core/src/postgres/startup-factory.ts index 7355dcdaec..83de84227e 100644 --- a/packages/core/src/postgres/startup-factory.ts +++ b/packages/core/src/postgres/startup-factory.ts @@ -37,7 +37,7 @@ */ import { join, resolve } from "node:path"; -import { existsSync } from "node:fs"; +import { existsSync, rmSync } from "node:fs"; import { createHash } from "node:crypto"; import { sql } from "drizzle-orm"; import { isValidSqliteDatabaseFile } from "../sqlite-validation.js"; @@ -53,7 +53,8 @@ import { createConnectionSetFromUrl, type PostgresConnections, } from "./connection.js"; -import { applySchemaBaseline } from "./schema-applier.js"; +import { applySchemaBaseline, MIGRATION_BOOKKEEPING_TABLE } from "./schema-applier.js"; +import type { PostgresJsDatabase } from "drizzle-orm/postgres-js"; import { createAsyncDataLayer, type AsyncDataLayer } from "./data-layer.js"; import { invalidateEmbeddedRuntimeUrl, @@ -234,9 +235,91 @@ async function stopEmbeddedRuntime( * Callers retain ownership of the returned resources and may replace the * administrative pool with an RLS-bound runtime pool after migration. */ +/* +FNXC:PostgresEmbedded 2026-07-18-01:10: +Issue #2286 auto-recovery. An embedded cluster initdb'd by an earlier version +on a non-UTF-8 OS locale (WIN1252/WIN1254) rejects the UTF-8 schema SQL with +`character ... has no equivalent in encoding`. Such a cluster can NEVER have +completed a boot: the whole baseline runs in one transaction whose query +string fails encoding conversion before execution, so no schema was applied +and the SQLite auto-migration (which runs after schema boot) never moved any +data. Deleting the data dir and re-initdb'ing (now forced UTF-8) is therefore +lossless — but only when proven, so recovery requires ALL of: + 1. embedded mode (Fusion owns the data dir), + 2. the encoding-conversion error signature, + 3. the live cluster confirming a non-UTF-8 server_encoding, + 4. zero tables in project/central/archive and no recorded migration + versions (the never-successfully-booted state), + 5. this process owns the postmaster (never delete under a foreign owner). +One retry only; a second failure surfaces the manual re-init hint. +*/ +class NonUtf8EmbeddedClusterError extends Error { + constructor(readonly dataDir: string, override readonly cause: unknown) { + super(`embedded cluster at ${dataDir} has a non-UTF-8 encoding and is empty; auto re-initializing`); + } +} + +/** Matches PostgreSQL's encoding-conversion failure raised by a non-UTF-8 cluster. */ +export function isEncodingConversionError(chainText: string): boolean { + return /has no equivalent in encoding/i.test(chainText); +} + +/** + * Prove the cluster is in the recoverable state: non-UTF-8 server encoding AND + * no applied schema (no tables in Fusion's schemas, no migration versions). + * Any query failure means "not proven" — the caller keeps the original error. + */ +async function isEmptyNonUtf8Cluster(db: PostgresJsDatabase>): Promise { + const encodingRows = (await db.execute( + sql`SELECT current_setting('server_encoding') AS encoding`, + )) as unknown as Array<{ encoding: string }>; + const encoding = (encodingRows[0]?.encoding ?? "").toUpperCase(); + if (encoding === "UTF8" || encoding === "") return false; + + const tableRows = (await db.execute(sql` + SELECT count(*)::int AS tables + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname IN ('project', 'central', 'archive') AND c.relkind = 'r' + `)) as unknown as Array<{ tables: number }>; + if ((tableRows[0]?.tables ?? 1) !== 0) return false; + + const versionRows = (await db.execute(sql` + SELECT count(*)::int AS versions + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relname = ${MIGRATION_BOOKKEEPING_TABLE} AND c.relkind = 'r' + `)) as unknown as Array<{ versions: number }>; + if ((versionRows[0]?.versions ?? 0) !== 0) { + const applied = (await db.execute(sql.raw( + `SELECT count(*)::int AS applied FROM public.${MIGRATION_BOOKKEEPING_TABLE}`, + ))) as unknown as Array<{ applied: number }>; + if ((applied[0]?.applied ?? 1) !== 0) return false; + } + return true; +} + async function bootSchemaBackend( options: Pick, bypassProjectIsolation = false, +): Promise { + try { + return await bootSchemaBackendOnce(options, bypassProjectIsolation); + } catch (error) { + if (!(error instanceof NonUtf8EmbeddedClusterError)) throw error; + log.warn( + `startup-factory: embedded cluster at ${error.dataDir} was created with a non-UTF-8 OS-locale ` + + `encoding by an earlier version and never completed a boot (issue #2286). ` + + `Re-initializing it as UTF-8 and retrying once.`, + ); + rmSync(error.dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); + return await bootSchemaBackendOnce(options, bypassProjectIsolation); + } +} + +async function bootSchemaBackendOnce( + options: Pick, + bypassProjectIsolation = false, ): Promise { const env = options.env ?? process.env; const backend = options.backend ?? resolveBackend(env); @@ -252,10 +335,12 @@ async function bootSchemaBackend( let embeddedRuntimeUrl: string | null = null; let embeddedOwnsProcess = false; let resolvedBackend = backend; + let embeddedDataDir: string | null = null; if (backend.mode === "embedded") { const { EmbeddedPostgresLifecycle, defaultEmbeddedDataDir, DEFAULT_EMBEDDED_DATABASE } = await import("./embedded-lifecycle.js"); const dataDir = resolve(options.embeddedDataDir ?? defaultEmbeddedDataDir()); + embeddedDataDir = dataDir; log.log(`startup-factory: starting embedded PostgreSQL (data dir ${dataDir})`); embeddedLifecycle = new EmbeddedPostgresLifecycle({ dataDir, @@ -308,6 +393,23 @@ async function bootSchemaBackend( embeddedOwnsProcess, }; } catch (error) { + /* + FNXC:PostgresEmbedded 2026-07-18-01:10: + Classify the #2286 non-UTF-8-cluster state while the connection is still + open (the proof queries need it), then tear down before signalling the + retry wrapper. Never recover a joined instance (embeddedOwnsProcess false): + deleting a data dir under a foreign postmaster is not ours to do. + */ + let recoverable = false; + if ( + embeddedLifecycle !== null && + embeddedOwnsProcess && + connections !== undefined && + embeddedDataDir !== null && + isEncodingConversionError(error instanceof Error ? `${error.message} ${String(error.cause ?? "")}` : String(error)) + ) { + recoverable = await isEmptyNonUtf8Cluster(connections.migration).catch(() => false); + } await connections?.close().catch(() => undefined); await stopEmbeddedRuntime( embeddedLifecycle, @@ -315,6 +417,9 @@ async function bootSchemaBackend( embeddedRuntimeUrl, embeddedOwnsProcess, ).catch(() => undefined); + if (recoverable && embeddedDataDir !== null) { + throw new NonUtf8EmbeddedClusterError(embeddedDataDir, error); + } throw error; } } diff --git a/scripts/verify-windows-encoding-recovery.mjs b/scripts/verify-windows-encoding-recovery.mjs new file mode 100644 index 0000000000..a5399aeb8a --- /dev/null +++ b/scripts/verify-windows-encoding-recovery.mjs @@ -0,0 +1,127 @@ +// FNXC:PostgresEmbedded 2026-07-18-01:10: +// End-to-end proof of the issue #2286 auto-recovery. Simulates a machine hit +// by the old bug: creates a real embedded cluster with a non-UTF-8 server +// encoding (initdb --encoding=WIN1252, the runner's own OS-locale outcome +// pre-fix) under a throwaway HOME, then boots the full `fn serve` against it. +// The startup factory must detect the encoding-conversion failure, prove the +// cluster is empty, delete and re-initdb it as UTF-8, and reach a healthy +// /api/health — with no operator intervention. +import { spawn, spawnSync } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const repoRoot = resolve(scriptDir, ".."); +const cliBin = join(repoRoot, "packages", "cli", "bin.mjs"); +const PORT = 55497; +const HEALTH_TIMEOUT_MS = 240_000; + +function sh(cmd, args) { + const r = spawnSync(cmd, args, { encoding: "utf8" }); + return { status: r.status, out: `${r.stdout || ""}${r.stderr || ""}`.trim() }; +} + +const home = mkdtempSync(join(tmpdir(), "fusion-encoding-recovery-home-")); +const project = mkdtempSync(join(tmpdir(), "fusion-encoding-recovery-project-")); +const dataDir = join(home, ".fusion", "embedded-postgres", "default"); +mkdirSync(dirname(dataDir), { recursive: true }); + +// Build the bad cluster with the REAL lifecycle: caller initdb flags are +// appended after the UTF-8 defaults and initdb takes the last occurrence, so +// this recreates exactly what pre-fix initdb produced on a WIN1252 locale. +const { EmbeddedPostgresLifecycle } = await import( + pathToFileURL(join(repoRoot, "packages", "core", "dist", "postgres", "embedded-lifecycle.js")).href +); +const seed = new EmbeddedPostgresLifecycle({ + dataDir, + database: "fusion", + user: "postgres", + password: "password", + port: PORT, + initdbFlags: ["--encoding=WIN1252", "--locale=C"], + onLog: (m) => console.log(`[seed] ${m}`), + onError: (e) => console.error(`[seed:err] ${String(e)}`), +}); +await seed.start(); +await seed.stop(); +console.log(`recovery-verify: seeded non-UTF-8 cluster at ${dataDir}`); + +let output = ""; +const child = spawn( + process.execPath, + [cliBin, "serve", "--port", String(PORT), "--host", "127.0.0.1", "--paused"], + { + cwd: project, + env: { + ...process.env, + HOME: home, + USERPROFILE: home, + FUSION_SKIP_ONBOARDING: "1", + DATABASE_URL: undefined, + FUSION_NO_EMBEDDED_PG: undefined, + PORT: undefined, + }, + stdio: ["ignore", "pipe", "pipe"], + }, +); +child.stdout.on("data", (d) => (output += d)); +child.stderr.on("data", (d) => (output += d)); +let exited = false; +child.once("exit", () => (exited = true)); + +async function pollHealth() { + const deadline = Date.now() + HEALTH_TIMEOUT_MS; + while (Date.now() < deadline) { + if (exited) throw new Error(`serve exited before becoming healthy.\n${output.slice(-4000)}`); + try { + const controller = new AbortController(); + const abortTimer = setTimeout(() => controller.abort(), 2000); + const res = await fetch(`http://127.0.0.1:${PORT}/api/health`, { signal: controller.signal }); + clearTimeout(abortTimer); + if (res.ok) return; + } catch { + // not up yet + } + await new Promise((r) => setTimeout(r, 500)); + } + throw new Error(`serve did not become healthy in ${HEALTH_TIMEOUT_MS}ms.\n${output.slice(-4000)}`); +} + +let failed = false; +try { + await pollHealth(); + console.log(`recovery-verify: /api/health OK on :${PORT} after auto-recovery`); + if (!/Re-initializing it as UTF-8 and retrying once/.test(output)) { + throw new Error( + `serve became healthy but the auto-recovery log line is missing — the seed cluster may not have triggered the #2286 path.\n${output.slice(-4000)}`, + ); + } + console.log("recovery-verify: auto-recovery log line confirmed"); +} catch (err) { + console.error(`recovery-verify: FAIL — ${err instanceof Error ? err.message : String(err)}`); + failed = true; +} finally { + try { + child.kill("SIGKILL"); + } catch { + // already gone + } + try { + const pid = parseInt(readFileSync(join(dataDir, "postmaster.pid"), "utf8").split("\n")[0], 10); + if (Number.isFinite(pid) && pid > 0) sh("taskkill", ["/pid", String(pid), "/f", "/t"]); + } catch { + // no postmaster left + } + for (const dir of [home, project]) { + try { + rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); + } catch (cleanupErr) { + console.error(`recovery-verify: cleanup warning: ${String(cleanupErr)}`); + } + } +} + +if (failed) process.exit(1); +console.log("recovery-verify: PASS — non-UTF-8 cluster auto-recovered to a healthy boot");