root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix(core): auto-repair empty non-UTF-8 embedded Postgres clusters on boot (#2286)

Browse Source 
Users whose embedded cluster was initdb'd with an OS-locale encoding by
a pre-fix version now self-heal with zero manual steps: on the
encoding-conversion schema failure the startup factory proves the
cluster is non-UTF-8 AND empty (the baseline transaction never applied,
so no schema or migrated data can exist) and that this process owns the
postmaster, then deletes the data dir and reboots once with the UTF-8
initdb defaults. Joined instances and unproven states keep the manual
re-init hint; one retry ever, so no loops.

Verified on the elevated windows-latest runner: CI seeds a real WIN1252
cluster via initdb and proves a stock 'fn serve' auto-recovers it to a
healthy /api/health (run 29633351848, all jobs green). Also caps the
desktop-windows embedded-PG smoke at 30 min and adds a skip input.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-07-17 23:18:40 -07:00
parent d4ee80a818
commit 95e011f890
6 changed files with 289 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.changeset/embedded-pg-encoding-auto-reinit.md Normal file
View File

@@ -0,0 +1,7 @@
---
"@runfusion/fusion": patch
---
summary: Fusion now auto-repairs embedded PostgreSQL clusters left in the non-UTF-8 encoding state by earlier versions.
category: fix
dev: "Issue #2286 follow-up: on the encoding-conversion schema failure, the startup factory proves the embedded cluster is non-UTF-8 AND empty (no tables in project/central/archive, no recorded migrations — guaranteed for affected installs since the baseline never applied) and that this process owns the postmaster, then deletes the data dir and re-boots once with the UTF-8 initdb defaults. Joined instances and any non-proven state keep the manual re-init hint."

13
.github/workflows/desktop-windows.yml vendored
View File

@@ -2,6 +2,11 @@ name: Desktop Windows Build
on:
workflow_dispatch:
inputs:
skip_pg_smoke:
description: "Skip the embedded-PG smoke (already covered by verify-elevated-restricted.yml)"
type: boolean
default: false
jobs:
build-windows-exe:
@@ -29,7 +34,13 @@ jobs:
# Create the user and warm its profile once here, outside any test window;
# the launcher resets the user's password before each run, but the warmed
# profile persists, so every later launch is ~0.5s.
# FNXC:WindowsDesktopPackaging 2026-07-18-01:40:
# The smoke wedged a runner for 45+ min with Start-Process -Wait and no
# step timeout (baseline ~18 min). Cap it, and allow skipping it via
# dispatch input when the dedicated verify-elevated-restricted workflow
# already proves embedded PG on this ref — packaging does not depend on it.
- name: Prewarm embedded-PG helper user profile
if: ${{ !inputs.skip_pg_smoke }}
shell: pwsh
run: |
$user = "fusion-pg"
@@ -53,6 +64,8 @@ jobs:
# the normal embedded-postgres path — no in-launcher Start-Process
# -Credential / staging / process-kill races.
- name: Smoke embedded Postgres on Windows
if: ${{ !inputs.skip_pg_smoke }}
timeout-minutes: 30
shell: pwsh
run: |
$user = "fusion-pg"

7
.github/workflows/verify-elevated-restricted.yml vendored
View File

@@ -9,7 +9,7 @@ name: Verify Elevated Restricted-Token Postgres
on:
workflow_dispatch:
push:
branches: [feature/win-elevated-no-user]
branches: [feature/win-elevated-no-user, fix/embedded-pg-encoding-auto-reinit]
jobs:
verify-elevated-restricted:
@@ -75,6 +75,11 @@ jobs:
- name: Boot smoke (fn --help + serve /api/health, elevated)
run: pnpm smoke:boot
# Issue #2286 auto-recovery: seed a real non-UTF-8 cluster, then prove a
# full `fn serve` self-heals it to UTF-8 and reaches a healthy state.
- name: Encoding auto-recovery (seeded WIN1252 cluster)
run: node scripts/verify-windows-encoding-recovery.mjs
- name: Assert no fusion-pg account was created
shell: pwsh
run: |

29
packages/core/src/__tests__/postgres/startup-factory.test.ts
View File

@@ -25,6 +25,7 @@ import { join } from "node:path";
import { tmpdir } from "node:os";
import {
createTaskStoreForBackend,
isEncodingConversionError,
shouldUsePostgresBackend,
isEmbeddedPgRequested,
isEmbeddedPgOptedOut,
@@ -196,3 +197,31 @@ describe("startup-factory: backend descriptor propagation", () => {
).rejects.toThrow();
});
});
/*
FNXC:PostgresEmbedded 2026-07-18-01:10:
Issue #2286 auto-recovery trigger. The classifier must catch PostgreSQL's
encoding-conversion failure raised when a non-UTF-8 cluster (WIN1252/WIN1254
from a pre-fix initdb on a non-UTF-8 OS locale) receives the UTF-8 schema
SQL — and nothing else, so ordinary schema errors never delete a data dir.
*/
describe("isEncodingConversionError (#2286 recovery trigger)", () => {
it("matches the encoding-conversion failure from a non-UTF-8 cluster", () => {
expect(
isEncodingConversionError(
'character with byte sequence 0xe2 0x86 0x92 in encoding "UTF8" has no equivalent in encoding "WIN1254"',
),
).toBe(true);
expect(
isEncodingConversionError(
'Failed query: CREATE TABLE ... params: caused by: character with byte sequence 0xe2 0x86 0x92 in encoding "UTF8" has no equivalent in encoding "WIN1252"',
),
).toBe(true);
});
it("does not match unrelated schema or connection errors", () => {
expect(isEncodingConversionError('syntax error at or near "CREATE"')).toBe(false);
expect(isEncodingConversionError("connection refused")).toBe(false);
expect(isEncodingConversionError('FATAL: invalid value for parameter "shared_memory_type"')).toBe(false);
});
});

109
packages/core/src/postgres/startup-factory.ts
View File

@@ -37,7 +37,7 @@
*/
import { join, resolve } from "node:path";
import { existsSync } from "node:fs";
import { existsSync, rmSync } from "node:fs";
import { createHash } from "node:crypto";
import { sql } from "drizzle-orm";
import { isValidSqliteDatabaseFile } from "../sqlite-validation.js";
@@ -53,7 +53,8 @@ import {
createConnectionSetFromUrl,
type PostgresConnections,
} from "./connection.js";
import { applySchemaBaseline } from "./schema-applier.js";
import { applySchemaBaseline, MIGRATION_BOOKKEEPING_TABLE } from "./schema-applier.js";
import type { PostgresJsDatabase } from "drizzle-orm/postgres-js";
import { createAsyncDataLayer, type AsyncDataLayer } from "./data-layer.js";
import {
invalidateEmbeddedRuntimeUrl,
@@ -234,9 +235,91 @@ async function stopEmbeddedRuntime(
* Callers retain ownership of the returned resources and may replace the
* administrative pool with an RLS-bound runtime pool after migration.
*/
/*
FNXC:PostgresEmbedded 2026-07-18-01:10:
Issue #2286 auto-recovery. An embedded cluster initdb'd by an earlier version
on a non-UTF-8 OS locale (WIN1252/WIN1254) rejects the UTF-8 schema SQL with
`character ... has no equivalent in encoding`. Such a cluster can NEVER have
completed a boot: the whole baseline runs in one transaction whose query
string fails encoding conversion before execution, so no schema was applied
and the SQLite auto-migration (which runs after schema boot) never moved any
data. Deleting the data dir and re-initdb'ing (now forced UTF-8) is therefore
lossless — but only when proven, so recovery requires ALL of:
1. embedded mode (Fusion owns the data dir),
2. the encoding-conversion error signature,
3. the live cluster confirming a non-UTF-8 server_encoding,
4. zero tables in project/central/archive and no recorded migration
versions (the never-successfully-booted state),
5. this process owns the postmaster (never delete under a foreign owner).
One retry only; a second failure surfaces the manual re-init hint.
*/
class NonUtf8EmbeddedClusterError extends Error {
constructor(readonly dataDir: string, override readonly cause: unknown) {
super(`embedded cluster at ${dataDir} has a non-UTF-8 encoding and is empty; auto re-initializing`);
}
}
/** Matches PostgreSQL's encoding-conversion failure raised by a non-UTF-8 cluster. */
export function isEncodingConversionError(chainText: string): boolean {
return /has no equivalent in encoding/i.test(chainText);
}
/**
* Prove the cluster is in the recoverable state: non-UTF-8 server encoding AND
* no applied schema (no tables in Fusion's schemas, no migration versions).
* Any query failure means "not proven" — the caller keeps the original error.
*/
async function isEmptyNonUtf8Cluster(db: PostgresJsDatabase<Record<string, never>>): Promise<boolean> {
const encodingRows = (await db.execute(
sql`SELECT current_setting('server_encoding') AS encoding`,
)) as unknown as Array<{ encoding: string }>;
const encoding = (encodingRows[0]?.encoding ?? "").toUpperCase();
if (encoding === "UTF8" || encoding === "") return false;
const tableRows = (await db.execute(sql`
SELECT count(*)::int AS tables
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname IN ('project', 'central', 'archive') AND c.relkind = 'r'
`)) as unknown as Array<{ tables: number }>;
if ((tableRows[0]?.tables ?? 1) !== 0) return false;
const versionRows = (await db.execute(sql`
SELECT count(*)::int AS versions
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = 'public' AND c.relname = ${MIGRATION_BOOKKEEPING_TABLE} AND c.relkind = 'r'
`)) as unknown as Array<{ versions: number }>;
if ((versionRows[0]?.versions ?? 0) !== 0) {
const applied = (await db.execute(sql.raw(
`SELECT count(*)::int AS applied FROM public.${MIGRATION_BOOKKEEPING_TABLE}`,
))) as unknown as Array<{ applied: number }>;
if ((applied[0]?.applied ?? 1) !== 0) return false;
}
return true;
}
async function bootSchemaBackend(
options: Pick<CreateTaskStoreForBackendOptions, "env" | "backend" | "embeddedPgRequested" | "embeddedDataDir" | "poolMax">,
bypassProjectIsolation = false,
): Promise<SchemaBackendBootResult> {
try {
return await bootSchemaBackendOnce(options, bypassProjectIsolation);
} catch (error) {
if (!(error instanceof NonUtf8EmbeddedClusterError)) throw error;
log.warn(
`startup-factory: embedded cluster at ${error.dataDir} was created with a non-UTF-8 OS-locale ` +
`encoding by an earlier version and never completed a boot (issue #2286). ` +
`Re-initializing it as UTF-8 and retrying once.`,
);
rmSync(error.dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 });
return await bootSchemaBackendOnce(options, bypassProjectIsolation);
}
}
async function bootSchemaBackendOnce(
options: Pick<CreateTaskStoreForBackendOptions, "env" | "backend" | "embeddedPgRequested" | "embeddedDataDir" | "poolMax">,
bypassProjectIsolation = false,
): Promise<SchemaBackendBootResult> {
const env = options.env ?? process.env;
const backend = options.backend ?? resolveBackend(env);
@@ -252,10 +335,12 @@ async function bootSchemaBackend(
let embeddedRuntimeUrl: string | null = null;
let embeddedOwnsProcess = false;
let resolvedBackend = backend;
let embeddedDataDir: string | null = null;
if (backend.mode === "embedded") {
const { EmbeddedPostgresLifecycle, defaultEmbeddedDataDir, DEFAULT_EMBEDDED_DATABASE } =
await import("./embedded-lifecycle.js");
const dataDir = resolve(options.embeddedDataDir ?? defaultEmbeddedDataDir());
embeddedDataDir = dataDir;
log.log(`startup-factory: starting embedded PostgreSQL (data dir ${dataDir})`);
embeddedLifecycle = new EmbeddedPostgresLifecycle({
dataDir,
@@ -308,6 +393,23 @@ async function bootSchemaBackend(
embeddedOwnsProcess,
};
} catch (error) {
/*
FNXC:PostgresEmbedded 2026-07-18-01:10:
Classify the #2286 non-UTF-8-cluster state while the connection is still
open (the proof queries need it), then tear down before signalling the
retry wrapper. Never recover a joined instance (embeddedOwnsProcess false):
deleting a data dir under a foreign postmaster is not ours to do.
*/
let recoverable = false;
if (
embeddedLifecycle !== null &&
embeddedOwnsProcess &&
connections !== undefined &&
embeddedDataDir !== null &&
isEncodingConversionError(error instanceof Error ? `${error.message} ${String(error.cause ?? "")}` : String(error))
) {
recoverable = await isEmptyNonUtf8Cluster(connections.migration).catch(() => false);
}
await connections?.close().catch(() => undefined);
await stopEmbeddedRuntime(
embeddedLifecycle,
@@ -315,6 +417,9 @@ async function bootSchemaBackend(
embeddedRuntimeUrl,
embeddedOwnsProcess,
).catch(() => undefined);
if (recoverable && embeddedDataDir !== null) {
throw new NonUtf8EmbeddedClusterError(embeddedDataDir, error);
}
throw error;
}
}

127
scripts/verify-windows-encoding-recovery.mjs Normal file
View File

@@ -0,0 +1,127 @@
// FNXC:PostgresEmbedded 2026-07-18-01:10:
// End-to-end proof of the issue #2286 auto-recovery. Simulates a machine hit
// by the old bug: creates a real embedded cluster with a non-UTF-8 server
// encoding (initdb --encoding=WIN1252, the runner's own OS-locale outcome
// pre-fix) under a throwaway HOME, then boots the full `fn serve` against it.
// The startup factory must detect the encoding-conversion failure, prove the
// cluster is empty, delete and re-initdb it as UTF-8, and reach a healthy
// /api/health — with no operator intervention.
import { spawn, spawnSync } from "node:child_process";
import { mkdtempSync, mkdirSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
const scriptDir = dirname(fileURLToPath(import.meta.url));
const repoRoot = resolve(scriptDir, "..");
const cliBin = join(repoRoot, "packages", "cli", "bin.mjs");
const PORT = 55497;
const HEALTH_TIMEOUT_MS = 240_000;
function sh(cmd, args) {
const r = spawnSync(cmd, args, { encoding: "utf8" });
return { status: r.status, out: `${r.stdout || ""}${r.stderr || ""}`.trim() };
}
const home = mkdtempSync(join(tmpdir(), "fusion-encoding-recovery-home-"));
const project = mkdtempSync(join(tmpdir(), "fusion-encoding-recovery-project-"));
const dataDir = join(home, ".fusion", "embedded-postgres", "default");
mkdirSync(dirname(dataDir), { recursive: true });
// Build the bad cluster with the REAL lifecycle: caller initdb flags are
// appended after the UTF-8 defaults and initdb takes the last occurrence, so
// this recreates exactly what pre-fix initdb produced on a WIN1252 locale.
const { EmbeddedPostgresLifecycle } = await import(
pathToFileURL(join(repoRoot, "packages", "core", "dist", "postgres", "embedded-lifecycle.js")).href
);
const seed = new EmbeddedPostgresLifecycle({
dataDir,
database: "fusion",
user: "postgres",
password: "password",
port: PORT,
initdbFlags: ["--encoding=WIN1252", "--locale=C"],
onLog: (m) => console.log(`[seed] ${m}`),
onError: (e) => console.error(`[seed:err] ${String(e)}`),
});
await seed.start();
await seed.stop();
console.log(`recovery-verify: seeded non-UTF-8 cluster at ${dataDir}`);
let output = "";
const child = spawn(
process.execPath,
[cliBin, "serve", "--port", String(PORT), "--host", "127.0.0.1", "--paused"],
{
cwd: project,
env: {
...process.env,
HOME: home,
USERPROFILE: home,
FUSION_SKIP_ONBOARDING: "1",
DATABASE_URL: undefined,
FUSION_NO_EMBEDDED_PG: undefined,
PORT: undefined,
},
stdio: ["ignore", "pipe", "pipe"],
},
);
child.stdout.on("data", (d) => (output += d));
child.stderr.on("data", (d) => (output += d));
let exited = false;
child.once("exit", () => (exited = true));
async function pollHealth() {
const deadline = Date.now() + HEALTH_TIMEOUT_MS;
while (Date.now() < deadline) {
if (exited) throw new Error(`serve exited before becoming healthy.\n${output.slice(-4000)}`);
try {
const controller = new AbortController();
const abortTimer = setTimeout(() => controller.abort(), 2000);
const res = await fetch(`http://127.0.0.1:${PORT}/api/health`, { signal: controller.signal });
clearTimeout(abortTimer);
if (res.ok) return;
} catch {
// not up yet
}
await new Promise((r) => setTimeout(r, 500));
}
throw new Error(`serve did not become healthy in ${HEALTH_TIMEOUT_MS}ms.\n${output.slice(-4000)}`);
}
let failed = false;
try {
await pollHealth();
console.log(`recovery-verify: /api/health OK on :${PORT} after auto-recovery`);
if (!/Re-initializing it as UTF-8 and retrying once/.test(output)) {
throw new Error(
`serve became healthy but the auto-recovery log line is missing — the seed cluster may not have triggered the #2286 path.\n${output.slice(-4000)}`,
);
}
console.log("recovery-verify: auto-recovery log line confirmed");
} catch (err) {
console.error(`recovery-verify: FAIL — ${err instanceof Error ? err.message : String(err)}`);
failed = true;
} finally {
try {
child.kill("SIGKILL");
} catch {
// already gone
}
try {
const pid = parseInt(readFileSync(join(dataDir, "postmaster.pid"), "utf8").split("\n")[0], 10);
if (Number.isFinite(pid) && pid > 0) sh("taskkill", ["/pid", String(pid), "/f", "/t"]);
} catch {
// no postmaster left
}
for (const dir of [home, project]) {
try {
rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 });
} catch (cleanupErr) {
console.error(`recovery-verify: cleanup warning: ${String(cleanupErr)}`);
}
}
}
if (failed) process.exit(1);
console.log("recovery-verify: PASS — non-UTF-8 cluster auto-recovered to a healthy boot");