diff --git a/.changeset/fix-chat-mailbox-nul-byte-sanitize.md b/.changeset/fix-chat-mailbox-nul-byte-sanitize.md new file mode 100644 index 0000000000..6173b88d56 --- /dev/null +++ b/.changeset/fix-chat-mailbox-nul-byte-sanitize.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Fix a crash where chat messages and mailbox sends containing a raw NUL byte would abort mid-conversation. +category: fix +dev: PostgreSQL text/jsonb columns reject U+0000 outright ("unsupported Unicode escape sequence" / "\u0000 cannot be converted to text"). Tool output piped into a chat message or agent mailbox send could carry a literal NUL byte and crash addChatMessage/addChatRoomMessage/sendMessage with an uncaught PostgresError. Extracted the existing stripNulChars/deepStripNulChars sanitizer (previously only used by the one-time SQLite migration) into a shared packages/core/src/postgres/nul-sanitize.ts module and wired it into all three live write paths. Also fixes a related embedded-Postgres startup race (JoinedInstanceUnreachableError) where a joiner could hit ECONNREFUSED before the owning process's listener was ready; now retried once, mirroring the existing NonUtf8EmbeddedClusterError retry pattern. diff --git a/packages/core/src/__tests__/nul-sanitize.test.ts b/packages/core/src/__tests__/nul-sanitize.test.ts new file mode 100644 index 0000000000..2853593277 --- /dev/null +++ b/packages/core/src/__tests__/nul-sanitize.test.ts @@ -0,0 +1,86 @@ +/** + * FNXC:PostgresMigrationNulSanitize 2026-07-20: + * Unit coverage for the shared NUL-byte sanitizer (packages/core/src/postgres/nul-sanitize.ts), + * extracted from sqlite-migrator.ts so live write paths (async-chat-store.ts, + * async-message-store.ts) can reuse the same tested behavior. No PostgreSQL + * connection required — pure function coverage. + */ + +import { describe, it, expect } from "vitest"; +import { + stripNulChars, + deepStripNulChars, + sanitizeTextValue, + sanitizeJsonbValue, +} from "../postgres/nul-sanitize.js"; + +describe("stripNulChars", () => { + it("removes U+0000 from a string", () => { + expect(stripNulChars("hello\u0000world")).toBe("helloworld"); + }); + + it("is a no-op when no NUL is present", () => { + expect(stripNulChars("clean string")).toBe("clean string"); + }); + + it("strips multiple NUL bytes", () => { + expect(stripNulChars("a\u0000b\u0000c\u0000")).toBe("abc"); + }); +}); + +describe("deepStripNulChars", () => { + it("strips NUL from nested string values and object keys", () => { + const input = { + ["key\u0000nul"]: "value\u0000nul", + nested: ["a\u0000b", { deeper: "c\u0000d" }], + }; + expect(deepStripNulChars(input)).toEqual({ + keynul: "valuenul", + nested: ["ab", { deeper: "cd" }], + }); + }); + + it("passes through non-string primitives and null unchanged", () => { + expect(deepStripNulChars(42)).toBe(42); + expect(deepStripNulChars(true)).toBe(true); + expect(deepStripNulChars(null)).toBe(null); + }); +}); + +describe("sanitizeTextValue", () => { + it("strips NUL from a string value", () => { + expect(sanitizeTextValue("diag\u0000nostic dump")).toBe("diagnostic dump"); + }); + + it("passes through null/undefined unchanged", () => { + expect(sanitizeTextValue(null)).toBe(null); + expect(sanitizeTextValue(undefined)).toBe(undefined); + }); +}); + +describe("sanitizeJsonbValue", () => { + it("deep-strips NUL from an object destined for a jsonb column", () => { + expect(sanitizeJsonbValue({ note: "tail\u0000end" })).toEqual({ note: "tailend" }); + }); + + it("passes through null/undefined unchanged", () => { + expect(sanitizeJsonbValue(null)).toBe(null); + expect(sanitizeJsonbValue(undefined)).toBe(undefined); + }); + + it("handles the exact failure signature observed in production: a raw NUL embedded in a diagnostics dump", () => { + // Real-world trigger: an agent piped a Windows CLI diagnostics dump + // (tasklist/netstat output) directly into a chat message body. The dump + // contained a raw NUL byte, which Postgres's json_ereport_error rejected + // with "unsupported Unicode escape sequence" / "\u0000 cannot be + // converted to text", aborting the chat write mid-conversation. + const diagnosticDump = + "===FUSION DB AGENTS===\n===NODES===\n\u0000===RUNNING PROCESSES===\n"; + expect(sanitizeTextValue(diagnosticDump)).toBe( + "===FUSION DB AGENTS===\n===NODES===\n===RUNNING PROCESSES===\n", + ); + expect(sanitizeJsonbValue({ toolOutput: diagnosticDump })).toEqual({ + toolOutput: "===FUSION DB AGENTS===\n===NODES===\n===RUNNING PROCESSES===\n", + }); + }); +}); diff --git a/packages/core/src/__tests__/postgres/chat-store-content-search-edit.pg.test.ts b/packages/core/src/__tests__/postgres/chat-store-content-search-edit.pg.test.ts index 0967ec37b7..d8544f98f9 100644 --- a/packages/core/src/__tests__/postgres/chat-store-content-search-edit.pg.test.ts +++ b/packages/core/src/__tests__/postgres/chat-store-content-search-edit.pg.test.ts @@ -234,4 +234,37 @@ pgDescribe("async chat store content search + edit primitives (PostgreSQL)", () updateChatMessageMetadata(ctx.layer.db, "msg-missing", { x: 1 }), ).rejects.toThrow(/not found/); }); + + /* + FNXC:PostgresMigrationNulSanitize 2026-07-20: + Production incident: the CEO agent's chat turn crashed mid-conversation + because a tool call piped raw Windows CLI diagnostics output (containing a + literal U+0000 byte) straight into the chat message body. Postgres rejected + the INSERT with PostgresError code 22P05 ("unsupported Unicode escape + sequence" / "\u0000 cannot be converted to text"), which aborted the write + and killed the turn. addChatMessage now sanitizes content/thinkingOutput + (text) and metadata/attachments (jsonb) before insert — verify a message + containing a raw NUL byte round-trips instead of throwing. + */ + it("addChatMessage strips a raw U+0000 byte instead of throwing (regression for the CEO-agent chat crash)", async () => { + ctx = await setupCtx(); + const session = await makeSession(ctx); + + const diagnosticDump = + "===FUSION DB AGENTS===\n===NODES===\n\u0000===RUNNING PROCESSES===\n"; + const sent = await addMessage(ctx, session.id, "assistant", diagnosticDump, { + toolOutput: `payload\u0000tail`, + }); + + expect(sent.content).toBe( + "===FUSION DB AGENTS===\n===NODES===\n===RUNNING PROCESSES===\n", + ); + expect(sent.metadata).toEqual({ toolOutput: "payloadtail" }); + + const [persisted] = await getChatMessages(ctx.layer.db, session.id); + expect(persisted.content).toBe( + "===FUSION DB AGENTS===\n===NODES===\n===RUNNING PROCESSES===\n", + ); + expect(persisted.metadata).toEqual({ toolOutput: "payloadtail" }); + }); }); diff --git a/packages/core/src/__tests__/postgres/message-store.pg.test.ts b/packages/core/src/__tests__/postgres/message-store.pg.test.ts index 3a53ce0ab6..9d98bcfbd6 100644 --- a/packages/core/src/__tests__/postgres/message-store.pg.test.ts +++ b/packages/core/src/__tests__/postgres/message-store.pg.test.ts @@ -122,4 +122,41 @@ pgTest("MessageStore send (PostgreSQL backend mode)", () => { .filter((message) => message.metadata?.kind === "postgres-migration-complete"); expect(completionMessages).toHaveLength(1); }); + + /* + FNXC:PostgresMigrationNulSanitize 2026-07-20: + Same NUL-byte hazard as the chat-store regression (chat-store-content-search-edit.pg.test.ts): + agent-to-agent/agent-to-user mailbox content can carry raw tool output + containing a literal U+0000 byte, which Postgres's text/jsonb columns + reject outright ("unsupported Unicode escape sequence" / "\u0000 cannot be + converted to text"). sendMessage now sanitizes content/metadata before + insert — verify a NUL-laden send round-trips instead of throwing. + */ + it("sendMessage strips a raw U+0000 byte instead of throwing", async () => { + const { MessageStore } = await import("../../message-store.js"); + const store = new MessageStore(null, { asyncLayer: h.layer() }); + + const diagnosticDump = + "===FUSION DB AGENTS===\n===NODES===\n\u0000===RUNNING PROCESSES===\n"; + const sent = await store.sendMessage({ + fromId: "agent-ceo", + fromType: "agent", + toId: "user-x", + toType: "user", + content: diagnosticDump, + type: "agent-to-user", + metadata: { toolOutput: "payload\u0000tail" }, + }); + + expect(sent.content).toBe( + "===FUSION DB AGENTS===\n===NODES===\n===RUNNING PROCESSES===\n", + ); + expect(sent.metadata?.toolOutput).toBe("payloadtail"); + + const fetched = await store.getMessage(sent.id); + expect(fetched?.content).toBe( + "===FUSION DB AGENTS===\n===NODES===\n===RUNNING PROCESSES===\n", + ); + expect(fetched?.metadata?.toolOutput).toBe("payloadtail"); + }); }); diff --git a/packages/core/src/async-chat-store.ts b/packages/core/src/async-chat-store.ts index 8b6ee7216e..4182df6a4a 100644 --- a/packages/core/src/async-chat-store.ts +++ b/packages/core/src/async-chat-store.ts @@ -19,6 +19,7 @@ import { and, asc, desc, eq, gt, ilike, inArray, isNull, lte, ne, or as orFn, sql as drizzleSql } from "drizzle-orm"; import * as schema from "./postgres/schema/index.js"; import type { AsyncDataLayer, DbTransaction } from "./postgres/data-layer.js"; +import { sanitizeTextValue, sanitizeJsonbValue } from "./postgres/nul-sanitize.js"; import type { ChatAttachment, ChatInFlightGenerationState, @@ -190,21 +191,37 @@ export async function addChatMessage( handle: QueryHandle, message: ChatMessage, ): Promise { + // FNXC:PostgresMigrationNulSanitize 2026-07-20: agent/tool output persisted + // here can contain a raw NUL byte (e.g. piped-through Windows CLI dumps), + // which Postgres text/jsonb columns reject outright. Sanitize before + // insert instead of letting the write throw mid-conversation, and return + // the sanitized value so the in-memory result matches what was persisted + // (the original unsanitized `message` object must not be handed back). + const sanitizedAttachments = message.attachments === undefined + ? undefined + : sanitizeJsonbValue(message.attachments); + const sanitized: ChatMessage = { + ...message, + content: sanitizeTextValue(message.content), + thinkingOutput: sanitizeTextValue(message.thinkingOutput), + metadata: sanitizeJsonbValue(message.metadata), + attachments: sanitizedAttachments, + }; await handle.insert(schema.project.chatMessages).values({ - id: message.id, - sessionId: message.sessionId, - role: message.role, - content: message.content, - thinkingOutput: message.thinkingOutput, - metadata: message.metadata, - attachments: message.attachments ?? null, - createdAt: message.createdAt, + id: sanitized.id, + sessionId: sanitized.sessionId, + role: sanitized.role, + content: sanitized.content, + thinkingOutput: sanitized.thinkingOutput, + metadata: sanitized.metadata, + attachments: sanitizedAttachments ?? null, + createdAt: sanitized.createdAt, }); await handle .update(schema.project.chatSessions) - .set({ updatedAt: message.createdAt }) - .where(eq(schema.project.chatSessions.id, message.sessionId)); - return message; + .set({ updatedAt: sanitized.createdAt }) + .where(eq(schema.project.chatSessions.id, sanitized.sessionId)); + return sanitized; } /** @@ -432,23 +449,36 @@ export async function addChatRoomMessage( handle: QueryHandle, message: ChatRoomMessage, ): Promise { + // FNXC:PostgresMigrationNulSanitize 2026-07-20: same NUL-byte hazard as + // addChatMessage above — sanitize before insert, and return the sanitized + // value so the in-memory result matches what was persisted. + const sanitizedAttachments = message.attachments === undefined + ? undefined + : sanitizeJsonbValue(message.attachments); + const sanitized: ChatRoomMessage = { + ...message, + content: sanitizeTextValue(message.content), + thinkingOutput: sanitizeTextValue(message.thinkingOutput), + metadata: sanitizeJsonbValue(message.metadata), + attachments: sanitizedAttachments, + }; await handle.insert(schema.project.chatRoomMessages).values({ - id: message.id, - roomId: message.roomId, - role: message.role, - content: message.content, - thinkingOutput: message.thinkingOutput, - metadata: message.metadata, - attachments: message.attachments ?? null, - senderAgentId: message.senderAgentId, - mentions: message.mentions, - createdAt: message.createdAt, + id: sanitized.id, + roomId: sanitized.roomId, + role: sanitized.role, + content: sanitized.content, + thinkingOutput: sanitized.thinkingOutput, + metadata: sanitized.metadata, + attachments: sanitizedAttachments ?? null, + senderAgentId: sanitized.senderAgentId, + mentions: sanitized.mentions, + createdAt: sanitized.createdAt, }); await handle .update(schema.project.chatRooms) - .set({ updatedAt: message.createdAt }) - .where(eq(schema.project.chatRooms.id, message.roomId)); - return message; + .set({ updatedAt: sanitized.createdAt }) + .where(eq(schema.project.chatRooms.id, sanitized.roomId)); + return sanitized; } /** diff --git a/packages/core/src/async-message-store.ts b/packages/core/src/async-message-store.ts index 69affaf878..bdd0f25835 100644 --- a/packages/core/src/async-message-store.ts +++ b/packages/core/src/async-message-store.ts @@ -22,6 +22,7 @@ import { and, desc, eq, inArray, lte, or, sql } from "drizzle-orm"; import { randomUUID } from "node:crypto"; import * as schema from "./postgres/schema/index.js"; import type { AsyncDataLayer, DbTransaction } from "./postgres/data-layer.js"; +import { sanitizeTextValue, sanitizeJsonbValue } from "./postgres/nul-sanitize.js"; import { DASHBOARD_USER_ID, type Message, @@ -96,21 +97,32 @@ export async function sendMessage( handle: QueryHandle, message: PersistedMessage, ): Promise { + // FNXC:PostgresMigrationNulSanitize 2026-07-20: agent-to-agent/agent-to-user + // mailbox content can carry raw tool output (including a stray NUL byte + // from piped Windows CLI dumps), which Postgres text/jsonb columns reject + // outright. Sanitize before insert instead of letting the send throw, and + // return the sanitized value so the in-memory result matches what was + // persisted (the original unsanitized `message` object must not be handed + // back to the caller). + const sanitizedContent = sanitizeTextValue(message.content); + const sanitizedMetadata = sanitizeJsonbValue(message.metadata); await handle.insert(schema.project.messages).values({ id: message.id, fromId: message.fromId, fromType: message.fromType, toId: message.toId, toType: message.toType, - content: message.content, + content: sanitizedContent, type: message.type, read: message.read ? 1 : 0, - metadata: message.metadata, + metadata: sanitizedMetadata, createdAt: message.createdAt, updatedAt: message.updatedAt, }); return rowToMessage({ ...message, + content: sanitizedContent, + metadata: sanitizedMetadata, read: message.read ? 1 : 0, }); } diff --git a/packages/core/src/message-store.ts b/packages/core/src/message-store.ts index 56826a797a..bb1817d0da 100644 --- a/packages/core/src/message-store.ts +++ b/packages/core/src/message-store.ts @@ -18,6 +18,7 @@ import { createLogger } from "./logger.js"; import { DASHBOARD_USER_ID, normalizeMessageParticipant, validateMessageMetadata, type Message, type MessageCreateInput, type MessageFilter, type MessageType, type Mailbox, type ParticipantType } from "./types.js"; import type { AsyncDataLayer } from "./postgres/data-layer.js"; import * as asyncMessageStore from "./async-message-store.js"; +import { sanitizeTextValue, sanitizeJsonbValue } from "./postgres/nul-sanitize.js"; const messageStoreLog = createLogger("message-store"); @@ -176,10 +177,17 @@ export class MessageStore extends EventEmitter { fromType: from.type, toId: to.id, toType: to.type, - content: input.content, + // FNXC:PostgresMigrationNulSanitize 2026-07-21: sanitize here so the + // exact same object is persisted, emitted (message:sent/message:received, + // consumed by the agent wake hook), and returned to the caller. The + // async-message-store.ts sendMessage() layer also sanitizes before its + // own insert, but this class discarded that function's return value and + // used its own locally-built (unsanitized) `message` object for + // everything else — this closes that gap. + content: sanitizeTextValue(input.content), type: input.type, read: false, - metadata: input.metadata, + metadata: sanitizeJsonbValue(input.metadata), createdAt: now, updatedAt: now, }; @@ -263,10 +271,15 @@ export class MessageStore extends EventEmitter { fromType: from.type, toId: to.id, toType: to.type, - content: input.content, + // FNXC:PostgresMigrationNulSanitize 2026-07-21: see sendMessage() above + // for why this must sanitize here rather than rely solely on + // asyncMessageStore.sendMessageOnce's own internal sanitization — + // that function only returns a boolean, so this class's locally-built + // `message` object is what actually gets emitted/returned. + content: sanitizeTextValue(input.content), type: input.type, read: false, - metadata: input.metadata, + metadata: sanitizeJsonbValue(input.metadata), createdAt: now, updatedAt: now, }; diff --git a/packages/core/src/postgres/nul-sanitize.ts b/packages/core/src/postgres/nul-sanitize.ts new file mode 100644 index 0000000000..02e8e9634d --- /dev/null +++ b/packages/core/src/postgres/nul-sanitize.ts @@ -0,0 +1,67 @@ +/* +FNXC:PostgresMigrationNulSanitize 2026-07-17-10:05 (extracted 2026-07-20): +PostgreSQL rejects U+0000 in text/varchar ("invalid byte sequence" / "\u0000 +cannot be converted to text.") and in json/jsonb ("unsupported Unicode escape +sequence"). Any write path that persists free-form agent/tool output as +Postgres text or jsonb — chat messages, chat room messages, and the +agent/user mailbox — can receive a raw NUL byte from unsanitized upstream +tool output (e.g. Windows CLI dumps piped straight into a message body) and +crash the write with an uncaught PostgresError. + +This module was originally written for the one-time SQLite -> PostgreSQL +first-boot migration (see sqlite-migrator.ts) and is promoted here so live +write paths (async-chat-store.ts, async-message-store.ts) can reuse the same +tested sanitization instead of duplicating it or, worse, not sanitizing at +all. +*/ + +// eslint-disable-next-line no-control-regex -- matching the NUL control character is the point +const NUL_CHAR_RE = /\u0000/g; + +/** Strip U+0000 (NUL) characters from a plain string. Cheap no-op when absent. */ +export function stripNulChars(text: string): string { + return text.includes("\u0000") ? text.replace(NUL_CHAR_RE, "") : text; +} + +/** + * Recursively strip U+0000 from all string values and object keys in a + * parsed JSON-like value (object/array/string/number/boolean/null). + */ +export function deepStripNulChars(value: unknown): unknown { + if (typeof value === "string") { + return stripNulChars(value); + } + if (Array.isArray(value)) { + return value.map(deepStripNulChars); + } + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value as Record).map( + ([key, entry]) => [stripNulChars(key), deepStripNulChars(entry)], + ), + ); + } + return value; +} + +/** + * Sanitize a value that Drizzle will bind as a `text`/`varchar` column. + * Strings are NUL-stripped; everything else (including null/undefined) + * passes through unchanged. + */ +export function sanitizeTextValue(value: T): T { + return (typeof value === "string" ? (stripNulChars(value) as unknown as T) : value); +} + +/** + * Sanitize a value that Drizzle will bind as a `jsonb` column. Accepts + * either a JS value (object/array/etc, including null/undefined) that will + * be strip-cleaned recursively, since Drizzle's jsonb columns take JS values + * directly (not JSON strings) at the call sites this is used from. + */ +export function sanitizeJsonbValue(value: T): T { + if (value === null || value === undefined) { + return value; + } + return deepStripNulChars(value) as T; +} \ No newline at end of file diff --git a/packages/core/src/postgres/sqlite-migrator.ts b/packages/core/src/postgres/sqlite-migrator.ts index 671f9f9949..5a1735e6e4 100644 --- a/packages/core/src/postgres/sqlite-migrator.ts +++ b/packages/core/src/postgres/sqlite-migrator.ts @@ -1412,33 +1412,15 @@ function classifyColumnType(pgCol: { } /* -FNXC:PostgresMigrationNulSanitize 2026-07-17-10:05: +FNXC:PostgresMigrationNulSanitize 2026-07-17-10:05 (moved to nul-sanitize.ts 2026-07-20): PostgreSQL rejects U+0000 in text/varchar ("invalid byte sequence" / "\u0000 cannot be converted to text") and in json/jsonb ("unsupported Unicode escape sequence"), but SQLite TEXT stores it freely, so legacy databases can contain NUL bytes that abort the first-boot auto-migration. Strip U+0000 from every migrated string — plain text cells, string values and object keys inside JSON documents, and opaque legacy-preservation text cells — rather than failing the cutover. Sanitization happens inside convertValue/tagLegacyCell so the content-checksum verification (computeSourceCanonicalRows reuses convertValue) compares the sanitized source against the sanitized target and still passes. + +stripNulChars/deepStripNulChars now live in ./nul-sanitize.js and are also +reused by the live chat/mailbox write paths (async-chat-store.ts, +async-message-store.ts), which can receive the same U+0000 byte from +unsanitized tool output at runtime, not just from legacy SQLite migration. */ -// eslint-disable-next-line no-control-regex -- matching the NUL control character is the point -const NUL_CHAR_RE = /\u0000/g; - -function stripNulChars(text: string): string { - return text.includes("\u0000") ? text.replace(NUL_CHAR_RE, "") : text; -} - -/** Recursively strip U+0000 from all string values and object keys in a parsed JSON document. */ -function deepStripNulChars(value: unknown): unknown { - if (typeof value === "string") { - return stripNulChars(value); - } - if (Array.isArray(value)) { - return value.map(deepStripNulChars); - } - if (value !== null && typeof value === "object") { - return Object.fromEntries( - Object.entries(value as Record).map( - ([key, entry]) => [stripNulChars(key), deepStripNulChars(entry)], - ), - ); - } - return value; -} +import { stripNulChars, deepStripNulChars } from "./nul-sanitize.js"; /** * Convert a SQLite value to its PostgreSQL representation based on the column diff --git a/packages/core/src/postgres/startup-factory.ts b/packages/core/src/postgres/startup-factory.ts index a45e203360..8dc5bad7eb 100644 --- a/packages/core/src/postgres/startup-factory.ts +++ b/packages/core/src/postgres/startup-factory.ts @@ -281,6 +281,35 @@ class NonUtf8EmbeddedClusterError extends Error { } } +/* +FNXC:PostgresStartupRace 2026-07-20-22:10: +Companion to embedded-lifecycle.ts's FNXC:PostgresStartupRace 2026-07-15-20:45 +note: isAlreadyRunning() joins a data dir optimistically off postmaster.pid +without probing liveness (a stale pid file from a crash must still resolve to +a port, by design), and ensureJoinedDatabase() is explicitly best-effort — +"never convert an optimistic join into a hard startup failure". But nothing +downstream actually retried: a joiner that raced the true owner's TCP bind +(process started, postmaster.pid written and "database system is ready" +logged, but the listener not yet accepting) got exactly one shot at +createConnectionSetFromUrl(), which throws ECONNREFUSED, which +bootSchemaBackendOnce turns into a hard `startup-factory: failed to +initialize PostgreSQL schema backend` error — observed reproducibly via +`pnpm smoke:boot`'s isolated embedded-instance boot. This class marks that +one specific case (joined instance, i.e. embeddedOwnsProcess === false, +connection-refused) as retryable, mirroring the existing +NonUtf8EmbeddedClusterError one-retry pattern below. +*/ +class JoinedInstanceUnreachableError extends Error { + constructor(override readonly cause: unknown) { + super("embedded postgres: joined instance was not yet accepting connections"); + } +} + +/** Matches a TCP-level connection-refused failure (ECONNREFUSED / "connect ECONNREFUSED"). */ +function isConnectionRefusedError(chainText: string): boolean { + return /ECONNREFUSED/i.test(chainText); +} + /** Matches PostgreSQL's encoding-conversion failure raised by a non-UTF-8 cluster. */ export function isEncodingConversionError(chainText: string): boolean { return /has no equivalent in encoding/i.test(chainText); @@ -328,6 +357,15 @@ async function bootSchemaBackend( try { return await bootSchemaBackendOnce(options, bypassProjectIsolation); } catch (error) { + if (error instanceof JoinedInstanceUnreachableError) { + log.warn( + "startup-factory: joined embedded postgres instance was not yet accepting connections " + + "(startup race with the true owner's TCP bind — see FNXC:PostgresStartupRace 2026-07-20-22:10). " + + "Retrying once after a short delay.", + ); + await new Promise((resolve) => setTimeout(resolve, 500)); + return await bootSchemaBackendOnce(options, bypassProjectIsolation); + } if (!(error instanceof NonUtf8EmbeddedClusterError)) throw error; log.warn( `startup-factory: embedded cluster at ${error.dataDir} was created with a non-UTF-8 OS-locale ` + @@ -443,6 +481,18 @@ async function bootSchemaBackendOnce( ) { recoverable = await isEmptyNonUtf8Cluster(connections.migration).catch(() => false); } + /* + FNXC:PostgresStartupRace 2026-07-20-22:10: + A joined (not owned) embedded instance that fails with a connection-refused + is the documented startup race (embedded-lifecycle.ts FNXC:PostgresStartupRace + 2026-07-15-20:45): we joined off postmaster.pid before the true owner's + listener was accepting connections. This is retryable — see bootSchemaBackend's + JoinedInstanceUnreachableError handling — rather than a hard failure. + */ + const joinedConnectionRefused = + embeddedLifecycle !== null && + !embeddedOwnsProcess && + isConnectionRefusedError(error instanceof Error ? `${error.message} ${String(error.cause ?? "")}` : String(error)); await connections?.close().catch(() => undefined); await stopEmbeddedRuntime( embeddedLifecycle, @@ -453,6 +503,9 @@ async function bootSchemaBackendOnce( if (recoverable && embeddedDataDir !== null) { throw new NonUtf8EmbeddedClusterError(embeddedDataDir, error); } + if (joinedConnectionRefused) { + throw new JoinedInstanceUnreachableError(error); + } throw error; } }