diff --git a/.changeset/fn-8127-embedded-backup.md b/.changeset/fn-8127-embedded-backup.md new file mode 100644 index 0000000000..b5d193108b --- /dev/null +++ b/.changeset/fn-8127-embedded-backup.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Fix fn backup and scheduled database backups in the default embedded PostgreSQL setup. +category: fix +dev: Tracks embedded runtime URLs with owner/joiner generation leases so stale shutdowns cannot target or clear a newer cluster. diff --git a/docs/settings-reference.md b/docs/settings-reference.md index 90911dc57a..e4a8d06483 100644 --- a/docs/settings-reference.md +++ b/docs/settings-reference.md @@ -673,6 +673,9 @@ GitLab configuration examples: leave both URL fields blank for GitLab.com (`http | `autoBackupSchedule` | `string` | `"0 2 * * *"` | Backup cron schedule. | | `autoBackupRetention` | `number` | `7` | Number of backups to retain. | | `autoBackupDir` | `string` | `".fusion/backups"` | Relative backup directory path. | + +Database backups work with both external PostgreSQL and Fusion's default embedded PostgreSQL deployment. `fn backup` and the built-in **Database Backup** cron/routine use `pg_dump` and `pg_restore`; install PostgreSQL client tools or configure their paths so both executables are available on `PATH`. They are not bundled with `embedded-postgres`. + | `memoryBackupEnabled` | `boolean` | `false` | Enable scheduled memory backups. | | `memoryBackupSchedule` | `string` | `"0 3 * * *"` | Memory backup cron schedule. | | `memoryBackupRetention` | `number` | `14` | Number of memory backups to retain. | diff --git a/packages/core/src/__tests__/backup.test.ts b/packages/core/src/__tests__/backup.test.ts new file mode 100644 index 0000000000..9692e965a1 --- /dev/null +++ b/packages/core/src/__tests__/backup.test.ts @@ -0,0 +1,115 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + BackupManager, + createBackupManager, + resolveBackendConnectionString, +} from "../backup.js"; +import { + clearActiveEmbeddedRuntimeUrl, + getActiveEmbeddedRuntimeUrl, + invalidateEmbeddedRuntimeUrl, + registerEmbeddedRuntimeUrl, + releaseEmbeddedRuntimeLease, +} from "../postgres/active-backend-registry.js"; + +const embeddedUrl = "postgresql://postgres:embedded-secret@127.0.0.1:55432/fusion"; +const externalUrl = "postgresql://operator:external-secret@db.example.test:5432/fusion"; + +afterEach(() => { + clearActiveEmbeddedRuntimeUrl(); + vi.unstubAllEnvs(); +}); + +describe("embedded backup runtime URL registry", () => { + it("resolves a registered embedded backend and lets BackupManager construct", () => { + vi.stubEnv("DATABASE_URL", ""); + registerEmbeddedRuntimeUrl(embeddedUrl, { ownsProcess: true }); + + expect(resolveBackendConnectionString()).toBe(embeddedUrl); + expect(() => createBackupManager("/tmp/project/.fusion")).not.toThrow(); + }); + + it("keeps an external DATABASE_URL ahead of the embedded registry", () => { + vi.stubEnv("DATABASE_URL", externalUrl); + registerEmbeddedRuntimeUrl(embeddedUrl, { ownsProcess: true }); + + expect(resolveBackendConnectionString()).toBe(externalUrl); + }); + + it("preserves the actionable error before an embedded lifecycle boots", () => { + vi.stubEnv("DATABASE_URL", ""); + + expect(resolveBackendConnectionString()).toBeUndefined(); + expect(() => new BackupManager("/tmp/project/.fusion")).toThrow( + "BackupManager requires a PostgreSQL connection string", + ); + }); + + it("keeps an owner URL live when only a joiner releases", () => { + const owner = registerEmbeddedRuntimeUrl(embeddedUrl, { ownsProcess: true }); + const joiner = registerEmbeddedRuntimeUrl(embeddedUrl, { ownsProcess: false }); + + releaseEmbeddedRuntimeLease(joiner); + expect(getActiveEmbeddedRuntimeUrl()).toBe(embeddedUrl); + + releaseEmbeddedRuntimeLease(owner); + expect(getActiveEmbeddedRuntimeUrl()).toBeUndefined(); + }); + + it("invalidates every joiner when the postmaster owner stops", () => { + registerEmbeddedRuntimeUrl(embeddedUrl, { ownsProcess: true }); + registerEmbeddedRuntimeUrl(embeddedUrl, { ownsProcess: false }); + + invalidateEmbeddedRuntimeUrl(embeddedUrl); + expect(resolveBackendConnectionString()).toBeUndefined(); + expect(() => new BackupManager("/tmp/project/.fusion")).toThrow( + "BackupManager requires a PostgreSQL connection string", + ); + }); + + it("makes an old joiner release inert after owner invalidation and re-registration", () => { + registerEmbeddedRuntimeUrl(embeddedUrl, { ownsProcess: true }); + const oldJoiner = registerEmbeddedRuntimeUrl(embeddedUrl, { ownsProcess: false }); + invalidateEmbeddedRuntimeUrl(embeddedUrl); + registerEmbeddedRuntimeUrl(embeddedUrl, { ownsProcess: true }); + + releaseEmbeddedRuntimeLease(oldJoiner); + expect(getActiveEmbeddedRuntimeUrl()).toBe(embeddedUrl); + }); + + it("makes leases from a test reset inert for a re-registered URL", () => { + const oldLease = registerEmbeddedRuntimeUrl(embeddedUrl, { ownsProcess: false }); + clearActiveEmbeddedRuntimeUrl(); + registerEmbeddedRuntimeUrl(embeddedUrl, { ownsProcess: true }); + + releaseEmbeddedRuntimeLease(oldLease); + expect(getActiveEmbeddedRuntimeUrl()).toBe(embeddedUrl); + }); + + it("does not let a stale owner invalidate a replacement cluster that reused its URL", () => { + const oldOwner = registerEmbeddedRuntimeUrl(embeddedUrl, { ownsProcess: true }); + const replacementOwner = registerEmbeddedRuntimeUrl(embeddedUrl, { ownsProcess: true }); + + invalidateEmbeddedRuntimeUrl(embeddedUrl, oldOwner); + expect(getActiveEmbeddedRuntimeUrl()).toBe(embeddedUrl); + + releaseEmbeddedRuntimeLease(replacementOwner); + expect(getActiveEmbeddedRuntimeUrl()).toBeUndefined(); + }); + + it("uses the last live registration and ignores unknown invalidation/releases", () => { + const firstUrl = "postgresql://postgres:a@127.0.0.1:55431/fusion"; + const first = registerEmbeddedRuntimeUrl(firstUrl, { ownsProcess: true }); + const second = registerEmbeddedRuntimeUrl(embeddedUrl, { ownsProcess: false }); + expect(getActiveEmbeddedRuntimeUrl()).toBe(embeddedUrl); + + invalidateEmbeddedRuntimeUrl("postgresql://unknown@127.0.0.1:9/missing"); + releaseEmbeddedRuntimeLease({} as never); + expect(getActiveEmbeddedRuntimeUrl()).toBe(embeddedUrl); + + releaseEmbeddedRuntimeLease(second); + expect(getActiveEmbeddedRuntimeUrl()).toBe(firstUrl); + releaseEmbeddedRuntimeLease(first); + expect(getActiveEmbeddedRuntimeUrl()).toBeUndefined(); + }); +}); diff --git a/packages/core/src/backup.ts b/packages/core/src/backup.ts index 46a3e9f228..bdd77b7a1e 100644 --- a/packages/core/src/backup.ts +++ b/packages/core/src/backup.ts @@ -3,6 +3,7 @@ import { CronExpressionParser } from "cron-parser"; import { getDefaultCentralDbPath } from "./central-db.js"; import { PgBackupManager, type PgBackupPair, type PgDumpResult } from "./postgres/pg-backup.js"; import { resolveBackend } from "./postgres/backend-resolver.js"; +import { getActiveEmbeddedRuntimeUrl } from "./postgres/active-backend-registry.js"; import type { ProjectSettings } from "./types.js"; export interface BackupFileInfo { @@ -247,20 +248,18 @@ export function createBackupManager( } /** - * FNXC:BackendFlip 2026-06-26-14:35: - * Resolve the PostgreSQL connection string for backup operations from the - * runtime backend. Returns the runtime URL when the backend is external - * (DATABASE_URL set). Returns undefined for embedded mode (the default - * production path since flip-embedded-pg-default when DATABASE_URL is unset), - * because the embedded lifecycle provides its URL asynchronously at startup - * and cannot be resolved synchronously here. + * FNXC:PostgresBackup 2026-07-16-12:40: + * External deployments resolve directly from DATABASE_URL. Embedded PostgreSQL + * learns its URL only during asynchronous startup, so use the active lifecycle + * registry as the synchronous fallback. It is intentionally undefined before + * boot or after owner shutdown, preserving BackupManager's actionable error. */ -function resolveBackendConnectionString(): string | undefined { +export function resolveBackendConnectionString(): string | undefined { const backend = resolveBackend(); if (backend.mode === "external" && backend.runtimeUrl) { return backend.runtimeUrl; } - return undefined; + return getActiveEmbeddedRuntimeUrl(); } /* diff --git a/packages/core/src/index.gate.ts b/packages/core/src/index.gate.ts index 65ad33aa4b..54e1c91fd7 100644 --- a/packages/core/src/index.gate.ts +++ b/packages/core/src/index.gate.ts @@ -1305,8 +1305,17 @@ export { syncBackupAutomation, syncBackupRoutine, BACKUP_SCHEDULE_NAME, + resolveBackendConnectionString, } from "./backup.js"; export type { BackupInfo, BackupOptions, BackupFileInfo, BackupPairInfo } from "./backup.js"; +export { + registerEmbeddedRuntimeUrl, + releaseEmbeddedRuntimeLease, + invalidateEmbeddedRuntimeUrl, + getActiveEmbeddedRuntimeUrl, + clearActiveEmbeddedRuntimeUrl, +} from "./postgres/active-backend-registry.js"; +export type { EmbeddedRuntimeLease } from "./postgres/active-backend-registry.js"; export { MemoryBackupManager, createMemoryBackupManager, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 8787902481..ad5a081ba0 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1357,8 +1357,17 @@ export { syncBackupAutomation, syncBackupRoutine, BACKUP_SCHEDULE_NAME, + resolveBackendConnectionString, } from "./backup.js"; export type { BackupInfo, BackupOptions, BackupFileInfo, BackupPairInfo } from "./backup.js"; +export { + registerEmbeddedRuntimeUrl, + releaseEmbeddedRuntimeLease, + invalidateEmbeddedRuntimeUrl, + getActiveEmbeddedRuntimeUrl, + clearActiveEmbeddedRuntimeUrl, +} from "./postgres/active-backend-registry.js"; +export type { EmbeddedRuntimeLease } from "./postgres/active-backend-registry.js"; export { MemoryBackupManager, createMemoryBackupManager, diff --git a/packages/core/src/postgres/active-backend-registry.ts b/packages/core/src/postgres/active-backend-registry.ts new file mode 100644 index 0000000000..b3b0ad1eab --- /dev/null +++ b/packages/core/src/postgres/active-backend-registry.ts @@ -0,0 +1,119 @@ +/* + * FNXC:PostgresBackup 2026-07-16-12:40: + * Embedded PostgreSQL learns its credential-bearing runtime URL asynchronously, + * while backup construction resolves synchronously. This process-local registry + * bridges that gap without logging credentials. Leases represent individual + * lifecycles within a physical cluster generation: a joiner's release cannot + * clear a newer generation, and owner shutdown invalidates every lease because + * it is the only lifecycle that actually stops the postmaster. + */ + +/** Opaque handle for one embedded-backend lifecycle registration. */ +declare const embeddedRuntimeLeaseBrand: unique symbol; +export interface EmbeddedRuntimeLease { + readonly [embeddedRuntimeLeaseBrand]: true; +} + +interface Generation { + readonly url: string; + readonly epoch: number; + readonly id: number; + readonly leases: Set; + latestRegistration: number; +} + +interface LeaseMetadata { + readonly url: string; + readonly epoch: number; + readonly generation: number; + readonly ownsProcess: boolean; +} + +const generationsByUrl = new Map(); +const nextGenerationByUrl = new Map(); +const leaseMetadata = new WeakMap(); +let registrationSequence = 0; +let registryEpoch = 0; + +/** Register a booted embedded lifecycle and return its release-only lease. */ +export function registerEmbeddedRuntimeUrl( + url: string, + options: { ownsProcess: boolean }, +): EmbeddedRuntimeLease { + let generation = generationsByUrl.get(url); + // FNXC:PostgresBackup 2026-07-16-12:40: An owner started a new postmaster, + // so URL reuse must create a new generation rather than retain stale leases. + if (!generation || options.ownsProcess) { + const id = (nextGenerationByUrl.get(url) ?? 0) + 1; + nextGenerationByUrl.set(url, id); + generation = { url, epoch: registryEpoch, id, leases: new Set(), latestRegistration: 0 }; + generationsByUrl.set(url, generation); + } + + const lease = {} as EmbeddedRuntimeLease; + generation.leases.add(lease); + generation.latestRegistration = ++registrationSequence; + leaseMetadata.set(lease, { + url, + epoch: generation.epoch, + generation: generation.id, + ownsProcess: options.ownsProcess, + }); + return lease; +} + +/** Release exactly one lifecycle lease; stale generation handles are inert. */ +export function releaseEmbeddedRuntimeLease(lease: EmbeddedRuntimeLease): void { + const metadata = leaseMetadata.get(lease); + if (!metadata) return; + const generation = generationsByUrl.get(metadata.url); + if ( + !generation + || generation.epoch !== metadata.epoch + || generation.id !== metadata.generation + ) return; + + generation.leases.delete(lease); + if (generation.leases.size === 0) { + generationsByUrl.delete(metadata.url); + } +} + +/** + * Invalidate all leases for a cluster generation after its owner stops it. + * A lease-aware invalidation cannot remove a newer cluster that reused the URL. + */ +export function invalidateEmbeddedRuntimeUrl(url: string, lease?: EmbeddedRuntimeLease): void { + if (!lease) { + generationsByUrl.delete(url); + return; + } + const metadata = leaseMetadata.get(lease); + const generation = generationsByUrl.get(url); + if ( + metadata?.url === url + && generation?.epoch === metadata.epoch + && generation.id === metadata.generation + ) { + generationsByUrl.delete(url); + } +} + +/** Return the most recently registered URL whose generation remains live. */ +export function getActiveEmbeddedRuntimeUrl(): string | undefined { + let latest: Generation | undefined; + for (const generation of generationsByUrl.values()) { + if (generation.leases.size > 0 && (!latest || generation.latestRegistration > latest.latestRegistration)) { + latest = generation; + } + } + return latest?.url; +} + +/** Reset process-local state for isolated tests. */ +export function clearActiveEmbeddedRuntimeUrl(): void { + generationsByUrl.clear(); + nextGenerationByUrl.clear(); + registrationSequence = 0; + registryEpoch += 1; +} diff --git a/packages/core/src/postgres/embedded-lifecycle.ts b/packages/core/src/postgres/embedded-lifecycle.ts index ac14444aea..49652d29c9 100644 --- a/packages/core/src/postgres/embedded-lifecycle.ts +++ b/packages/core/src/postgres/embedded-lifecycle.ts @@ -966,6 +966,11 @@ export class EmbeddedPostgresLifecycle { return this.options.port ?? this.resolvedPort; } + /** True when this lifecycle started the postmaster rather than joining it. */ + getOwnsProcess(): boolean { + return this.ownsProcess; + } + /** True when the embedded postgres process is currently running. */ isRunning(): boolean { return this.running; diff --git a/packages/core/src/postgres/index.ts b/packages/core/src/postgres/index.ts index d1b32d2acb..d6b945be22 100644 --- a/packages/core/src/postgres/index.ts +++ b/packages/core/src/postgres/index.ts @@ -37,6 +37,15 @@ export { type CreateConnectionOptions, } from "./connection.js"; +export { + registerEmbeddedRuntimeUrl, + releaseEmbeddedRuntimeLease, + invalidateEmbeddedRuntimeUrl, + getActiveEmbeddedRuntimeUrl, + clearActiveEmbeddedRuntimeUrl, + type EmbeddedRuntimeLease, +} from "./active-backend-registry.js"; + export { redactUrlPassword, redactUrlQueryPassword, diff --git a/packages/core/src/postgres/startup-factory.ts b/packages/core/src/postgres/startup-factory.ts index 544bf140d2..00148b0ecd 100644 --- a/packages/core/src/postgres/startup-factory.ts +++ b/packages/core/src/postgres/startup-factory.ts @@ -55,6 +55,12 @@ import { } from "./connection.js"; import { applySchemaBaseline } from "./schema-applier.js"; import { createAsyncDataLayer, type AsyncDataLayer } from "./data-layer.js"; +import { + invalidateEmbeddedRuntimeUrl, + registerEmbeddedRuntimeUrl, + releaseEmbeddedRuntimeLease, + type EmbeddedRuntimeLease, +} from "./active-backend-registry.js"; import { runLoadedPluginSchemaInitHooks, type LoadedPluginSchemaContract } from "./plugin-schema-hook.js"; import { lookupRegisteredProjectIdByPath, @@ -76,6 +82,7 @@ import { type EmbeddedLifecycleLike = { start(): Promise; stop(): Promise; + getOwnsProcess(): boolean; }; const log = createLogger("startup-factory"); @@ -166,6 +173,35 @@ interface SchemaBackendBootResult { readonly backend: ResolvedBackend; readonly connections: PostgresConnections; readonly embeddedLifecycle: EmbeddedLifecycleLike | null; + readonly embeddedRuntimeLease: EmbeddedRuntimeLease | null; + readonly embeddedRuntimeUrl: string | null; + readonly embeddedOwnsProcess: boolean; +} + +/** + * FNXC:PostgresBackup 2026-07-16-12:40: + * stop() only stops a postmaster owned by this lifecycle. Consequently owner + * teardown burns the URL generation for every joiner, while joiner teardown + * releases only its opaque lease. This keeps synchronous backup resolution + * aligned with physical cluster liveness without logging the credential URL. + */ +async function stopEmbeddedRuntime( + lifecycle: EmbeddedLifecycleLike | null, + lease: EmbeddedRuntimeLease | null, + runtimeUrl: string | null, + ownsProcess: boolean, +): Promise { + try { + await lifecycle?.stop(); + } finally { + if (lease && runtimeUrl) { + if (ownsProcess) { + invalidateEmbeddedRuntimeUrl(runtimeUrl, lease); + } else { + releaseEmbeddedRuntimeLease(lease); + } + } + } } /** @@ -189,6 +225,9 @@ async function bootSchemaBackend( } let embeddedLifecycle: EmbeddedLifecycleLike | null = null; + let embeddedRuntimeLease: EmbeddedRuntimeLease | null = null; + let embeddedRuntimeUrl: string | null = null; + let embeddedOwnsProcess = false; let resolvedBackend = backend; if (backend.mode === "embedded") { const { EmbeddedPostgresLifecycle, defaultEmbeddedDataDir, DEFAULT_EMBEDDED_DATABASE } = @@ -203,6 +242,13 @@ async function bootSchemaBackend( }); try { resolvedBackend = await embeddedLifecycle.start(); + if (resolvedBackend.runtimeUrl) { + embeddedRuntimeUrl = resolvedBackend.runtimeUrl; + embeddedOwnsProcess = embeddedLifecycle.getOwnsProcess(); + embeddedRuntimeLease = registerEmbeddedRuntimeUrl(embeddedRuntimeUrl, { + ownsProcess: embeddedOwnsProcess, + }); + } } catch (error) { await embeddedLifecycle.stop().catch(() => undefined); throw new Error( @@ -230,10 +276,22 @@ async function bootSchemaBackend( bypassProjectIsolation, }); await applySchemaBaseline(connections.migration); - return { backend: resolvedBackend, connections, embeddedLifecycle }; + return { + backend: resolvedBackend, + connections, + embeddedLifecycle, + embeddedRuntimeLease, + embeddedRuntimeUrl, + embeddedOwnsProcess, + }; } catch (error) { await connections?.close().catch(() => undefined); - await embeddedLifecycle?.stop().catch(() => undefined); + await stopEmbeddedRuntime( + embeddedLifecycle, + embeddedRuntimeLease, + embeddedRuntimeUrl, + embeddedOwnsProcess, + ).catch(() => undefined); throw error; } } @@ -252,7 +310,14 @@ export async function createCentralBackendLayer( options: Pick = {}, ): Promise { const boot = await bootSchemaBackend(options, true); - const { backend: resolvedBackend, connections, embeddedLifecycle } = boot; + const { + backend: resolvedBackend, + connections, + embeddedLifecycle, + embeddedRuntimeLease, + embeddedRuntimeUrl, + embeddedOwnsProcess, + } = boot; try { /* FNXC:CentralPostgresCutover 2026-07-14-19:06: @@ -301,12 +366,22 @@ export async function createCentralBackendLayer( releaseConnections, async shutdown(): Promise { await releaseConnections(); - await embeddedLifecycle?.stop().catch(() => undefined); + await stopEmbeddedRuntime( + embeddedLifecycle, + embeddedRuntimeLease, + embeddedRuntimeUrl, + embeddedOwnsProcess, + ).catch(() => undefined); }, }; } catch (error) { await connections.close().catch(() => undefined); - await embeddedLifecycle?.stop().catch(() => undefined); + await stopEmbeddedRuntime( + embeddedLifecycle, + embeddedRuntimeLease, + embeddedRuntimeUrl, + embeddedOwnsProcess, + ).catch(() => undefined); throw error; } } @@ -444,7 +519,13 @@ export async function createTaskStoreForBackend( ); } let { connections } = boot; - const { backend: resolvedBackend, embeddedLifecycle } = boot; + const { + backend: resolvedBackend, + embeddedLifecycle, + embeddedRuntimeLease, + embeddedRuntimeUrl, + embeddedOwnsProcess, + } = boot; /* FNXC:PostgresMigration 2026-07-10: @@ -661,9 +742,12 @@ export async function createTaskStoreForBackend( } } catch (err) { await connections.close().catch(() => undefined); - if (embeddedLifecycle) { - await embeddedLifecycle.stop().catch(() => undefined); - } + await stopEmbeddedRuntime( + embeddedLifecycle, + embeddedRuntimeLease, + embeddedRuntimeUrl, + embeddedOwnsProcess, + ).catch(() => undefined); throw new Error( `startup-factory: SQLite → PostgreSQL first-boot auto-migration failed (refusing to boot an empty database over existing SQLite data; restore the retained backup and run 'fn db migrate' manually): ${ err instanceof Error ? err.message : String(err) @@ -751,9 +835,12 @@ export async function createTaskStoreForBackend( log.log(`startup phase backend.taskStore.construct: ${Date.now() - constructT0}ms`); } catch (err) { await asyncLayer.close().catch(() => undefined); - if (embeddedLifecycle) { - await embeddedLifecycle.stop().catch(() => undefined); - } + await stopEmbeddedRuntime( + embeddedLifecycle, + embeddedRuntimeLease, + embeddedRuntimeUrl, + embeddedOwnsProcess, + ).catch(() => undefined); throw new Error( `startup-factory: failed to construct PostgreSQL-backed TaskStore: ${ err instanceof Error ? err.message : String(err) @@ -829,7 +916,12 @@ export async function createTaskStoreForBackend( } if (shutdownEmbedded) { try { - await shutdownEmbedded.stop(); + await stopEmbeddedRuntime( + shutdownEmbedded, + embeddedRuntimeLease, + embeddedRuntimeUrl, + embeddedOwnsProcess, + ); } catch (err) { log.warn(`startup-factory: embedded PostgreSQL stop failed during shutdown: ${ err instanceof Error ? err.message : String(err)