diff --git a/.changeset/fn-8523-skip-completed-sqlite-rescan.md b/.changeset/fn-8523-skip-completed-sqlite-rescan.md new file mode 100644 index 0000000000..0e04c87250 --- /dev/null +++ b/.changeset/fn-8523-skip-completed-sqlite-rescan.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Stop completed PostgreSQL migrations from re-scanning retained SQLite backups at startup. +category: fix +dev: Core, central, and plugin sources now honor their independent completion markers before SQLite access. diff --git a/docs/storage.md b/docs/storage.md index 0c6e76440e..3edc1105d9 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -8,6 +8,8 @@ See the [2026-07-14 PostgreSQL runtime cutover review](./postgres-migration-revi - During a first-boot cutover, `fn dashboard`, `fn serve`, and `fn daemon --port ` keep their known HTTP port available with a migration holding page. Open dashboard tabs poll `/api/health` and show the migration banner with live progress. - After a successful cutover, the usual dismissible data-migrated notice may appear. If the durable cutover marker remains `running` or `failed`, real-server `/api/health` reports `status: "degraded"` with migration detail and the dashboard keeps the migration banner visible. Do not delete retained legacy `.fusion/fusion.db` backups; check logs and run `fn db migrate` after fixing a failure. +- Retained `fusion.db`, `archive.db`, and `fusion-central.db` files are migration inputs and operator backups only. Startup reads a source only while its matching `fusion_sqlite_migrations` key is incomplete: `project:` gates core/archive/identity work, `central:legacy-sqlite` gates central work, and `project-plugins:` independently gates plugin adoption. A completed core marker never suppresses a still-incomplete plugin bridge. +- Root-directory startup resolves a core key from an explicit project ID or the PostgreSQL `central.projects` rootDir mapping before using the deterministic fallback. A migration that learns an ID from legacy central SQLite must first materialize the same canonical rootDir-to-ID mapping in PostgreSQL before recording completion, so future boots never need the retained database to rediscover identity. ## Embedded PostgreSQL startup resources diff --git a/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts b/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts index 5db70019e6..2f66a28c50 100644 --- a/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts +++ b/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts @@ -37,6 +37,8 @@ import { formatMigrationProgress, migrateLegacyProjectPluginRows, migrateSqliteToPostgres, + projectPluginSqliteMigrationKey, + recordSqliteMigrationComplete, toSnakeCase, type MigrationProgressEvent, } from "../../postgres/sqlite-migrator.js"; @@ -1298,6 +1300,19 @@ pgDescribe("SQLite-to-PostgreSQL migrator", () => { expect(preserved).toEqual([{ name: "Shared from B", enabled: 1 }]); }); + it("does not probe a retained plugin SQLite path after its marker completes", async () => { + await applySchemaBaseline(ctx!.db); + const projectPath = join(ctx!.fusionDir, "project-complete"); + await recordSqliteMigrationComplete( + ctx!.db, + projectPluginSqliteMigrationKey(projectPath), + ); + + // A directory is not a SQLite database and would throw if the bridge opened it. + await expect(migrateLegacyProjectPluginRows(ctx!.db, ctx!.fusionDir, projectPath)) + .resolves.toBeUndefined(); + }); + it("treats missing SQLite files and databases without plugins as a no-op", async () => { await applySchemaBaseline(ctx!.db); await expect(migrateLegacyProjectPluginRows( diff --git a/packages/core/src/__tests__/postgres/startup-factory-integration.test.ts b/packages/core/src/__tests__/postgres/startup-factory-integration.test.ts index e462af2672..63a53a32db 100644 --- a/packages/core/src/__tests__/postgres/startup-factory-integration.test.ts +++ b/packages/core/src/__tests__/postgres/startup-factory-integration.test.ts @@ -15,7 +15,7 @@ import { afterEach, describe, it, expect } from "vitest"; import { execSync } from "node:child_process"; import { mkdtemp, rm } from "node:fs/promises"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { tmpdir } from "node:os"; import { createTaskStoreForBackend } from "../../postgres/startup-factory.js"; import { getSqliteMigrationState } from "../../postgres/sqlite-migrator.js"; @@ -195,7 +195,7 @@ pgDescribe("startup-factory: external PostgreSQL boot (integration)", () => { const admin = postgres(testUrl, { max: 1 }); try { - await admin`CREATE TABLE public.fusion_sqlite_migrations ( + await admin`CREATE TABLE IF NOT EXISTS public.fusion_sqlite_migrations ( migration_key text PRIMARY KEY, project_id text, status text NOT NULL CHECK (status IN ('running', 'complete', 'failed')), @@ -285,7 +285,7 @@ pgDescribe("startup-factory: external PostgreSQL boot (integration)", () => { const admin = postgres(testUrl, { max: 1 }); try { - await admin`CREATE TABLE public.fusion_sqlite_migrations ( + await admin`CREATE TABLE IF NOT EXISTS public.fusion_sqlite_migrations ( migration_key text PRIMARY KEY, project_id text, status text NOT NULL CHECK (status IN ('running', 'complete', 'failed')), @@ -328,6 +328,111 @@ pgDescribe("startup-factory: external PostgreSQL boot (integration)", () => { } }); + it("does not reopen a completed central backup while a fallback project migration remains pending", async () => { + rootDir = await mkdtemp(join(tmpdir(), "startup-factory-completed-central-")); + const globalDir = await mkdtemp(join(tmpdir(), "startup-factory-completed-central-global-")); + dbName = uniqueDbName(); + adminExec(`CREATE DATABASE "${dbName}"`); + const testUrl = `${PG_TEST_URL_BASE}/${dbName}`; + + const first = await createTaskStoreForBackend({ + rootDir, + globalSettingsDir: globalDir, + env: { DATABASE_URL: testUrl }, + poolMax: 1, + }); + const fallbackProjectId = first.taskStore.getAsyncLayer()!.projectId!; + await first.shutdown(); + seedLegacyTask(rootDir, "FN-CENTRAL-MARKER-1", "Central marker boundary"); + seedLegacyRegistry(globalDir, [{ id: "must-not-resolve-from-sqlite", path: rootDir }]); + + const admin = postgres(testUrl, { max: 1 }); + try { + await admin` + INSERT INTO public.fusion_sqlite_migrations + (migration_key, project_id, status, last_error, updated_at) + VALUES ('central:legacy-sqlite', NULL, 'complete', NULL, now()) + `; + } finally { + await admin.end(); + } + + const result = await createTaskStoreForBackend({ + rootDir, + globalSettingsDir: globalDir, + env: { DATABASE_URL: testUrl }, + poolMax: 1, + }); + try { + expect(await getSqliteMigrationState( + result.taskStore.getAsyncLayer()!.db, + `project:${fallbackProjectId}`, + )).toMatchObject({ status: "complete" }); + expect(await getSqliteMigrationState( + result.taskStore.getAsyncLayer()!.db, + "project:must-not-resolve-from-sqlite", + )).toBeNull(); + } finally { + await result.shutdown(); + } + }); + + it("keeps completed core, central, and plugin SQLite backups inert across repeated boots", async () => { + rootDir = await mkdtemp(join(tmpdir(), "startup-factory-completed-backups-")); + const globalDir = await mkdtemp(join(tmpdir(), "startup-factory-completed-global-")); + dbName = uniqueDbName(); + adminExec(`CREATE DATABASE "${dbName}"`); + const testUrl = `${PG_TEST_URL_BASE}/${dbName}`; + const projectId = "completed-project"; + const first = await createTaskStoreForBackend({ + rootDir, + projectId, + globalSettingsDir: globalDir, + env: { DATABASE_URL: testUrl }, + poolMax: 1, + }); + await first.shutdown(); + seedLegacyTask(rootDir, "legacy-must-stay-inert", "inert backup task"); + seedLegacyPlugin(rootDir); + seedLegacyRegistry(globalDir, [{ id: projectId, path: rootDir }]); + + const admin = postgres(testUrl, { max: 1 }); + try { + await admin` + INSERT INTO public.fusion_sqlite_migrations + (migration_key, project_id, status, last_error, updated_at) + VALUES + (${`project:${projectId}`}, ${projectId}, 'complete', NULL, now()), + ('central:legacy-sqlite', NULL, 'complete', NULL, now()), + (${`project-plugins:${resolve(rootDir)}`}, NULL, 'complete', NULL, now()) + ON CONFLICT (migration_key) DO UPDATE SET status = 'complete', last_error = NULL + `; + } finally { + await admin.end(); + } + + for (let boot = 0; boot < 2; boot += 1) { + const result = await createTaskStoreForBackend({ + rootDir, + projectId, + globalSettingsDir: globalDir, + env: { DATABASE_URL: testUrl }, + poolMax: 1, + }); + try { + expect(await result.taskStore.listTasks()).not.toContainEqual( + expect.objectContaining({ id: "legacy-must-stay-inert" }), + ); + const installs = await result.taskStore.getAsyncLayer()!.db.execute(sql` + SELECT id FROM central.plugin_installs WHERE id = 'legacy-startup-plugin' + `); + expect(installs).toEqual([]); + } finally { + await result.shutdown(); + } + } + }); + /* * FNXC:PostgresMigration 2026-07-10: * First-boot auto-migration (review data-loss trap): booting the PG backend diff --git a/packages/core/src/postgres/sqlite-migrator.ts b/packages/core/src/postgres/sqlite-migrator.ts index 5a1735e6e4..c6e1f7080d 100644 --- a/packages/core/src/postgres/sqlite-migrator.ts +++ b/packages/core/src/postgres/sqlite-migrator.ts @@ -1069,12 +1069,25 @@ function normalizeLegacyJson(value: string | null, fallback: string): string { } } +/** Build the canonical, path-scoped migration key for retained plugin state. */ +export function projectPluginSqliteMigrationKey(projectPath: string): string { + return `project-plugins:${resolve(projectPath)}`; +} + /** Backfill the split PostgreSQL plugin model once from retained project SQLite. */ export async function migrateLegacyProjectPluginRows( db: PostgresJsDatabase>, sqlitePath: string, projectPath: string, ): Promise { + const migrationKey = projectPluginSqliteMigrationKey(projectPath); + /* + FNXC:PostgresMigration 2026-07-22-12:00: + Plugin migration is independently terminal. Read its PostgreSQL marker before + even probing retained fusion.db so completed backups cannot add startup I/O. + The transaction repeats the check under its advisory lock for concurrent boot. + */ + if (await isSqliteMigrationComplete(db, migrationKey)) return; await db.transaction(async (tx) => { await migrateLegacyProjectPluginRowsOnSession( tx as unknown as PostgresJsDatabase>, @@ -1089,23 +1102,18 @@ async function migrateLegacyProjectPluginRowsOnSession( sqlitePath: string, projectPath: string, ): Promise { - if (!sqliteTableExists(sqlitePath, "plugins")) return; - await acquireSqliteMigrationStateLock(db); const canonicalProjectPath = resolve(projectPath); - const migrationKey = `project-plugins:${canonicalProjectPath}`; + const migrationKey = projectPluginSqliteMigrationKey(projectPath); + await acquireSqliteMigrationStateLock(db); await ensureMigrationStateTable(db); await db.execute(sql`SELECT pg_advisory_xact_lock(hashtextextended(${migrationKey}, 0))`); - const completed = (await db.execute(sql` - SELECT 1 AS complete - FROM public.${sql.identifier(SQLITE_MIGRATION_STATE_TABLE)} - WHERE migration_key = ${migrationKey} AND status = 'complete' - LIMIT 1 - `)) as unknown as Array<{ complete: number }>; + const completed = await isSqliteMigrationComplete(db, migrationKey); /* FNXC:PluginLegacyMigration 2026-07-14-23:51: Retained SQLite is immutable cutover evidence, not a recurring authority. Once a project's plugin rows have been split into PostgreSQL install metadata and path-scoped state, a durable marker prevents later edits to fusion.db from changing live plugin behavior on restart. */ - if (completed.length > 0) return; + if (completed) return; + if (!sqliteTableExists(sqlitePath, "plugins")) return; const sqlite = openSqlite(sqlitePath); let rows: LegacyProjectPluginMigrationRow[]; try { diff --git a/packages/core/src/postgres/startup-factory.ts b/packages/core/src/postgres/startup-factory.ts index 8dc5bad7eb..91118fcf93 100644 --- a/packages/core/src/postgres/startup-factory.ts +++ b/packages/core/src/postgres/startup-factory.ts @@ -874,7 +874,31 @@ export async function createTaskStoreForBackend( try { const fusionDir = join(rootDir, ".fusion"); const legacySqlitePath = join(fusionDir, "fusion.db"); - if (existsSync(legacySqlitePath)) { + const { + migrateSqliteToPostgres, + defaultMigrationSources, + formatMigrationProgress, + isSqliteMigrationComplete, + completeSqliteMigration, + recordSqliteMigrationComplete, + CENTRAL_SQLITE_MIGRATION_KEY, + } = await import("./sqlite-migrator.js"); + const fallbackProjectId = fallbackProjectIdForRoot(rootDir); + let migrationProjectId = options.projectId + ?? (await lookupRegisteredProjectIdByPath(connections.migration, rootDir)) + ?? fallbackProjectId; + let migrationKey = `project:${migrationProjectId}`; + let migrationComplete = await isSqliteMigrationComplete(connections.migration, migrationKey); + + /* + FNXC:PostgresMigration 2026-07-22-12:00: + A completed source marker, not a retained backup's existence, is the + terminal cutover boundary. Resolve identity from explicit options or the + PostgreSQL registry before considering legacy SQLite; this makes retained + fusion.db and archive.db inert on every later CLI, dashboard, desktop, + and engine boot. + */ + if (!migrationComplete && existsSync(legacySqlitePath)) { let globalDir = options.globalSettingsDir; if (!globalDir) { try { @@ -885,10 +909,19 @@ export async function createTaskStoreForBackend( } } - let migrationProjectId = options.projectId - ?? (await lookupRegisteredProjectIdByPath(connections.migration, rootDir)); + let resolvedFromLegacyCentral = false; + const centralMigrationComplete = await isSqliteMigrationComplete( + connections.migration, CENTRAL_SQLITE_MIGRATION_KEY, + ); const legacyCentralPath = globalDir ? join(globalDir, "fusion-central.db") : undefined; - if (!migrationProjectId && legacyCentralPath && existsSync(legacyCentralPath) && isValidSqliteDatabaseFile(legacyCentralPath)) { + /* + FNXC:PostgresMigration 2026-07-22-12:00: + Central identity recovery is itself a legacy SQLite read, so its + independent completion marker must gate it even when project migration + remains pending. A completed central backup is never reopened merely to + resolve a fallback project key. + */ + if (!centralMigrationComplete && migrationProjectId === fallbackProjectId && legacyCentralPath && existsSync(legacyCentralPath) && isValidSqliteDatabaseFile(legacyCentralPath)) { const { DatabaseSync } = await import("../sqlite-adapter.js"); // FNXC:LegacySqliteBoundary 2026-07-14-18:42: central identity lookup is migration-only and read-only. const legacyCentral = new DatabaseSync(legacyCentralPath, { readOnly: true }); @@ -896,15 +929,14 @@ export async function createTaskStoreForBackend( const row = legacyCentral.prepare(`SELECT id FROM projects WHERE path = ? LIMIT 1`).get(rootDir) as | { id: string } | undefined; - migrationProjectId = row?.id; + migrationProjectId = row?.id ?? migrationProjectId; + resolvedFromLegacyCentral = Boolean(row?.id); } catch { // A pre-registry central database leaves legacy single-project startup unbound. } finally { legacyCentral.close(); } } - const fallbackProjectId = fallbackProjectIdForRoot(rootDir); - migrationProjectId ??= fallbackProjectId; if (migrationProjectId !== fallbackProjectId) { try { await rekeyFallbackProjectPartition(connections.migration, fallbackProjectId, migrationProjectId); @@ -944,9 +976,8 @@ export async function createTaskStoreForBackend( already populated) still avoid opening SQLite entirely. It runs only on the empty-PG path where one-time auto-migration is considered. */ - const migrationKey = `project:${migrationProjectId ?? rootDir}`; - const { migrateSqliteToPostgres, migrateLegacyProjectPluginRows, defaultMigrationSources, formatMigrationProgress, isSqliteMigrationComplete, completeSqliteMigration, recordSqliteMigrationComplete, CENTRAL_SQLITE_MIGRATION_KEY } = await import("./sqlite-migrator.js"); - const migrationComplete = await isSqliteMigrationComplete(connections.migration, migrationKey); + migrationKey = `project:${migrationProjectId}`; + migrationComplete = await isSqliteMigrationComplete(connections.migration, migrationKey); if (!migrationComplete && isValidSqliteDatabaseFile(legacySqlitePath)) { // The central (global-dir) source is optional: when no global dir is // resolvable (e.g. tests without an explicit dir), migrate only the @@ -955,9 +986,6 @@ export async function createTaskStoreForBackend( FNXC:PostgresMultiProjectCutover 2026-07-14-11:18: The central SQLite database is cluster-global, not a per-project source. Migrate and verify it once, then exclude it from later registered-project cutovers so mutable global rows are not compared with each project's accumulated PostgreSQL state. */ - const centralMigrationComplete = await isSqliteMigrationComplete( - connections.migration, CENTRAL_SQLITE_MIGRATION_KEY, - ); const sources = defaultMigrationSources(fusionDir, globalDir ?? join(fusionDir, "__no-global-dir__")) .filter((source) => !centralMigrationComplete || source.pgSchema !== "central") .filter((source) => existsSync(source.sqlitePath) && isValidSqliteDatabaseFile(source.sqlitePath)); @@ -1037,6 +1065,18 @@ export async function createTaskStoreForBackend( rootDir, }); } + /* + FNXC:PostgresMigration 2026-07-22-12:00: + A legacy-central identity can be marked complete only after the + same canonical rootDir resolves to it in PostgreSQL. Otherwise the + next rootDir-only boot would need SQLite to rediscover its key. + */ + if (resolvedFromLegacyCentral) { + const registeredId = await lookupRegisteredProjectIdByPath(connections.migration, rootDir); + if (registeredId !== migrationProjectId) { + throw new Error(`legacy project identity ${migrationProjectId} was not materialized for ${rootDir} before migration completion`); + } + } await completeSqliteMigration(connections.migration, migrationKey); if (sources.some((source) => source.pgSchema === "central")) { await recordSqliteMigrationComplete( @@ -1059,18 +1099,16 @@ export async function createTaskStoreForBackend( log.log(`startup-factory: SQLite → PostgreSQL auto-migration complete (${migratedRows} row(s) across ${report.tables.length} table(s))`); } } - /* - FNXC:PluginLegacyMigration 2026-07-15-02:09: - The retained-SQLite plugin bridge requires schema-marker and central plugin writes, so steady-state startup must run it through the privileged migration connection before that connection is replaced by the project-scoped fusion_runtime role. This bridge remains independently marker-gated because projects that completed the core cutover before plugin migration existed still need their plugin state recovered; every runtime surface receives the already-migrated store and PluginStore.init stays DDL-free. - */ - if (isValidSqliteDatabaseFile(legacySqlitePath)) { - await migrateLegacyProjectPluginRows( - connections.migration, - legacySqlitePath, - rootDir, - ); - } } + /* + FNXC:PostgresMigration 2026-07-22-12:00: + Core and plugin sources have independent completion markers. Dispatch the + plugin bridge after the core branch so a completed project cutover cannot + suppress a still-pending plugin adoption; the bridge itself checks its + marker before validating or opening the retained SQLite file. + */ + const { migrateLegacyProjectPluginRows } = await import("./sqlite-migrator.js"); + await migrateLegacyProjectPluginRows(connections.migration, legacySqlitePath, rootDir); } catch (err) { await connections.close().catch(() => undefined); await stopEmbeddedRuntime(