From 99870ba3297e7f227bcaed66b96dcddee4fc3583 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Tue, 14 Jul 2026 09:09:56 -0700 Subject: [PATCH] fix(core): recover partial PostgreSQL migrations --- .changeset/fix-null-project-id-migration.md | 7 ++ .../postgres/sqlite-migrator.test.ts | 71 ++++++++++++++-- .../startup-factory-integration.test.ts | 37 +++++++++ packages/core/src/postgres/sqlite-migrator.ts | 83 +++++++++++++++++-- 4 files changed, 186 insertions(+), 12 deletions(-) create mode 100644 .changeset/fix-null-project-id-migration.md diff --git a/.changeset/fix-null-project-id-migration.md b/.changeset/fix-null-project-id-migration.md new file mode 100644 index 0000000000..c1d7884126 --- /dev/null +++ b/.changeset/fix-null-project-id-migration.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Fix startup migration for legacy project rows whose project ID is null. +category: fix +dev: Bound SQLite cutovers now override nullable or stale source project IDs with the resolved registry identity. diff --git a/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts b/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts index 357f15f9e5..80982c05c9 100644 --- a/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts +++ b/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts @@ -37,6 +37,7 @@ import { migrateSqliteToPostgres, toSnakeCase, } from "../../postgres/sqlite-migrator.js"; +import { applySchemaBaseline } from "../../postgres/schema-applier.js"; const PG_TEST_URL_BASE = process.env.FUSION_PG_TEST_URL_BASE ?? "postgresql://localhost:5432"; @@ -78,7 +79,8 @@ CREATE TABLE IF NOT EXISTS tasks ( customFields TEXT DEFAULT '{}', deletedAt TEXT, createdAt TEXT NOT NULL, - updatedAt TEXT NOT NULL + updatedAt TEXT NOT NULL, + projectId TEXT ); `; @@ -472,6 +474,9 @@ pgDescribe("SQLite-to-PostgreSQL migrator", () => { /* FNXC:AutomationIsolation 2026-07-13-22:37: Legacy project databases do not carry project_id on automation rows. Migration must inject the resolved registry identity before verification so bound automation stores and cron runners see only their project's schedules, including when legacy automation IDs overlap. + + FNXC:AutomationIsolation 2026-07-14-08:52: + Real upgraded SQLite databases already have a nullable projectId column whose legacy rows can still be NULL. The resolved registry identity remains authoritative during the one-project-file cutover; migration must override that nullable source column instead of copying NULL into PostgreSQL's required project_id partition. */ it("injects and verifies the project partition for migrated automations", async () => { const sqlitePath = join(ctx!.fusionDir, "fusion.db"); @@ -480,15 +485,28 @@ pgDescribe("SQLite-to-PostgreSQL migrator", () => { legacy.exec(`CREATE TABLE automations ( id TEXT PRIMARY KEY, name TEXT NOT NULL, scheduleType TEXT NOT NULL, cronExpression TEXT NOT NULL, command TEXT NOT NULL, - createdAt TEXT NOT NULL, updatedAt TEXT NOT NULL + createdAt TEXT NOT NULL, updatedAt TEXT NOT NULL, + projectId TEXT )`); - legacy.prepare(`INSERT INTO automations VALUES (?, ?, ?, ?, ?, ?, ?)`).run( - "auto-shared", "Nightly", "cron", "0 0 * * *", "pnpm check", "2026-06-01", "2026-06-01", + legacy.prepare(`INSERT INTO automations VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run( + "auto-shared", "Nightly", "cron", "0 0 * * *", "pnpm check", "2026-06-01", "2026-06-01", null, + ); + legacy.prepare(`INSERT INTO automations VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run( + "auto-stale", "Weekly", "cron", "0 0 * * 0", "pnpm audit", "2026-06-01", "2026-06-01", "stale-project", ); } finally { legacy.close(); } + await applySchemaBaseline(ctx!.db); + await ctx!.db.execute(sql` + INSERT INTO project.automations + (project_id, id, name, schedule_type, cron_expression, command, created_at, updated_at) + VALUES + ('stale-project', 'auto-stale', 'Weekly', 'cron', '0 0 * * 0', 'pnpm audit', '2026-06-01', '2026-06-01'), + ('project-a', 'auto-stale', 'Weekly', 'cron', '0 0 * * 0', 'pnpm audit', '2026-06-01', '2026-06-01') + `); + for (const projectId of ["project-a", "project-b"]) { const report = await migrateTest( ctx!.db, @@ -496,16 +514,57 @@ pgDescribe("SQLite-to-PostgreSQL migrator", () => { { projectId }, ); expect(report.tables.find((table) => table.table === "automations")).toEqual( - expect.objectContaining({ sourceRows: 1, targetRows: 1, verified: true }), + expect.objectContaining({ sourceRows: 2, targetRows: 2, verified: true }), ); } const rows = (await ctx!.db.execute(sql` - SELECT project_id, id FROM project.automations WHERE id = 'auto-shared' ORDER BY project_id + SELECT project_id, id FROM project.automations + WHERE id IN ('auto-shared', 'auto-stale') + ORDER BY project_id, id `)) as unknown as Array<{ project_id: string; id: string }>; expect(rows).toEqual([ { project_id: "project-a", id: "auto-shared" }, + { project_id: "project-a", id: "auto-stale" }, { project_id: "project-b", id: "auto-shared" }, + { project_id: "project-b", id: "auto-stale" }, + ]); + }); + + /* + FNXC:PostgresMigrationRetry 2026-07-14-09:06: + Retrying after the former non-transactional migrator must repair a globally keyed row copied under a NULL project partition. Reconciliation may replace only an exact migrated-content match, so unrelated PostgreSQL state remains untouched. + */ + it("re-keys an exact globally keyed row left under a null project partition", async () => { + await applySchemaBaseline(ctx!.db); + await ctx!.db.execute(sql` + INSERT INTO project.tasks + (id, project_id, title, description, "column", dependencies, steps, comments, + custom_fields, deleted_at, created_at, updated_at) + VALUES + ('FN-100', NULL, 'First task', 'desc', 'todo', + '[{"taskId":"FN-99","type":"blocks"}]'::jsonb, + '[{"id":"s1","name":"step one"}]'::jsonb, + '[{"author":"agent","body":"hello"}]'::jsonb, + '{"priority":"high","labels":["a","b"]}'::jsonb, + NULL, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z') + `); + + const report = await migrateTest( + ctx!.db, + [{ sqlitePath: join(ctx!.fusionDir, "fusion.db"), pgSchema: "project" as const }], + { projectId: "project-retry" }, + ); + + expect(report.tables.find((table) => table.table === "tasks")).toEqual( + expect.objectContaining({ sourceRows: 2, targetRows: 2, verified: true }), + ); + const rows = await ctx!.db.execute(sql` + SELECT id, project_id FROM project.tasks ORDER BY id + `) as unknown as Array<{ id: string; project_id: string }>; + expect(rows).toEqual([ + { id: "FN-100", project_id: "project-retry" }, + { id: "FN-101", project_id: "project-retry" }, ]); }); diff --git a/packages/core/src/__tests__/postgres/startup-factory-integration.test.ts b/packages/core/src/__tests__/postgres/startup-factory-integration.test.ts index ecf6d549a6..9d8dddfed3 100644 --- a/packages/core/src/__tests__/postgres/startup-factory-integration.test.ts +++ b/packages/core/src/__tests__/postgres/startup-factory-integration.test.ts @@ -21,6 +21,7 @@ import { createTaskStoreForBackend } from "../../postgres/startup-factory.js"; import { mkdirSync } from "node:fs"; import { DatabaseSync } from "../../sqlite-adapter.js"; import postgres from "postgres"; +import { sql } from "drizzle-orm"; const PG_TEST_URL_BASE = process.env.FUSION_PG_TEST_URL_BASE ?? "postgresql://localhost:5432"; @@ -152,10 +153,17 @@ pgDescribe("startup-factory: external PostgreSQL boot (integration)", () => { */ it("auto-migrates legacy SQLite data into an empty PostgreSQL database on first boot", async () => { rootDir = await mkdtemp(join(tmpdir(), "startup-factory-automig-")); + const globalDir = join(rootDir, "global"); dbName = uniqueDbName(); adminExec(`CREATE DATABASE "${dbName}"`); const testUrl = `${PG_TEST_URL_BASE}/${dbName}`; + /* + FNXC:PostgresMigration 2026-07-14-08:52: + Startup migration must bind real upgraded SQLite automation rows whose nullable projectId is still NULL to the registry project before PostgreSQL enforces its required partition. This fixture reproduces the pnpm dev startup crash, not only the simpler legacy shape where the source column is absent. + */ + seedLegacyRegistry(globalDir, [{ id: "project-migrated", path: rootDir }]); + // Seed a minimal legacy fusion.db with one live task. const fusionDir = join(rootDir, ".fusion"); mkdirSync(fusionDir, { recursive: true }); @@ -168,16 +176,37 @@ pgDescribe("startup-factory: external PostgreSQL boot (integration)", () => { "column" TEXT NOT NULL, createdAt TEXT NOT NULL, updatedAt TEXT NOT NULL + ); + CREATE TABLE automations ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + scheduleType TEXT NOT NULL, + cronExpression TEXT NOT NULL, + command TEXT NOT NULL, + createdAt TEXT NOT NULL, + updatedAt TEXT NOT NULL, + projectId TEXT );`); legacy.prepare( `INSERT INTO tasks (id, title, description, "column", createdAt, updatedAt) VALUES (?, ?, ?, ?, ?, ?)`, ).run("FN-MIG-1", "Legacy task", "migrated from sqlite", "todo", "2026-06-01T00:00:00Z", "2026-06-01T00:00:00Z"); + legacy.prepare(`INSERT INTO automations VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run( + "automation-migrated", + "Nightly", + "cron", + "0 0 * * *", + "pnpm check", + "2026-06-01T00:00:00Z", + "2026-06-01T00:00:00Z", + null, + ); } finally { legacy.close(); } const first = await createTaskStoreForBackend({ rootDir, + globalSettingsDir: globalDir, env: { DATABASE_URL: testUrl }, }); expect(first).not.toBeNull(); @@ -199,6 +228,13 @@ pgDescribe("startup-factory: external PostgreSQL boot (integration)", () => { expect(notice!.tables).toBeGreaterThanOrEqual(1); expect(notice!.sqliteBackups).toContain(join(fusionDir, "fusion.db")); expect(notice!.dismissed).toBe(false); + + const migratedAutomations = (await first!.asyncLayer.db.execute(sql` + SELECT project_id, id FROM project.automations WHERE id = 'automation-migrated' + `)) as unknown as Array<{ project_id: string; id: string }>; + expect(migratedAutomations).toEqual([ + { project_id: "project-migrated", id: "automation-migrated" }, + ]); } finally { await first!.shutdown(); } @@ -206,6 +242,7 @@ pgDescribe("startup-factory: external PostgreSQL boot (integration)", () => { // Second boot: PG is no longer empty — must NOT attempt to re-migrate. const second = await createTaskStoreForBackend({ rootDir, + globalSettingsDir: globalDir, env: { DATABASE_URL: testUrl }, }); expect(second).not.toBeNull(); diff --git a/packages/core/src/postgres/sqlite-migrator.ts b/packages/core/src/postgres/sqlite-migrator.ts index 7a7c6f9e5e..c68e36e3ec 100644 --- a/packages/core/src/postgres/sqlite-migrator.ts +++ b/packages/core/src/postgres/sqlite-migrator.ts @@ -456,10 +456,13 @@ async function buildMigrationPlan( sqlite, targetColumnsByTable, ); + /* + FNXC:PostgresMigration 2026-07-14-08:52: + A legacy per-project SQLite table can already contain a nullable projectId column while its existing rows still hold NULL. The registry identity resolved for this one-project-file cutover is authoritative whenever the PostgreSQL target is partitioned, so insertion and verification must override absent, NULL, or stale source project IDs instead of copying them into the required project_id partition. + */ const partitionProjectId = projectId && source.pgSchema !== CENTRAL_SCHEMA && - !cols.some((column) => column.pgName === "project_id") && targetColumnNames.has("project_id") ? projectId : undefined; @@ -898,7 +901,12 @@ async function migrateTable( const rowCountOk = targetRows === sourceRows; let contentOk = true; if (rowCountOk && sourceRows > 0) { - const sourceChecksum = computeSourceContentChecksum(sqlite, plan.table, insertableCols); + const sourceChecksum = computeSourceContentChecksum( + sqlite, + plan.table, + insertableCols, + plan.partitionProjectId, + ); const targetChecksum = await computeTargetContentChecksum( db, plan.pgSchema, @@ -955,9 +963,10 @@ async function insertBatch( hasIdentityCol: boolean, ): Promise { if (rows.length === 0) return 0; + const hasMappedProjectId = cols.some((column) => column.pgName === "project_id"); const colList = [ ...cols.map((c) => quoteIdent(c.pgName)), - ...(plan.partitionProjectId ? [quoteIdent("project_id")] : []), + ...(plan.partitionProjectId && !hasMappedProjectId ? [quoteIdent("project_id")] : []), ].join(", "); const schemaQualifiedTable = `${quoteIdent(plan.pgSchema)}.${quoteIdent(plan.pgTable)}`; // OVERRIDING SYSTEM VALUE lets us write explicit values into GENERATED ALWAYS @@ -980,9 +989,68 @@ async function insertBatch( return sql`${value}`; }; + /* + FNXC:PostgresMigrationRetry 2026-07-14-09:06: + The former non-transactional migrator could commit a source row under its NULL or stale project_id before a later table aborted startup. On retry, re-key only a stale-partition row whose complete migrated column set still matches the SQLite source. If an identical authoritative composite-key row already exists, remove only its stale duplicate; updating globally keyed rows in place preserves dependent rows and avoids foreign-key cascades. + */ + if (plan.partitionProjectId && hasMappedProjectId) { + const staleRows = rows.filter( + (row) => row.project_id !== plan.partitionProjectId, + ); + if (staleRows.length > 0) { + const comparisonsFor = ( + alias: string, + row: Readonly>, + includeProjectId: boolean, + ) => cols + .filter((column) => includeProjectId || column.pgName !== "project_id") + .map((column) => { + const targetColumn = sql.raw(`${alias}.${quoteIdent(column.pgName)}`); + return sql`${targetColumn} IS NOT DISTINCT FROM ${buildCell(column, row[column.pgName])}`; + }); + const exactStalePredicates = staleRows.map((row) => + sql`(${sql.join(comparisonsFor("target", row, true), sql` AND `)})`, + ); + const duplicatePredicates = staleRows.map((row) => { + const authoritativeComparisons = comparisonsFor("authoritative", row, false); + return sql`( + ${sql.join(comparisonsFor("target", row, true), sql` AND `)} + AND EXISTS ( + SELECT 1 FROM ${sql.raw(schemaQualifiedTable)} AS authoritative + WHERE authoritative.project_id = ${plan.partitionProjectId} + AND ${sql.join(authoritativeComparisons, sql` AND `)} + ) + )`; + }); + const removedDuplicates = (await db.execute(sql` + DELETE FROM ${sql.raw(schemaQualifiedTable)} AS target + WHERE ${sql.join(duplicatePredicates, sql` OR `)} + RETURNING 1 + `)) as unknown as { length?: number }; + const rekeyed = (await db.execute(sql` + UPDATE ${sql.raw(schemaQualifiedTable)} AS target + SET project_id = ${plan.partitionProjectId} + WHERE ${sql.join(exactStalePredicates, sql` OR `)} + RETURNING 1 + `)) as unknown as { length?: number }; + const repairedCount = Number(removedDuplicates?.length ?? 0) + Number(rekeyed?.length ?? 0); + if (repairedCount > 0) { + log.log( + `Reconciled ${repairedCount} stale partition row(s) in ${plan.pgSchema}.${plan.pgTable}`, + ); + } + } + } + const valueRowsBuilt = rows.map((row) => { - const cells = cols.map((c) => buildCell(c, row[c.pgName])); - if (plan.partitionProjectId) cells.push(sql`${plan.partitionProjectId}`); + const cells = cols.map((c) => + c.pgName === "project_id" && plan.partitionProjectId + ? sql`${plan.partitionProjectId}` + : buildCell(c, row[c.pgName]), + ); + if (plan.partitionProjectId && !hasMappedProjectId) { + cells.push(sql`${plan.partitionProjectId}`); + } return sql`(${sql.join(cells, sql`, `)})`; }); @@ -1171,6 +1239,7 @@ function computeSourceContentChecksum( sqlite: DatabaseSync, table: string, cols: readonly ColumnMapping[], + partitionProjectId?: string, ): string { if (cols.length === 0) return ""; const selectCols = cols.map((c) => quoteIdent(c.sqliteName)).join(", "); @@ -1182,7 +1251,9 @@ function computeSourceContentChecksum( const hash = createHash("md5"); for (const row of rows) { for (const col of cols) { - const converted = convertValue(row[col.sqliteName], col.type, col.nullJsonbFallback); + const converted = col.pgName === "project_id" && partitionProjectId + ? partitionProjectId + : convertValue(row[col.sqliteName], col.type, col.nullJsonbFallback); hash.update(canonicalizeCell(converted)); hash.update("\u0001"); // cell separator }